{ "dataset": { "meta": { "domain": "resilience", "domain_slug": "resilience", "domain_number": 8, "title": "Apeiris Resilience Control Matrix", "description": "Apeiris Resilience Control Matrix: 48 machine-readable controls across 6 layers.", "version": "1.1.0", "published": "2026-07-02", "layers": 6, "controls_count": 48, "baseline_controls": [ "RV-01", "RV-08", "RP-01", "RP-08", "RO-01", "RO-08", "FO-01", "FO-08", "RE-01" ], "canonical_prefix": "apeiris://resilience/controls/", "attestation_artifact": "ResilienceAttestation", "attestation_control": "RG-08", "alias_domain": "resilienceverifier.ai", "frameworks": [ "anthropic_rsp", "aws_reliability", "cis_controls_v8", "cloudflare_resilience", "cobit_dss04", "dora", "enisa_ai", "google_sre", "iso_22301", "iso_27031", "microsoft_azure_resil", "nist_800_160_v2", "nist_csf", "nist_sp800_34", "openai_preparedness" ], "lenses": [ "grc_auditor", "it_operations", "security_architect", "site_reliability", "business_continuity" ], "license": "CC BY 4.0", "source": "https://apeiris.ai/domains/resilience/", "integration_endpoint": "https://apeiris.ai/integration/domains/resilience-controls-full.json", "source_freshness": { "status": "current", "checked_on": "2026-06-29", "review_cadence": "quarterly" }, "baseline_control_count": 9, "generated_at": "2026-06-29T00:00:00.000Z", "subtitle": "apeiris.ai/domains/resilience \u2014 Apeiris Resilience", "site": "https://apeiris.ai/domains/resilience", "corpus_url": "https://apeiris.ai/integration/domains/resilience-controls-full.json", "schema_version": "1.1.0", "schema_extended_on": "2026-06-29", "extended_schema_fields": [ "validation_objective", "evidence_required", "machine_tests", "human_review", "blocking_effect", "normative_status", "anti_patterns", "update_status" ] }, "controls": [ { "id": "RV-01", "layer": "RV", "plane": "lifecycle", "name": "AI System RTO/RPO Definition and Validation", "plain": "Every AI system in production must have formally documented Recovery Time Objectives and Recovery Point Objectives that are validated against actual recovery capabilities, business impact analysis findings, and regulatory obligations on at least an annual basis.", "threat": { "tags": [ "unvalidated-recovery-objectives", "sla-breach", "undefined-rto-rpo", "operational-blindness" ], "desc": "Without defined and validated RTO/RPO targets, AI system recovery efforts lack measurable goals, making it impossible to determine whether recovery is adequate or compliant. Organizations routinely discover during an actual incident that informal recovery assumptions are far more optimistic than real capabilities. Regulators in financial services and critical infrastructure now require documented and tested recovery objectives for digital operational resilience." }, "standard": [ { "id": "iso_22301", "section": "\u00a78.2.2", "title": "Business impact analysis" }, { "id": "nist_sp800_34", "section": "\u00a73.2", "title": "Recovery objectives determination" }, { "id": "dora", "section": "Art. 12", "title": "Backup policies and recovery objectives" }, { "id": "cobit_dss04", "section": "DSS04.02", "title": "Define and implement business continuity strategy" }, { "id": "enisa_ai", "section": "Report (2023-03-14)", "title": "Cybersecurity of AI and Standardisation \u2014 document-level reference" } ], "sources": [ { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RV-01 AI System RTO/RPO Definition and Validation control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RV-01 AI System RTO/RPO Definition and Validation control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RV-01 AI System RTO/RPO Definition and Validation control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RV-01 AI System RTO/RPO Definition and Validation control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RV-01 AI System RTO/RPO Definition and Validation control.", "reviewed_on": "2026-07-01" }, { "id": "enisa_ai_cybersec_2023", "title": "ENISA \u2014 Cybersecurity of AI and Standardisation", "authority": "European Union Agency for Cybersecurity (ENISA)", "source_type": "guidance", "normative_force": "best-practice", "version": "2023", "published_on": "2023-03-14", "retrieved_on": "2026-06-29", "canonical_url": "https://www.enisa.europa.eu/publications/cybersecurity-of-ai-and-standardisation", "license": "open-access", "status": "current", "flagship": false, "source_id": "enisa_ai_cybersec_2023", "relationship": "informative_reference", "rationale": "ENISA report \"Cybersecurity of AI and Standardisation\" (2023-03-14) surveys AI cybersecurity and the applicable standardisation landscape, informing the apeiris://resilience/controls/RV-01 AI System RTO/RPO Definition and Validation control at document level.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Conduct business impact analysis per AI system, document RTO/RPO in a recovery objectives register, and validate objectives against measured recovery capability through tabletop exercises and controlled failover tests at minimum annually.", "steps": [ "Inventory all production AI systems and classify each by criticality tier based on business impact analysis findings.", "Document RTO (maximum acceptable downtime) and RPO (maximum acceptable data loss window) for each AI system in a versioned recovery objectives register.", "Validate declared objectives annually by executing controlled recovery tests under realistic failure scenarios and recording actual vs. target recovery metrics.", "Review and update objectives after each significant architectural change, major incident, or business priority shift.", "Obtain formal sign-off from business owners and legal/compliance on all objectives to ensure regulatory alignment." ], "site_reliability": { "summary": "RTO/RPO values are the engineering contracts you build reliability against. Ensure every SLO and runbook references the validated objectives from the register.", "actions": [ "Pull recovery objectives register into SLO definitions and alert thresholds.", "Instrument recovery telemetry to measure actual vs. target RTO and RPO in real-time during incidents.", "Flag any architectural change that would violate documented objectives before deployment." ], "failure_signals": [ "Recovery tests consistently exceed documented RTO by more than 20%.", "RPO objectives are not reflected in backup frequency configuration.", "Recovery objectives register has not been reviewed in over 12 months." ] }, "it_operations": { "summary": "Operations teams must translate RTO/RPO targets into concrete runbook steps, tooling configurations, and monitoring thresholds that make objectives achievable under pressure.", "actions": [ "Align backup schedules and retention policies to RPO for each AI system tier.", "Configure monitoring alerts to trigger when recovery time is projected to exceed RTO.", "Maintain runbooks referencing specific recovery time checkpoints." ], "failure_signals": [ "Backup schedules do not match RPO requirements for any critical AI system.", "No runbook checkpoint exists at the 50% mark of the RTO window.", "Actual recovery time has exceeded RTO in the last two exercises without documented remediation." ] }, "grc_auditor": { "summary": "The recovery objectives register and validation records are the primary audit artifacts for demonstrating that RTO/RPO obligations are defined, realistic, and met.", "actions": [ "Request current recovery objectives register and verify all production AI systems are covered.", "Examine the most recent validation test report for each system and compare actual recovery times to documented objectives.", "Confirm business owner and compliance sign-off is present and current." ], "metrics": [ "Recovery objectives coverage: target 100% of production AI systems.", "RTO validation pass rate: target \u226595% of systems meeting objectives in most recent test.", "Objectives review currency: target 100% reviewed within 12 months." ], "failure_signals": [ "Any production AI system lacks documented objectives.", "Validation test results show repeated RTO overruns with no remediation plan.", "Business owner sign-off is absent or more than 18 months old." ] }, "business_continuity": { "summary": "RTO/RPO definitions must be grounded in business impact analysis and kept current as business priorities evolve, not treated as one-time engineering decisions.", "actions": [ "Lead annual business impact analysis sessions with AI system owners to confirm objective tiers.", "Ensure objectives account for downstream business processes that depend on each AI system.", "Incorporate AI system recovery objectives into the enterprise business continuity plan." ], "failure_signals": [ "Recovery objectives were set by engineering teams without business owner input.", "Business impact analysis has not been refreshed following a significant product or process change.", "Dependent business processes are not mapped to AI system recovery objectives." ] }, "security_architect": { "summary": "Recovery objectives are security-relevant: an AI system restored outside its RTO/RPO window may be brought back with stale credentials, expired certificates, or unpatched images. Validate that recovery paths preserve the security posture the objectives assume.", "actions": [ "Review the recovery objectives register to confirm restore paths rehydrate secrets, keys, and certificates within the RTO window.", "Require that recovery validation tests include verification that restored AI systems boot with current security baselines and patch levels.", "Flag any RTO/RPO change that would force recovery through less-hardened standby infrastructure." ], "failure_signals": [ "Restored systems come back with credentials or certificates that were rotated during the outage.", "Recovery tests validate timing but never verify security configuration of the restored environment.", "Standby environments used to meet RTO run older, unhardened images." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Most enterprises document high-level continuity objectives but have not validated AI-specific RTO/RPO through controlled testing." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "cloud-native" ], "implementers": [ "Business Continuity Team", "IT Operations", "Platform Engineering" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a78.2.2", "fit": "direct", "rationale": "ISO 22301:2019 \u00a78.2.2 requires a business impact analysis that determines the impacts of disruption and sets prioritized timeframes for resuming activities. Validated RTO/RPO targets for production AI systems are the direct output this clause requires, grounded in measured impact rather than assumption (\u00a78.3 covers continuity strategies and solutions).", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.2", "fit": "direct", "rationale": "NIST SP 800-34 Rev 1 \u00a73.2 defines the process for establishing recovery priorities and objectives as part of contingency planning. The guidance explicitly requires RTO and RPO determination as foundational steps before any recovery strategy is selected or tested.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 12", "fit": "direct", "rationale": "EU DORA Article 12 imposes binding requirements on financial entities to define recovery time and recovery point objectives for ICT systems supporting critical functions, with documentation subject to supervisory review. Organizations deploying AI in financial services must treat this as a legal obligation, not a best practice.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.02", "fit": "direct", "rationale": "COBIT 2019 DSS04.02 requires defining a business continuity strategy that includes recovery objectives aligned to business impact analysis. It provides the governance accountability model for ensuring objectives are owned, approved, and periodically reviewed by appropriate stakeholders.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "RC.RP-01", "fit": "direct", "rationale": "NIST CSF 2.0 RC.RP-01 requires that recovery plans reflect documented objectives and are executed against measurable targets. Validated RTO/RPO definitions are the prerequisite artifact for any conformant recovery planning activity under the Recover function.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "REL 9 \u2013 Back up data to satisfy recovery time objectives", "fit": "direct", "rationale": "RV-01 directly requires that every production AI system have formally documented and validated RTO and RPO targets aligned to business impact analysis findings. AWS Well-Architected Reliability Pillar REL-09 prescribes that workloads define and periodically test recovery time and recovery point objectives, including backup and restore procedures that meet those targets. This control and REL-09 share identical scope: defining measurable recovery objectives and verifying that infrastructure can actually meet them.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "SLO/Error Budget Policy \u2013 Defining and enforcing availability targets", "fit": "direct", "rationale": "RV-01 requires measurable, validated recovery objectives for production AI systems. Google SRE Service Level Objective and Error Budget Policy practices define the canonical operational framework for translating business availability needs into measurable targets \u2014 exactly the purpose of RTO/RPO definition. The error budget directly operationalizes acceptable downtime windows (mapping to RTO), and SRE error budget policies prescribe consequences and escalation paths when those windows are breached.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "BCDR \u2013 Recovery objectives (RTO/RPO) definition and validation", "fit": "direct", "rationale": "RV-01 mandates formal documentation of Recovery Time Objectives and Recovery Point Objectives validated against actual recovery capabilities. Microsoft Azure Resiliency BCDR guidance directly prescribes a structured process for defining RTO and RPO per workload, mapping them to Azure architecture patterns such as availability zones and geo-redundancy, and validating through failover testing. The Azure reliability documentation treats RTO/RPO definition as a prerequisite for BCDR design.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "enisa_ai", "requirement_id": "Cybersecurity of AI and Standardisation (2023) \u2014 document-level", "fit": "adjacent", "rationale": "ENISA's 'Cybersecurity of AI and Standardisation' report (14 March 2023) surveys the cybersecurity of AI systems and the standardisation landscape that applies to them, including how operational-resilience standards map onto AI-specific assets. The report has no numbered resilience clause; the citation is document-level and informative for aligning AI recovery objectives with EU standardisation work.", "normative_force": "best-practice", "source_version": "2023", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" } ], "canonical_id": "apeiris://resilience/controls/RV-01", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every production AI system must have a formally documented recovery objectives register containing its RTO and RPO values, grounded in a current business impact analysis, with validation evidence from controlled recovery tests demonstrating that actual recovery capability meets or exceeds those declared objectives within the last 12 months.", "evidence_required": [ "recovery_objectives_register with ai_system_id, criticality_tier, rto_minutes, rpo_minutes, bia_date, and business_owner_signoff_date for every production AI system", "controlled_recovery_test_report showing scenario_description, execution_date, declared_rto_minutes, declared_rpo_minutes, actual_rto_minutes, actual_rpo_minutes, and pass_fail verdict per system", "business_impact_analysis_document dated within 12 months referencing each AI system and the business process dependencies that drove objective setting", "formal_signoff_record from business_owner and compliance_reviewer confirming objectives are aligned with regulatory obligations" ], "machine_tests": [ "Query recovery_objectives_register \u2192 assert every production AI system has an entry with non-null rto_minutes, rpo_minutes, and bia_date within the last 12 months", "Compare recovery_objectives_register.rto_minutes to most_recent_test_report.actual_rto_minutes per system \u2192 assert actual \u2264 declared for \u226595% of systems", "Check business_owner_signoff_date per register entry \u2192 assert no entry is older than 18 months", "Validate backup_schedule_frequency per system \u2192 assert backup interval \u2264 rpo_minutes" ], "human_review": [ "Review the business impact analysis methodology to confirm AI system criticality tiers were determined through structured stakeholder input and not unilaterally assigned by engineering teams", "Assess whether declared RTO/RPO values reflect actual regulatory obligations for the AI system's deployment context, including any DORA, financial services, or critical infrastructure requirements", "Verify that recovery test scenarios were realistic and that the test environment was sufficiently representative of production conditions to make test results meaningful" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Setting RTO/RPO values based on assumed infrastructure capabilities rather than measured recovery tests, resulting in objectives that cannot be achieved under real failure conditions", "Applying a single RTO/RPO pair across all AI systems regardless of business criticality tier, causing over-investment in non-critical recovery and under-protection for critical inference workloads", "Documenting recovery objectives once at system launch and never refreshing them after architectural changes, model updates, or shifts in business process dependency", "Treating RTO/RPO as an engineering metric owned by the platform team without formal business owner and compliance sign-off", "Validating objectives only through tabletop exercises without any controlled technical recovery test that measures actual elapsed recovery time" ], "update_status": "current", "layer_code": "RV" }, { "id": "RV-02", "layer": "RV", "plane": "control", "name": "Resilience Control Verification Testing", "plain": "Resilience controls protecting AI systems must be regularly tested under simulated failure conditions to verify they function as designed, with test results documented and remediation tracked for any control that fails to perform as specified.", "threat": { "tags": [ "untested-controls", "false-assurance", "control-failure", "undetected-degradation" ], "desc": "Resilience controls that have never been tested under realistic failure conditions provide false assurance. Configuration drift, dependency changes, and software updates routinely invalidate controls that once worked, yet without testing the organization assumes they remain effective. Undiscovered control failures mean that when real incidents occur, the recovery infrastructure that was counted on simply does not activate." }, "standard": [ { "id": "iso_22301", "section": "\u00a78.5", "title": "Exercising and testing" }, { "id": "nist_sp800_34", "section": "\u00a73.5", "title": "Plan testing, training, and exercises (TT&E)" }, { "id": "nist_csf", "section": "RC.RP-02", "title": "Recovery actions are selected, scoped, prioritized, and performed" }, { "id": "cobit_dss04", "section": "DSS04.04", "title": "Exercise, test and review the BCP and DRP" } ], "sources": [ { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RV-02 Resilience Control Verification Testing control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RV-02 Resilience Control Verification Testing control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RV-02 Resilience Control Verification Testing control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RV-02 Resilience Control Verification Testing control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RV-02 Resilience Control Verification Testing control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Establish a resilience testing calendar covering all production AI systems; execute tests at defined intervals using scripted scenarios; record pass/fail status per control; raise tracked remediation items for any failure; confirm remediation through retest.", "steps": [ "Enumerate all resilience controls in scope for each AI system including failover, backup/restore, circuit breakers, rate limiting, and fallback routing.", "Define a test scenario library with at least three failure modes per control (single component failure, dependency failure, sustained load failure).", "Schedule and execute tests at minimum quarterly for tier-1 AI systems and semi-annually for lower-tier systems, capturing pass/fail evidence per test run.", "Raise a tracked remediation item for every control that fails to perform as specified, with an owner and resolution deadline.", "Retest all remediated controls within 30 days of remediation closure to confirm the fix is effective." ], "site_reliability": { "summary": "Resilience control testing is a core SRE practice. Automate what you can, schedule what you cannot, and treat every untested control as a latent incident.", "actions": [ "Build automated test harnesses for circuit breaker, failover, and backup/restore controls wherever possible.", "Integrate test execution into the deployment pipeline for controls that are affected by application changes.", "Track test coverage ratio and alert when a control has not been tested within its scheduled interval." ], "failure_signals": [ "Any tier-1 AI system resilience control has not been tested in more than 90 days.", "Automated test coverage is below 60% of enumerated controls.", "Open remediation items have exceeded their resolution deadline without extension approval." ] }, "it_operations": { "summary": "Operations teams own execution of scheduled resilience tests and must coordinate blackout windows, notify stakeholders, and ensure test environments are representative of production state.", "actions": [ "Maintain the resilience testing calendar and coordinate with application owners for each test window.", "Execute scripted test scenarios and record results in the resilience testing register.", "Escalate immediately when a test reveals a control failure that materially affects an active SLA." ], "failure_signals": [ "Testing calendar has gaps exceeding the scheduled interval for any critical AI system.", "Test results are not recorded with sufficient evidence to demonstrate what was tested and what outcome was observed.", "Control failures identified in testing are not escalated within the defined SLA." ] }, "grc_auditor": { "summary": "The resilience testing register and remediation log are the primary evidence artifacts that resilience controls are verified, not merely documented.", "actions": [ "Request the resilience testing register and verify all production AI systems are covered with current test results.", "Spot-check 20% of test records for adequate evidence quality (scenario description, execution steps, observed outcome, pass/fail determination).", "Review the open remediation log for age, ownership, and whether critical items have been escalated appropriately." ], "metrics": [ "Control test coverage: target 100% of enumerated controls tested within scheduled interval.", "Test pass rate: target \u226590% controls passing on first test per cycle.", "Remediation closure rate: target 100% of critical failures resolved within 30 days." ], "failure_signals": [ "Any production AI system has no test record within the last two scheduled intervals.", "More than 10% of controls fail each cycle without evidence of systemic root cause analysis.", "Remediation items are routinely extended past their deadline without documented justification." ] }, "business_continuity": { "summary": "Resilience control testing validates that the organization's recovery capabilities are real, giving leadership the evidence they need to assert that continuity objectives can be met.", "actions": [ "Require that all business continuity plan assertions about recovery capabilities reference test evidence from the resilience testing register.", "Escalate patterns of repeated control failures to executive sponsors with recommended investment actions.", "Ensure test schedules align with regulatory examination cycles so current evidence is always available." ], "failure_signals": [ "Business continuity plan assertions about AI system recovery cannot be backed by test evidence.", "Repeated failures of the same control have not triggered a root cause review.", "Test schedules are misaligned with regulatory examination timelines." ] }, "security_architect": { "summary": "Resilience controls are attack surface and defense at once: failover paths, backup restores, and fallback modes can bypass security controls if they are never tested together. Verify that resilience mechanisms preserve authentication, authorization, and logging when they activate.", "actions": [ "Include security-control assertions (authN/authZ intact, logging active) in every resilience control verification test.", "Review failover and fallback paths for security parity with primary paths before signing off test results.", "Track any test where a resilience mechanism activated with degraded security controls as a security finding, not just a resilience finding." ], "failure_signals": [ "Failover environments pass availability tests while running with permissive network policies.", "Verification tests never assert that audit logging continued during simulated failure.", "Security review of resilience test results is not part of the remediation workflow." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Many organizations conduct annual disaster recovery drills but lack systematic per-control verification testing for AI-specific resilience mechanisms." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "cloud-native", "eu-high-risk-ai" ], "implementers": [ "IT Operations", "Site Reliability Engineering", "GRC Team" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a78.5", "fit": "direct", "rationale": "ISO 22301:2019 \u00a78.5 requires that organizations validate continuity procedures through testing at planned intervals, evaluating test results against objectives and taking corrective action. This control implements the testing discipline that satisfies \u00a78.5 for AI system resilience controls specifically.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.5", "fit": "direct", "rationale": "NIST SP 800-34 Rev 1 \u00a73.5 defines the test, training, and exercise (TT&E) program for contingency plans, requiring that plans be exercised to verify procedures, identify gaps, and confirm that recovery objectives can be achieved. This control operationalizes that TT&E requirement for AI resilience controls.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "RC.RP-02", "fit": "partial", "rationale": "NIST CSF 2.0 RC.RP-02 requires that recovery actions are selected, scoped, prioritized, and performed. Systematic control verification testing supplies the evidence that those recovery actions actually function as designed before they are relied on in a real incident (CSF 2.0 has no RC.RP outcome for testing itself; test-driven improvement sits in ID.IM-02).", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 25", "fit": "direct", "rationale": "EU DORA Article 25 mandates that financial entities test their ICT continuity and recovery capabilities at least annually, with advanced testing requirements for significant entities. AI systems that support critical financial functions must meet these legally binding testing obligations.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.04", "fit": "direct", "rationale": "COBIT 2019 DSS04.04 requires the business continuity plan and disaster response plan to be exercised and tested on a regular basis, with results reviewed against objectives. It defines the governance accountability for test ownership and cadence for the resilience controls this control verifies (DSS04.05 covers plan review and maintenance).", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "REL 12", "fit": "direct", "rationale": "AWS Well-Architected Reliability Pillar REL 12 (Test reliability) requires workloads to be tested using game days and fault injection so that reliability mechanisms are proven under realistic failure conditions. This directly matches the simulated-failure verification testing this control requires (REL 13 covers disaster recovery planning).", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Production Readiness Review \u2013 Reliability and resilience control verification", "fit": "direct", "rationale": "RV-02 requires that resilience controls protecting AI systems be verified as functional before reliance on them, with deficiencies tracked to remediation. Google SRE Production Readiness Reviews mandate that reliability and resilience controls be evaluated and verified before a system is considered production-ready, and error budget policies require ongoing evidence of control effectiveness. The PRR checklist and error budget review together create the continuous verification cadence that RV-02 requires.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "BCDR \u2013 Testing and validation of failover and recovery controls", "fit": "partial", "rationale": "RV-02 mandates that resilience controls be tested under simulated failure conditions to verify they function as designed. Microsoft Azure Resiliency BCDR guidance requires periodic failover drills and recovery tests for workloads using availability zones and geo-redundancy to confirm that the configured resilience controls activate correctly. Azure reliability documentation explicitly states that BCDR designs must be tested, not merely configured, aligning with RV-02's focus on verified control effectiveness.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RV-02", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "All resilience controls protecting each production AI system must have been executed against their defined test scenarios within the scheduled testing interval, with pass/fail results recorded per control and all identified failures tracked to a remediation item with an assigned owner and resolution deadline.", "evidence_required": [ "resilience_testing_register with control_id, ai_system_id, test_scenario, execution_date, pass_fail_status, and evidence_summary for each control within the current testing interval", "remediation_log with control_id, failure_description, assigned_owner, deadline, and resolution_date for every control that failed its most recent test", "test_schedule_document listing each control's required test frequency and confirming no gaps exceed the prescribed interval", "retest_record confirming each remediated control was retested within 30 days of remediation closure and passed" ], "machine_tests": [ "Query resilience_testing_register per AI system \u2192 assert every enumerated control has an execution record within its scheduled interval (\u226490 days for tier-1, \u2264180 days for lower tiers)", "Filter test records by pass_fail_status=fail \u2192 assert every failure record has a linked remediation_log entry with non-null assigned_owner and deadline", "Check retest_record for each closed remediation item \u2192 assert retest_execution_date \u2264 remediation_closure_date + 30 days", "Calculate control test coverage ratio (tested_controls / enumerated_controls) per AI system \u2192 assert \u2265100% of controls appear in the register" ], "human_review": [ "Spot-check 20% of test records to verify that scenario descriptions, execution steps, and observed outcomes are documented with sufficient fidelity to demonstrate what was actually tested rather than a nominal compliance entry", "Review the open remediation log for any items that have been repeatedly extended past their deadline without documented justification and assess whether the pattern indicates systemic capability gaps", "Assess whether the test scenario library covers the failure modes most relevant to each AI system's architecture, including model API dependency failures and AI-specific inference pipeline faults" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Conducting only high-level disaster recovery tabletops that declare all controls passed without executing each individual control under its specific failure scenario", "Logging test results as pass without capturing what was observed, making it impossible to distinguish genuine verification from nominal compliance entries", "Closing remediation items by documenting that a fix was applied without executing a retest to confirm the control now behaves correctly", "Testing AI system resilience controls only at system launch and assuming they remain effective through subsequent software updates and configuration changes", "Exempting AI-specific controls such as circuit breakers, model fallback routing, and inference rate limiting from the testing calendar because they are deemed 'inherent' to the platform" ], "update_status": "current", "layer_code": "RV" }, { "id": "RV-03", "layer": "RV", "plane": "control", "name": "Chaos Engineering for AI Systems", "plain": "AI systems must be subject to deliberate fault injection experiments in controlled environments to discover resilience weaknesses, validate failure handling behavior, and confirm that fallback and recovery mechanisms activate correctly before those failure modes occur in production.", "threat": { "tags": [ "hidden-failure-modes", "cascading-failure", "untested-resilience", "undiscovered-weakness" ], "desc": "Complex AI systems exhibit failure modes that are not apparent from code review, architecture diagrams, or functional testing. Unexercised code paths in failure handlers frequently contain bugs that only surface when a real fault occurs. Without deliberate fault injection, organizations cannot distinguish between AI systems that are genuinely resilient and those that merely have not failed yet." }, "standard": [ { "id": "google_sre", "section": "Ch. 22", "title": "Addressing Cascading Failures" }, { "id": "nist_800_160_v2", "section": "\u00a72.1.3", "title": "Cyber resiliency techniques \u2014 Adaptive Response and Analytic Monitoring" }, { "id": "aws_reliability", "section": "REL 12", "title": "Test reliability through game days and fault injection" }, { "id": "nist_csf", "section": "ID.IM-02", "title": "Improvements identified from security tests and exercises" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RV-03 Chaos Engineering for AI Systems control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RV-03 Chaos Engineering for AI Systems control.", "reviewed_on": "2026-07-02" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RV-03 Chaos Engineering for AI Systems control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RV-03 Chaos Engineering for AI Systems control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RV-03 Chaos Engineering for AI Systems control.", "reviewed_on": "2026-07-01" }, { "id": "cloudflare_resilience_2024", "title": "Cloudflare DDoS Protection", "authority": "Cloudflare, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.cloudflare.com/ddos/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "cloudflare_resilience_2024", "relationship": "informative_reference", "rationale": "Establishes Cloudflare DDoS Protection requirements informing the apeiris://resilience/controls/RV-03 Chaos Engineering for AI Systems control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Establish a chaos engineering practice with a hypothesis-driven experiment model: define steady state, inject fault, observe impact, compare to hypothesis, and document findings. Run experiments first in non-production, then promote validated experiments to production during low-traffic windows.", "steps": [ "Define steady-state metrics for each AI system (inference latency p99, error rate, throughput) that serve as the baseline for chaos experiment evaluation.", "Build a fault injection library covering at least: model API latency injection, dependency service shutdown, network partition simulation, resource exhaustion (CPU/memory), and malformed input injection.", "Run each experiment class in a staging environment first, confirming the system under test behaves as hypothesized before introducing production experiments.", "Execute production chaos experiments during low-traffic windows with on-call SRE present, with an automated kill switch to abort if steady-state deviation exceeds a predefined threshold.", "Document all experiment hypotheses, execution steps, observed outcomes, deviations from expected behavior, and follow-on action items in a chaos experiment log." ], "site_reliability": { "summary": "Chaos engineering is the mechanism by which you build justified confidence in AI system resilience. Treat every hypothesis as a bet you want to lose \u2014 the goal is to find failures now, not later.", "actions": [ "Establish steady-state definitions and monitoring thresholds before any experiment runs.", "Maintain a prioritized backlog of chaos experiments based on risk model and time since last test.", "Automate experiment scaffolding (inject, observe, report) using a chaos engineering platform or custom tooling." ], "failure_signals": [ "AI system has never had a production chaos experiment executed against it.", "Experiment results show steady-state violation without a recovery mechanism activating.", "Chaos experiment backlog has not been reviewed or reprioritized in over 90 days." ] }, "security_architect": { "summary": "Chaos engineering reveals security-relevant failure modes including how the system behaves when authentication services degrade, rate limiting fails, or malformed inputs arrive at scale.", "actions": [ "Include security-relevant fault scenarios: authentication service degradation, authorization cache invalidation, and adversarial input at scale.", "Review experiment results for security control bypass conditions that emerge under fault.", "Ensure chaos experiment reports feed into the threat model update cycle." ], "failure_signals": [ "No security-relevant fault scenarios exist in the experiment library.", "Experiment results have never been reviewed through a security lens.", "Threat model has not been updated following chaos experiment findings." ] }, "it_operations": { "summary": "Operations teams must coordinate chaos experiments with change management, ensure monitoring is in place to detect induced faults, and participate in game day exercises that simulate real incident conditions.", "actions": [ "Register chaos experiments in the change management system with defined rollback criteria.", "Confirm monitoring and alerting coverage is active for the duration of each experiment.", "Participate in game day exercises to validate incident response procedures alongside fault injection." ], "failure_signals": [ "Chaos experiments are run without change management registration.", "Monitoring gaps allowed a fault to go undetected during an experiment.", "No game day exercises have been conducted in the last six months." ] }, "business_continuity": { "summary": "Chaos engineering provides the empirical evidence that business continuity claims about AI system resilience are grounded in tested reality rather than untested assumptions.", "actions": [ "Request chaos experiment logs as supporting evidence for business continuity plan assertions about AI system recovery capabilities.", "Escalate unresolved experiment findings that threaten RTO/RPO compliance to appropriate governance forums.", "Ensure chaos experiment frequency aligns with BCP review cycles." ], "failure_signals": [ "BCP claims about AI system resilience cannot be supported by chaos experiment evidence.", "Experiment findings threatening RTO/RPO compliance have not been escalated.", "Chaos experiment program has been suspended without a documented risk acceptance decision." ] }, "grc_auditor": { "summary": "Chaos experiments are audit evidence that resilience mechanisms work \u2014 but only if scoped, approved, and documented. Ensure the experiment program has governance guardrails and that findings are tracked to closure.", "actions": [ "Verify each experiment has a documented hypothesis, blast-radius limit, approval record, and abort criteria before execution.", "Sample experiment reports and confirm findings entered the remediation tracker with owners and due dates.", "Confirm production experiments on regulated AI systems have change-management and risk-acceptance records." ], "failure_signals": [ "Experiments run in production without documented approval or blast-radius controls.", "Findings from failed experiments have no remediation trail.", "The experiment program cannot produce a year's worth of run records on request." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Chaos engineering is widely practiced in mature cloud-native organizations but rarely applied specifically to AI inference pipelines, model API dependencies, or ML-specific failure modes." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "cloud-native", "high-risk-sector", "universal-enterprise", "multi-tenant" ], "implementers": [ "Site Reliability Engineering", "Platform Engineering", "DevOps" ], "frameworks": [ { "framework": "google_sre", "requirement_id": "Ch. 22", "fit": "direct", "rationale": "Google SRE Chapter 22 on Addressing Cascading Failures provides the foundational practices for fault injection and chaos engineering as resilience verification tools. The hypothesis-driven experiment model in this control directly implements the SRE discipline of testing systems under realistic failure conditions to build justified reliability confidence.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "aws_reliability", "requirement_id": "REL 12", "fit": "direct", "rationale": "AWS Well-Architected Reliability Pillar REL 12 (Test reliability) explicitly recommends fault injection testing and game days to validate workload reliability, with AWS Fault Injection Service as the reference tooling. Chaos engineering for AI systems is a direct implementation of this best practice.", "normative_force": "best-practice", "source_version": "2023", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "nist_800_160_v2", "requirement_id": "\u00a72.1.3", "fit": "direct", "rationale": "NIST SP 800-160 Vol 2 Rev 1 \u00a72.1.3 defines the cyber resiliency techniques \u2014 including Adaptive Response and Analytic Monitoring \u2014 whose effectiveness must be demonstrated under adversity rather than assumed (Chapter 3 contains only \u00a73.1-\u00a73.2). Chaos engineering experiments provide the adversity-driven evidence that these techniques and the fallback mechanisms they support activate correctly.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "ID.IM-02", "fit": "partial", "rationale": "NIST CSF 2.0 ID.IM-02 requires that improvements be identified from security tests and exercises, including those conducted in coordination with suppliers and third parties. Chaos experiments are exactly such exercises, generating structured findings that feed the improvement pipeline.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 26", "fit": "partial", "rationale": "EU DORA Article 26 requires advanced ICT resilience testing including threat-led penetration testing for significant financial entities. Chaos engineering provides the technical depth required to satisfy the spirit of advanced resilience testing obligations, particularly for AI systems in critical financial functions.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "microsoft_azure_resil", "requirement_id": "BCDR \u2013 Chaos testing and resilience validation (Azure Chaos Studio)", "fit": "partial", "rationale": "RV-03 requires AI systems to be subject to deliberate fault injection experiments that confirm fallback and recovery mechanisms activate correctly. Microsoft Azure provides Azure Chaos Studio as the native chaos engineering toolset within the Azure Resiliency framework, with guidance on fault injection patterns aligned to BCDR requirements. Chaos experiments designed to validate AI workload resilience within Azure directly satisfy RV-03's requirement for controlled fault injection before production exposure.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RV-03", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "The AI system must have an active chaos engineering program with at least one executed experiment per prioritized failure mode class, documented hypothesis-to-outcome records for each experiment, and tracked action items for every deviation from expected behavior that was discovered.", "evidence_required": [ "chaos_experiment_log with experiment_id, ai_system_id, fault_class, hypothesis, execution_date, environment (staging|production), steady_state_deviation_observed, and outcome_vs_hypothesis for each experiment", "fault_injection_library listing all fault classes covered (model_api_latency, dependency_shutdown, network_partition, resource_exhaustion, malformed_input) and the date each class was last exercised", "experiment_action_item_register with experiment_id, deviation_description, owner, and resolution_deadline for each finding where behavior deviated from hypothesis", "steady_state_baseline_document defining the metrics and thresholds used to evaluate each experiment's outcome" ], "machine_tests": [ "Query chaos_experiment_log per AI system \u2192 assert at least one experiment per fault_class in the fault_injection_library has been executed within the last 90 days", "Filter chaos_experiment_log by environment=production \u2192 assert at least one production experiment has been executed in the last 6 months for each tier-1 AI system", "Compare experiment records where steady_state_deviation_observed=true \u2192 assert each has a linked action_item in experiment_action_item_register with non-null owner", "Check experiment_action_item_register.resolution_deadline \u2192 assert no critical action item is past its deadline without a documented extension" ], "human_review": [ "Review the chaos experiment hypothesis backlog to assess whether experiment selection is driven by a risk model prioritizing the most impactful failure modes rather than whichever scenarios are easiest to inject", "Evaluate at least three experiment records for hypothesis quality \u2014 confirm that hypotheses specify measurable expected outcomes rather than generic statements that any result could satisfy", "Assess whether experiment findings have influenced architecture decisions or led to substantive improvements rather than being catalogued without action" ], "blocking_effect": "advisory", "normative_status": "voluntary-standard", "anti_patterns": [ "Running chaos experiments exclusively in staging environments where dependency topology, traffic patterns, and resource pressure differ materially from production, yielding results that do not validate production resilience", "Defining hypotheses as 'the system should be fine' without specifying measurable steady-state thresholds, making it impossible to determine whether the experiment revealed a genuine weakness", "Limiting fault injection to infrastructure-layer faults (node failure, network partition) without testing AI-specific failure modes such as model API timeout storms, embedding service degradation, or malformed inference response injection", "Treating chaos experiments as a one-time certification activity rather than a continuous practice, allowing configuration drift and software changes to invalidate previously validated resilience properties", "Running experiments without an active kill switch threshold, risking uncontrolled cascading failures during experiments with unexpectedly severe impact" ], "update_status": "current", "layer_code": "RV" }, { "id": "RV-04", "layer": "RV", "plane": "lifecycle", "name": "Recovery Capability Verification After Actual Incident", "plain": "Following every incident that triggers recovery procedures for an AI system, the organization must formally verify that recovery actions achieved the declared recovery objectives, document deviations between expected and actual recovery performance, and update recovery plans and controls to address discovered gaps.", "threat": { "tags": [ "unverified-recovery", "residual-failure", "incomplete-restoration", "recurrence-risk" ], "desc": "Organizations frequently declare incidents resolved without verifying that all affected AI system components have fully recovered to their pre-incident state and that recovery performance met RTO/RPO targets. Undetected residual failures in AI inference pipelines, model state, or data consistency can persist after formal incident closure, leading to silent degradation. Without post-incident recovery verification, the same gaps that caused prolonged recovery will recur in future incidents." }, "standard": [ { "id": "iso_22301", "section": "\u00a710.1", "title": "Nonconformity and corrective action" }, { "id": "nist_sp800_34", "section": "\u00a74.4", "title": "Reconstitution phase \u2014 validation and after-action reporting" }, { "id": "iso_27031", "section": "Cl. 11", "title": "Testing, exercising and audit of ICT readiness (ISO/IEC 27031:2025)" }, { "id": "nist_csf", "section": "ID.IM-04", "title": "Incident response and other cybersecurity plans are maintained and improved" } ], "sources": [ { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RV-04 Recovery Capability Verification After Actual Incident control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RV-04 Recovery Capability Verification After Actual Incident control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RV-04 Recovery Capability Verification After Actual Incident control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RV-04 Recovery Capability Verification After Actual Incident control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RV-04 Recovery Capability Verification After Actual Incident control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Embed a recovery verification gate in the incident closure process: before closing any incident that triggered AI system recovery procedures, the incident commander must confirm recovery metrics against objectives, obtain sign-off from the AI system owner, and initiate a post-incident review to document lessons and plan updates.", "steps": [ "Define recovery verification criteria for each AI system tier specifying what must be confirmed before the system is declared fully recovered (e.g., inference latency within SLO, data consistency checks passed, all downstream dependencies healthy).", "Embed a recovery verification checklist into the incident management system as a mandatory closure gate for incidents affecting AI systems.", "Conduct a post-incident review within five business days of closure for all incidents exceeding RTO or RPO targets, documenting root cause, timeline, and recovery performance gap analysis.", "Update recovery plans, runbooks, and resilience controls within 30 days of each post-incident review based on identified gaps.", "Track all plan updates as action items with owners and deadlines, reporting on completion to the business continuity governance forum." ], "it_operations": { "summary": "Recovery verification is the final step of incident response, not an optional post-mortem activity. Operators must confirm actual recovery performance before closing any incident involving AI system recovery.", "actions": [ "Execute the recovery verification checklist as part of the incident closure workflow, not as a separate optional process.", "Record actual RTO and RPO achieved for every incident that triggered recovery procedures.", "Flag incidents where recovery time exceeded the RTO to the business continuity team immediately upon closure." ], "failure_signals": [ "Incidents are closed without a completed recovery verification checklist.", "Actual RTO/RPO performance is not recorded in incident tickets.", "Residual failures have been discovered after formal incident closure in any of the last three incidents." ] }, "grc_auditor": { "summary": "Post-incident recovery verification records are among the most valuable evidence artifacts for demonstrating that resilience controls are genuinely effective, not merely documented.", "actions": [ "Request incident records for all AI system incidents in the review period and verify each has a completed recovery verification gate.", "Compare documented recovery objectives against actual performance for a sample of incidents.", "Review the post-incident action item register and verify items are closed within their committed timelines." ], "metrics": [ "Recovery verification completion rate: target 100% of qualifying incidents.", "Post-incident review initiation rate: target 100% of incidents exceeding RTO/RPO.", "Plan update action item closure rate: target \u226590% closed within 30-day deadline." ], "failure_signals": [ "Any qualifying incident closed without a recovery verification record.", "Post-incident reviews have not generated plan updates in consecutive review cycles despite incidents exceeding RTO/RPO.", "Action items have remained open beyond deadline without documented extension justification." ] }, "business_continuity": { "summary": "Post-incident learning is the primary mechanism for continuously improving business continuity capabilities. Every recovery gap that goes undocumented and unaddressed represents a risk that will manifest again.", "actions": [ "Chair or sponsor post-incident review sessions for significant AI system incidents.", "Ensure that recovery plan updates from post-incident reviews are propagated to all affected continuity plan documents.", "Incorporate post-incident performance trends into the annual business continuity program review." ], "failure_signals": [ "Business continuity plan has not been updated based on post-incident learning in over 12 months.", "Repeated incidents show the same recovery gaps without corrective plan updates.", "Post-incident review outcomes are not reaching the business continuity governance forum." ] }, "security_architect": { "summary": "Post-incident recovery verification must include security posture: an AI system that recovered on time but with weakened controls is not recovered. Add security regression checks to the verification protocol.", "actions": [ "Add post-recovery security checks (IAM policies, network segmentation, secret validity, logging pipelines) to the recovery verification checklist.", "Verify that any emergency access or control bypasses granted during recovery were revoked and documented.", "Review incidents with a security dimension jointly with the resilience team before closure." ], "failure_signals": [ "Emergency break-glass access from the incident is still active weeks later.", "Recovery verification reports contain no security posture assertions.", "Configuration drift introduced during recovery is discovered only at the next audit." ] }, "site_reliability": { "summary": "After every real incident, measure actual recovery performance against declared objectives with telemetry, not recollection. Deviations are engineering signals for architecture or runbook fixes.", "actions": [ "Instrument recovery milestones so actual RTO/RPO are computed automatically from incident telemetry.", "Run the post-incident verification within the postmortem window and attach measured vs. declared objectives to the postmortem.", "Convert every verified gap into a tracked reliability work item with an owner." ], "failure_signals": [ "Actual recovery times are reconstructed from chat logs instead of telemetry.", "Postmortems close without comparing recovery performance to the objectives register.", "The same recovery gap appears in consecutive incidents without an intervening fix." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Post-incident reviews are common but rarely include formal verification that recovery objectives were met; this distinction is not enforced as a closure gate in most organizations." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai" ], "implementers": [ "Incident Response Team", "IT Operations", "Business Continuity Team" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a710.1", "fit": "direct", "rationale": "ISO 22301:2019 \u00a710.1 requires that nonconformities \u2014 including those revealed by actual incidents \u2014 are reviewed, root causes determined, and corrective actions taken and verified for effectiveness. Post-incident recovery verification is the mechanism that surfaces those nonconformities for AI recovery capabilities (\u00a710.2 addresses continual improvement generally).", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a74.4", "fit": "direct", "rationale": "NIST SP 800-34 Rev 1 \u00a74.4 defines the reconstitution phase, which closes out recovery with validation testing, deactivation, and documentation of lessons learned from plan activation. This control formalizes that closure step for AI incidents by verifying that recovery met declared objectives and feeding gaps back into the plan.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_27031", "requirement_id": "Cl. 11", "fit": "direct", "rationale": "ISO/IEC 27031:2025 Clause 11 covers testing, exercising and audit of ICT readiness for business continuity, including review of performance after disruptions with results fed into improvement. Post-incident verification of actual recovery performance implements that review loop for AI systems.", "normative_force": "voluntary-standard", "source_version": "2025", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "ID.IM-04", "fit": "direct", "rationale": "NIST CSF 2.0 ID.IM-04 requires that incident response plans and other cybersecurity plans that affect operations are established, communicated, maintained, and improved. Post-incident recovery verification generates the lessons that drive those plan improvements.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 17", "fit": "partial", "rationale": "EU DORA Article 17 requires financial entities to conduct post-incident reviews for major ICT-related incidents and to implement corrective measures. AI systems involved in material incidents trigger this obligation, making post-incident recovery verification a component of DORA compliance for in-scope organizations.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "aws_reliability", "requirement_id": "REL 12 \u2013 Test and validate failure recovery", "fit": "direct", "rationale": "RV-04 requires that following actual incidents, organizations verify recovery achieved declared objectives, document deviations, and update recovery plans. AWS Well-Architected Reliability Pillar REL-12 prescribes that after-action reviews and failure recovery validations be conducted to confirm recovery procedures performed as designed and to capture improvements. Both RV-04 and REL-12 address the same imperative: treating actual incidents as verification events that must be reviewed to strengthen future recovery capability.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Postmortem Culture \u2013 Blameless postmortems with tracked action items", "fit": "direct", "rationale": "RV-04 mandates formal post-incident recovery verification including documentation of deviations between expected and actual recovery performance and updates to recovery plans. Google SRE Postmortem Culture requires blameless, structured postmortem documents that capture what happened, why recovery took as long as it did, and what specific actions will prevent recurrence \u2014 precisely the artifacts RV-04 requires. The SRE postmortem practice with tracked action items directly instantiates the recovery-gap-to-remediation cycle that RV-04 formalizes.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "BCDR \u2013 Post-incident review and recovery objective validation", "fit": "partial", "rationale": "RV-04 requires formal verification that post-incident recovery achieved declared RTO/RPO objectives and that gaps are documented and addressed. Microsoft Azure Resiliency BCDR guidance prescribes post-incident reviews to assess whether recovery objectives were met, whether failover automation performed correctly, and what BCDR plan updates are required. Azure's emphasis on recovery objective validation after actual failover events aligns directly with RV-04's requirement to verify rather than assume successful recovery.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RV-04", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every incident that triggered AI system recovery procedures must have a completed recovery verification record confirming actual RTO and RPO achieved against declared objectives, a post-incident review initiated within five business days for any incident that exceeded declared objectives, and all identified plan gaps tracked to closed remediation items within 30 days.", "evidence_required": [ "incident_recovery_verification_record per qualifying incident with incident_id, ai_system_id, declared_rto_minutes, declared_rpo_minutes, actual_rto_minutes, actual_rpo_minutes, all_components_verified_healthy flag, and closure_gate_completed timestamp", "post_incident_review_document with incident_id, root_cause_analysis, recovery_performance_gap_analysis, and action_items for each incident that exceeded RTO or RPO targets", "plan_update_action_item_register with action_id, source_incident_id, gap_description, assigned_owner, deadline, and resolution_date for each gap identified in post-incident reviews", "residual_failure_check_record confirming all AI system components (inference endpoint, model state, data consistency, downstream dependencies) were individually verified healthy before incident closure" ], "machine_tests": [ "Query incident_management_system for all incidents with recovery_procedures_triggered=true \u2192 assert each has a linked incident_recovery_verification_record with closure_gate_completed timestamp before incident.resolved_at", "Filter incidents where actual_rto > declared_rto OR actual_rpo > declared_rpo \u2192 assert each has a post_incident_review_document with initiation_date \u2264 incident.resolved_at + 5 business days", "Query plan_update_action_item_register \u2192 assert all items have resolution_date \u2264 deadline (within 30 days) or a documented extension approval", "Check residual_failure_check_record per incident \u2192 assert all_components_verified_healthy=true before incident closure" ], "human_review": [ "Review post-incident review documents for analytical depth \u2014 confirm that root cause analysis identifies specific control or architecture gaps rather than attributing failures to 'unexpected circumstances'", "Assess whether plan update action items from post-incident reviews are substantive changes to recovery procedures or runbooks, versus minor documentation edits that do not address the underlying gap", "Verify that repeated incidents showing the same recovery gap have triggered a root cause review at the program level rather than being addressed only at the individual incident level" ], "blocking_effect": "requires-review", "normative_status": "certification-standard", "anti_patterns": [ "Closing incidents by timestamping the last recovery action without executing a verification checklist to confirm all AI system components have returned to healthy state, leaving residual failures silently present", "Treating post-incident review as optional and conducting it only when management requests it rather than as a mandatory closure gate for incidents exceeding RTO or RPO", "Generating post-incident action items that describe symptoms rather than root causes, resulting in remediation that does not address the underlying recovery gap", "Recording declared and actual RTO/RPO as the same value when actual recovery time was not measured, creating false compliance evidence in the incident record", "Cascading the same root cause across multiple incidents without a program-level corrective action because each incident is handled by a different team with no cross-incident learning process" ], "update_status": "current", "layer_code": "RV" }, { "id": "RV-05", "layer": "RV", "plane": "control", "name": "Third-Party AI Dependency Resilience Assessment", "plain": "AI systems relying on external model APIs, cloud services, data providers, or other third-party dependencies must be subject to formal resilience assessments that evaluate the dependency's availability commitments, historical reliability, failure notification practices, and the AI system's ability to tolerate or substitute for dependency unavailability.", "threat": { "tags": [ "third-party-failure", "vendor-dependency", "supply-chain-disruption", "api-unavailability" ], "desc": "AI systems that depend on external model APIs, cloud inference services, or third-party data providers inherit those providers' failure modes without inheriting their visibility or control. A provider outage that was within their SLA tolerance may still catastrophically degrade an AI system that has no fallback path. Organizations frequently discover their tolerance for third-party AI dependency downtime only when a real outage exposes that no mitigation was in place." }, "standard": [ { "id": "dora", "section": "Art. 28-30", "title": "ICT third-party risk management requirements" }, { "id": "iso_22301", "section": "\u00a78.2.3", "title": "Third-party continuity requirements" }, { "id": "nist_csf", "section": "GV.SC-07", "title": "Third-party resilience aligned with enterprise requirements" }, { "id": "cobit_dss04", "section": "DSS04.03", "title": "Third-party continuity obligations" } ], "sources": [ { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RV-05 Third-Party AI Dependency Resilience Assessment control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RV-05 Third-Party AI Dependency Resilience Assessment control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RV-05 Third-Party AI Dependency Resilience Assessment control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RV-05 Third-Party AI Dependency Resilience Assessment control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RV-05 Third-Party AI Dependency Resilience Assessment control.", "reviewed_on": "2026-07-01" }, { "id": "cloudflare_resilience_2024", "title": "Cloudflare DDoS Protection", "authority": "Cloudflare, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.cloudflare.com/ddos/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "cloudflare_resilience_2024", "relationship": "informative_reference", "rationale": "Establishes Cloudflare DDoS Protection requirements informing the apeiris://resilience/controls/RV-05 Third-Party AI Dependency Resilience Assessment control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Inventory all third-party AI dependencies, classify each by criticality based on business impact analysis, obtain or derive availability SLAs, assess each dependency's historical reliability against SLA commitments, evaluate mitigation options (fallback models, caching, graceful degradation), and document residual risk with owner sign-off.", "steps": [ "Enumerate all third-party AI dependencies for each production AI system including model inference APIs, embedding services, vector database providers, and data enrichment services.", "Classify each dependency as critical (no substitution available, failure degrades primary function), high (substitution possible with degraded quality), or standard (substitution transparent to users).", "Review published SLAs, historical uptime records, and incident communication practices for each critical or high dependency over the most recent 12-month period.", "Evaluate and document mitigation options for each dependency including fallback model routing, response caching, queue-based decoupling, and graceful degradation patterns.", "Formally accept or remediate residual risk for dependencies where SLA coverage is insufficient for business requirements, with sign-off from the AI system owner." ], "security_architect": { "summary": "Third-party AI dependencies represent a supply-chain resilience risk. Architect fallback paths and circuit breakers into AI system designs before signing vendor contracts, not after experiencing the first outage.", "actions": [ "Require that every AI system design document includes a dependency failure impact analysis and fallback architecture.", "Review vendor contracts for SLA terms, incident notification obligations, and termination rights before approval.", "Recommend circuit breaker and fallback routing patterns for all critical dependencies during architecture review." ], "failure_signals": [ "AI system architecture has no documented fallback path for any critical third-party dependency.", "Vendor contracts for critical AI dependencies lack meaningful SLA terms or incident notification obligations.", "Circuit breaker implementation is absent for any dependency classified as critical." ] }, "it_operations": { "summary": "Operations teams must monitor third-party AI dependency health in real time, have runbooks for dependency failure scenarios, and know exactly what the AI system does when each critical dependency becomes unavailable.", "actions": [ "Deploy synthetic monitoring against all critical third-party AI API dependencies to detect degradation before it impacts users.", "Maintain runbooks for each critical dependency failure scenario documenting expected system behavior and operator response actions.", "Participate in third-party provider incident notification workflows to receive early warning of planned or unplanned outages." ], "failure_signals": [ "Synthetic monitoring is absent for any critical third-party AI dependency.", "No runbook exists for the failure of any critical AI dependency.", "Operations team was not notified of a third-party provider incident until user impact was reported." ] }, "grc_auditor": { "summary": "Third-party AI dependency risk assessments are required by multiple regulatory frameworks. Audit evidence must demonstrate dependencies are inventoried, assessed, and that residual risk is formally accepted or mitigated.", "actions": [ "Request the third-party AI dependency register and verify completeness against known production AI systems.", "Review the most recent resilience assessment for each critical dependency and confirm SLA adequacy analysis is present.", "Verify that residual risk acceptance sign-offs are current and at the appropriate authority level." ], "metrics": [ "Dependency inventory completeness: target 100% of known third-party AI dependencies registered.", "Critical dependency assessment currency: target 100% assessed within 12 months.", "Residual risk acceptance coverage: target 100% of critical dependencies with formal risk acceptance or remediation plan." ], "failure_signals": [ "Third-party AI dependency register is absent or materially incomplete.", "Any critical dependency has not been assessed within the last 12 months.", "Residual risk acceptance is missing for dependencies with SLA gaps." ] }, "business_continuity": { "summary": "Third-party AI dependency resilience gaps represent business continuity risks that must be disclosed to leadership, included in continuity planning, and addressed through vendor management or architectural mitigation.", "actions": [ "Ensure the business continuity plan explicitly addresses scenarios where critical third-party AI dependencies are unavailable for extended periods.", "Escalate dependencies where provider SLA commitments are materially below business requirements to vendor management for contract renegotiation.", "Include third-party dependency resilience status in the annual business continuity program review." ], "failure_signals": [ "Business continuity plan does not address extended unavailability of any critical third-party AI dependency.", "SLA gaps for critical dependencies have not been escalated to vendor management.", "Third-party dependency resilience is not included in business continuity program review reporting." ] }, "site_reliability": { "summary": "Third-party AI dependencies set a ceiling on your achievable SLOs. Quantify each provider's real availability and failure behavior, and engineer for the day the dependency is degraded or gone.", "actions": [ "Track per-provider availability, latency, and error budgets from your own telemetry, not just vendor status pages.", "Exercise failover to substitute providers or degraded modes for each critical external dependency at least annually.", "Wire provider health signals into alerting with runbook links for dependency-outage response." ], "failure_signals": [ "Provider outages are discovered from user reports before monitoring detects them.", "Declared SLOs exceed what the weakest critical dependency can mathematically support.", "Failover to an alternate provider has never been executed outside a document." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Organizations routinely inventory third-party software dependencies but rarely conduct formal resilience assessments specific to AI inference and model API dependencies." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise", "cloud-native" ], "implementers": [ "Vendor Management", "IT Operations", "Business Continuity Team" ], "frameworks": [ { "framework": "dora", "requirement_id": "Art. 28-30", "fit": "direct", "rationale": "EU DORA Articles 28 through 30 establish binding requirements for ICT third-party risk management including pre-contract due diligence, contractual provisions, and ongoing monitoring of critical ICT third-party service providers. AI model API providers and cloud inference services used by financial entities fall within this scope.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "iso_22301", "requirement_id": "\u00a78.2.3", "fit": "direct", "rationale": "ISO 22301:2019 \u00a78.2.3 requires organizations to consider the continuity requirements imposed on third-party suppliers and assess whether those suppliers can meet obligations. For AI systems with external model dependencies, this means evaluating whether providers' continuity commitments are adequate for the AI system's recovery objectives.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "GV.SC-07", "fit": "direct", "rationale": "NIST CSF 2.0 GV.SC-07 requires that risks posed by third parties in the supply chain be identified, assessed, and managed. Third-party AI dependency resilience assessments directly address the supply chain risk identification and management activities this subcategory requires.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.03", "fit": "partial", "rationale": "COBIT 2019 DSS04.03 requires that third-party continuity obligations be identified and validated as part of the continuity strategy. This provides the governance framework for ensuring that third-party AI dependency assessments are conducted with appropriate oversight and that findings are escalated to the appropriate decision authority.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.5", "fit": "partial", "rationale": "NIST SP 800-34 Rev 1 \u00a73.5 includes considerations for systems with external dependencies in contingency planning, noting that organizations must address dependencies that could affect recovery. Third-party AI model APIs represent a class of external dependency that must be accounted for in any complete contingency plan.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "REL 1", "fit": "partial", "rationale": "AWS Well-Architected Reliability Pillar REL 1 (Manage service quotas and constraints) requires understanding and managing the quotas and constraints of services a workload depends on \u2014 including third-party AI APIs with rate limits and capacity commitments. Third-party dependency resilience assessment operationalizes this for external model providers.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Production Readiness Review \u2013 Third-party dependency resilience evaluation", "fit": "partial", "rationale": "RV-05 requires formal resilience assessments covering external AI dependencies' availability commitments, failure notification practices, and the AI system's ability to substitute for their unavailability. Google SRE Production Readiness Reviews include mandatory evaluation of third-party service dependencies, their SLOs, and whether the system under review has adequate fallback paths. The PRR framework provides a structured template for the dependency resilience assessment that RV-05 mandates before production reliance.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "BCDR \u2013 Third-party service dependency continuity planning", "fit": "partial", "rationale": "RV-05 mandates formal assessment of third-party AI dependency resilience including continuity of service and fallback capabilities. Microsoft Azure BCDR guidance requires that business continuity plans account for third-party cloud and SaaS dependencies, with specific attention to SLA coverage gaps and fallback architecture for critical external services. For AI systems hosted on Azure, RV-05 third-party dependency assessments align with Azure BCDR requirements for external service resilience.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "cloudflare_resilience", "requirement_id": "Load balancing and failover \u2013 Multi-origin failover for AI API endpoints", "fit": "adjacent", "rationale": "RV-05 requires assessment of an AI system's ability to tolerate or substitute for third-party API dependency unavailability. Cloudflare's load balancing and failover capabilities can implement origin failover between competing AI API providers, enabling automatic substitution when a primary third-party dependency becomes unavailable. Assessing Cloudflare failover configuration as part of RV-05 third-party dependency assessment provides evidence of technical substitution capability at the edge.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RV-05", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every third-party AI dependency (model inference APIs, embedding services, vector database providers, data enrichment services) for each production AI system must be registered and classified, with a current resilience assessment completed within the last 12 months for all critical and high dependencies, and documented fallback architecture or formal risk acceptance for any dependency whose SLA coverage is insufficient for the AI system's recovery objectives.", "evidence_required": [ "third_party_ai_dependency_register with dependency_id, ai_system_id, dependency_type, criticality_classification (critical|high|standard), and sla_coverage_adequate flag for all production AI systems", "dependency_resilience_assessment per critical or high dependency within last 12 months, including sla_terms, historical_uptime_12mo_percent, incident_notification_practice, and fallback_option_evaluated", "fallback_architecture_document or risk_acceptance_record with owner_signoff for each dependency where sla_coverage_adequate=false", "synthetic_monitoring_coverage_report confirming active health checks are deployed against all critical third-party AI API endpoints" ], "machine_tests": [ "Query third_party_ai_dependency_register \u2192 assert all known production AI systems have at least one dependency entry and no critical or high dependency lacks a resilience_assessment within the last 12 months", "Filter register by criticality=critical AND sla_coverage_adequate=false \u2192 assert each has a linked fallback_architecture_document or risk_acceptance_record with non-null owner_signoff", "Query synthetic_monitoring_coverage_report \u2192 assert every dependency with criticality=critical has an active synthetic monitor with check_interval \u2264 60 seconds", "Check risk_acceptance_record.owner_signoff_date per entry \u2192 assert no acceptance is older than 12 months without a renewal record" ], "human_review": [ "Review the dependency resilience assessment methodology to confirm that historical uptime data was obtained from independent sources (status pages, third-party monitoring) rather than solely from vendor self-reporting", "Assess the adequacy of documented fallback architectures for critical dependencies \u2014 confirm that fallback paths are technically implemented and tested, not merely described as design intent", "Evaluate whether the criticality classifications assigned to each dependency are realistic given the AI system's failure mode analysis, particularly for dependencies classified as standard where evidence suggests failure would materially degrade the AI system" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Relying solely on vendor-published SLA percentages as evidence of adequate availability without reviewing historical incident records or comparing SLA commitments against the AI system's own RTO requirements", "Treating third-party AI API dependencies as outside the scope of the organization's resilience program because they are 'the vendor's responsibility', leaving the AI system with no fallback when the dependency fails", "Designing fallback architecture on paper without implementing or testing it, discovering during an actual outage that the fallback path contains configuration errors or is not deployed in the production environment", "Classifying AI model API dependencies as 'standard' criticality when the AI system's primary function is entirely dependent on that API, understating risk and deferring resilience investment", "Completing dependency assessments at procurement time and never refreshing them despite provider SLA changes, historical incidents, or changes to the AI system's business criticality" ], "update_status": "current", "layer_code": "RV" }, { "id": "RV-06", "layer": "RV", "plane": "control", "name": "Multi-Region and Multi-AZ Resilience Verification", "plain": "AI systems deployed across multiple cloud regions or availability zones must have their geographic failover and cross-zone recovery capabilities verified through controlled testing to confirm that automatic and manual failover procedures function correctly and that AI system behavior is consistent across regions.", "threat": { "tags": [ "regional-outage", "failover-failure", "single-region-dependency", "geographic-concentration" ], "desc": "Architectures designed for multi-region resilience frequently contain untested assumptions about failover automation, data synchronization state, and AI model availability across regions. A cloud provider regional outage that triggers an untested failover path may reveal that model versions are inconsistent between regions, inference endpoints are not available in the failover region, or traffic routing automation contains configuration errors that prevent automatic activation." }, "standard": [ { "id": "aws_reliability", "section": "REL-10", "title": "Deploy AI workloads across multiple locations" }, { "id": "microsoft_azure_resil", "section": "RE:05", "title": "Design for redundancy \u2014 availability zones and regions" }, { "id": "iso_22301", "section": "\u00a78.4.5", "title": "Recovery" }, { "id": "nist_csf", "section": "PR.IR-04", "title": "Adequate resource capacity to ensure availability is maintained" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RV-06 Multi-Region and Multi-AZ Resilience Verification control.", "reviewed_on": "2026-07-01" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RV-06 Multi-Region and Multi-AZ Resilience Verification control.", "reviewed_on": "2026-07-01" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RV-06 Multi-Region and Multi-AZ Resilience Verification control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RV-06 Multi-Region and Multi-AZ Resilience Verification control.", "reviewed_on": "2026-07-02" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RV-06 Multi-Region and Multi-AZ Resilience Verification control.", "reviewed_on": "2026-07-02" }, { "id": "cloudflare_resilience_2024", "title": "Cloudflare DDoS Protection", "authority": "Cloudflare, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.cloudflare.com/ddos/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "cloudflare_resilience_2024", "relationship": "informative_reference", "rationale": "Establishes Cloudflare DDoS Protection requirements informing the apeiris://resilience/controls/RV-06 Multi-Region and Multi-AZ Resilience Verification control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Map all AI system components to their regional deployments, verify model version consistency across regions, execute controlled region-level and AZ-level failover tests with traffic verification, confirm that AI system behavior is identical in the failover region, and document failover time against RTO.", "steps": [ "Document the regional deployment architecture for each AI system including inference endpoints, model storage, feature stores, and downstream dependencies per region.", "Verify that AI model versions, configurations, and inference parameters are synchronized across all deployed regions and that a mechanism exists to detect version drift.", "Execute AZ-level failover tests quarterly by simulating AZ unavailability and confirming that traffic routes correctly to surviving AZs within the target recovery time.", "Execute region-level failover tests at least annually by simulating primary region unavailability and confirming that the secondary region activates correctly and AI system behavior is equivalent.", "Measure and record actual failover time for each test and compare against RTO; confirm AI system output quality and consistency in the failover region through automated functional verification." ], "site_reliability": { "summary": "Multi-region and multi-AZ resilience is an architecture property that must be continuously verified \u2014 configuration drift routinely breaks assumptions that were true at initial deployment.", "actions": [ "Automate model version consistency checks across regions as a continuous health check.", "Instrument failover tests to capture detailed telemetry including traffic cutover latency, inference availability, and error rates during and after failover.", "Alert on any regional configuration drift that could affect failover correctness before the next planned test window." ], "failure_signals": [ "Model versions are not consistent across deployed regions.", "Failover test has not been executed for any tier-1 AI system in the last 90 days.", "Actual failover time exceeded RTO in the most recent test without a remediation plan." ] }, "security_architect": { "summary": "Geographic failover must preserve security properties. Verify that authentication, authorization, encryption, and audit logging function correctly in the failover region and that the failover path does not introduce security control gaps.", "actions": [ "Confirm that identity providers, authorization services, and secrets management are available in the failover region before declaring the region ready.", "Verify that audit logging and security monitoring are active in the failover region and that logs are not lost during the failover transition.", "Review failover runbooks for security control verification steps that confirm security posture is maintained post-failover." ], "failure_signals": [ "Failover test procedures do not include verification of security control availability in the failover region.", "Audit log continuity was broken during a failover test without a documented recovery.", "Identity or authorization service unavailability in the failover region has not been flagged as a blocking issue." ] }, "it_operations": { "summary": "Operations teams must be able to execute regional failover confidently under pressure, which requires regular practice in controlled conditions and runbooks that leave no ambiguity about actions, order, or success criteria.", "actions": [ "Maintain detailed failover runbooks for each AI system covering both automatic and manual failover scenarios.", "Execute scheduled failover drills that require operators to execute runbook steps without real-time guidance to surface gaps in runbook completeness.", "Verify that all on-call engineers have executed at least one regional failover drill in the last 12 months." ], "failure_signals": [ "Failover runbooks have not been updated following the last architectural change.", "On-call engineers have not personally participated in a failover drill.", "Manual failover requires escalation to a specialist who is not on-call rotation." ] }, "business_continuity": { "summary": "Geographic redundancy is a business continuity strategy, not just an infrastructure pattern. Confirm that multi-region failover keeps the business functions the BIA prioritized within their tolerable outage windows.", "actions": [ "Map each multi-region AI deployment to the business functions and RTOs it protects in the BIA.", "Include regional-failure scenarios in continuity exercises and validate business-process continuity, not just technical failover.", "Document residual single-region dependencies (data residency, provider quotas) as accepted risks with owners." ], "failure_signals": [ "Regional failover succeeds technically while dependent business processes still stall.", "The BIA assumes geographic redundancy that engineering has quietly descoped.", "Data-residency constraints that block failover are absent from the continuity plan." ] }, "grc_auditor": { "summary": "Multi-region claims appear in regulatory filings, DORA registers, and customer contracts. Verify that failover evidence is current, tested, and matches what the organization asserts externally.", "actions": [ "Sample failover test records and confirm cadence, scope, and results match documented commitments.", "Reconcile multi-region assertions in contracts and regulatory documentation against actual verified architecture.", "Confirm model-version and data-consistency checks are part of documented failover verification." ], "failure_signals": [ "External assurances cite regional redundancy that has no test evidence behind it.", "Failover tests exist but predate major architectural changes.", "No record demonstrates that models and routing behave identically across regions." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Multi-region deployment is common in cloud-native organizations but failover verification specific to AI inference workloads \u2014 including model consistency checks and output quality verification \u2014 is rarely formalized." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "cloud-native", "high-risk-sector", "eu-high-risk-ai", "multi-tenant", "federated-enterprise" ], "implementers": [ "Cloud Infrastructure Team", "Site Reliability Engineering", "Platform Engineering" ], "frameworks": [ { "framework": "aws_reliability", "requirement_id": "REL 10", "fit": "direct", "rationale": "AWS Well-Architected Reliability Pillar REL-10 requires workloads to be deployed across multiple locations and verified through testing. The pillar explicitly calls for game days and fault injection to validate multi-region and multi-AZ resilience, which directly corresponds to the verification testing approach defined in this control.", "normative_force": "best-practice", "source_version": "2023", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "RE:05", "fit": "direct", "rationale": "Microsoft Azure Well-Architected Reliability recommendation RE:05 requires designing for redundancy across availability zones and regions, with failover behavior validated through testing. Multi-region and multi-AZ verification for AI workloads directly implements this recommendation.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "iso_22301", "requirement_id": "\u00a78.4.5", "fit": "partial", "rationale": "ISO 22301:2019 \u00a78.4.5 (Recovery) requires documented procedures to restore and return business activities from the measures adopted during a disruption. Verified multi-region failover is the cloud-native implementation of those recovery arrangements for AI services (\u00a78.4.4 covers the business continuity plans themselves).", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 12(1)-(2)", "fit": "direct", "rationale": "EU DORA Article 12(1)-(2) requires backup policies and documented restoration and recovery procedures and methods, including backup systems that can be activated without jeopardising security or availability. Verified multi-region and multi-AZ failover demonstrates those restoration capabilities under geographic failure (Article 12(5)'s secondary-processing-site requirements bind central securities depositories specifically).", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_csf", "requirement_id": "PR.IR-04", "fit": "partial", "rationale": "NIST CSF 2.0 PR.IR-04 requires that adequate resource capacity to ensure availability is maintained. Multi-region deployment with verified failover is a primary mechanism for maintaining that capacity when a zone or region is lost.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "google_sre", "requirement_id": "Error budget policies \u2013 Multi-region SLO maintenance and failover", "fit": "partial", "rationale": "RV-06 requires that multi-region and multi-AZ AI workload failover procedures be verified to function correctly and that AI behavior is consistent across regions. Google SRE error budget policies and SLO management address multi-region availability by requiring that error budgets account for regional failure scenarios and that failover mechanisms are proven to maintain SLO compliance during region loss events. Verified cross-region failover per RV-06 is a prerequisite for credibly maintaining SLO commitments across geographic failure domains.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "cloudflare_resilience", "requirement_id": "Load balancing and failover \u2013 Geo-distributed AI inference endpoint resilience", "fit": "direct", "rationale": "RV-06 requires that geographic failover and cross-region recovery capabilities for AI systems be verified through controlled testing, confirming that model versions are consistent and routing automation works correctly. Cloudflare's load balancing and failover DNS capabilities are purpose-built for multi-region traffic management, providing automatic failover between AI inference endpoints across geographic regions with health-check-driven routing. Testing Cloudflare load balancer failover configuration is a direct verification mechanism for the multi-region resilience that RV-06 requires AI systems to demonstrate.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RV-06", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "For each multi-region or multi-AZ AI system deployment, AZ-level failover must be verified quarterly and region-level failover annually through controlled tests confirming that traffic routes correctly within the declared RTO, AI model versions are consistent across all regions, and AI system output quality is functionally equivalent in the failover region.", "evidence_required": [ "regional_deployment_map per AI system documenting all inference endpoints, model storage locations, feature stores, and dependencies per region and availability zone", "az_failover_test_record with test_date, simulated_az, actual_failover_time_seconds, traffic_routing_verified flag, and rto_met flag for each quarterly AZ-level test", "region_failover_test_record with test_date, simulated_primary_region, failover_region, actual_failover_time_seconds, model_version_consistency_verified flag, output_quality_verified flag, and rto_met flag for each annual region test", "model_version_consistency_report showing model_version per region at time of test and confirming versions are identical across all deployed regions" ], "machine_tests": [ "Query az_failover_test_record per tier-1 AI system \u2192 assert at least one test in the last 90 days with rto_met=true and traffic_routing_verified=true", "Query region_failover_test_record \u2192 assert at least one test in the last 365 days per AI system with model_version_consistency_verified=true and output_quality_verified=true", "Compare model_version per region in model_version_consistency_report \u2192 assert all regions serving inference show identical model_version values with drift_detected=false", "Check region_failover_test_record.actual_failover_time_seconds \u2192 assert value \u2264 (declared_rto_minutes \u00d7 60) for every test" ], "human_review": [ "Review failover test procedures to confirm that security controls \u2014 identity providers, authorization services, secrets management, and audit logging \u2014 were explicitly verified as available and functional in the failover region after each test", "Assess whether AI output quality verification in the failover region used representative test inputs across the full capability range, not just a minimal smoke test that could miss behavioral divergence introduced by configuration differences", "Evaluate whether on-call engineers have personally executed failover procedures under drill conditions, rather than relying solely on automation, to confirm they can manually intervene when automatic failover fails" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Assuming multi-region resilience is functioning correctly because the infrastructure was correctly configured at deployment time, without executing failover tests to detect configuration drift introduced by subsequent changes", "Verifying only that traffic routes to the failover region without checking that model versions are consistent across regions, AI system behavior is equivalent, and security controls are active in the failover region", "Executing failover tests during non-representative conditions (empty traffic, minimal load) that do not reveal resource contention, capacity limits, or AI inference queue backpressure that would occur during a real regional failure", "Declaring failover successful based on HTTP 200 response from the failover region without executing AI functional quality checks that confirm the inference pipeline is producing correct outputs", "Maintaining failover runbooks that reference infrastructure or configuration details that have become stale, so that manual failover under real incident conditions fails at a step that worked during the last documented drill" ], "update_status": "current", "layer_code": "RV" }, { "id": "RV-07", "layer": "RV", "plane": "control", "name": "Degraded-Mode AI Operation Testing", "plain": "AI systems must be designed and tested to operate in a useful degraded mode when full capabilities are unavailable, with degraded-mode behavior explicitly defined, tested under simulated partial failure conditions, and validated as acceptable by business stakeholders before being relied upon as a resilience strategy.", "threat": { "tags": [ "full-outage-dependency", "graceful-degradation-failure", "user-impact", "capability-loss" ], "desc": "AI systems that have no defined degraded-mode behavior become completely unavailable when any component of their full capability stack is disrupted. Partial failures that could be tolerated with graceful degradation instead cascade into total service loss. Degraded modes that have not been tested before production use frequently deliver outputs of unacceptable quality or exhibit unexpected behaviors that create new risks rather than maintaining acceptable service levels." }, "standard": [ { "id": "google_sre", "section": "Ch. 22", "title": "Addressing Cascading Failures \u2014 load shedding and graceful degradation" }, { "id": "nist_800_160_v2", "section": "\u00a72.1.2", "title": "Cyber resiliency objectives \u2014 Continue (maintain essential functions during adversity)" }, { "id": "aws_reliability", "section": "REL 11", "title": "Design your workload to withstand component failures" }, { "id": "nist_csf", "section": "RC.RP-02", "title": "Recovery actions selected, scoped and performed" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RV-07 Degraded-Mode AI Operation Testing control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RV-07 Degraded-Mode AI Operation Testing control.", "reviewed_on": "2026-07-02" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RV-07 Degraded-Mode AI Operation Testing control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RV-07 Degraded-Mode AI Operation Testing control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RV-07 Degraded-Mode AI Operation Testing control.", "reviewed_on": "2026-07-01" }, { "id": "cloudflare_resilience_2024", "title": "Cloudflare DDoS Protection", "authority": "Cloudflare, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.cloudflare.com/ddos/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "cloudflare_resilience_2024", "relationship": "informative_reference", "rationale": "Establishes Cloudflare DDoS Protection requirements informing the apeiris://resilience/controls/RV-07 Degraded-Mode AI Operation Testing control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Define explicit degraded-mode behavior for each AI system capability tier, obtain business stakeholder acceptance of degraded-mode output quality, build degraded-mode logic into AI system architecture, and test degraded-mode activation and output under simulated partial failure conditions in staging and production environments.", "steps": [ "For each AI system, define the degraded-mode capability tiers (e.g., full capability, cached-response mode, simplified-model fallback, human-in-the-loop escalation, service unavailable) with clear trigger conditions for each tier.", "Obtain formal business stakeholder acceptance of the output quality and limitations for each degraded-mode tier before the system is deployed in a configuration that relies on that tier.", "Implement degraded-mode logic in the AI system architecture using patterns such as circuit breakers, response caching, fallback model routing, and feature flag-controlled capability reduction.", "Test degraded-mode activation by injecting the trigger conditions in a staging environment and verifying that the system transitions correctly, produces acceptable outputs, and surfaces appropriate user-facing notifications.", "Execute quarterly degraded-mode tests in production using feature flags or controlled partial failures, validating that outputs remain within stakeholder-accepted quality bounds." ], "site_reliability": { "summary": "Degraded-mode operation is an SRE design principle: design your system for graceful degradation from the start and verify that every degraded tier actually activates and behaves as intended.", "actions": [ "Define SLOs for each degraded-mode tier and instrument the system to detect which tier is active and whether SLOs for that tier are being met.", "Build automated degraded-mode activation into the system architecture so it does not require manual operator intervention during incidents.", "Test degraded-mode transitions regularly using feature flags in production during low-traffic periods." ], "failure_signals": [ "AI system has no defined degraded-mode behavior \u2014 it is either fully functional or completely unavailable.", "Degraded-mode logic has not been tested under realistic trigger conditions.", "Degraded-mode SLOs are not defined or monitored separately from full-capability SLOs." ] }, "security_architect": { "summary": "Degraded-mode operation must not compromise security properties. Verify that fallback models, cached responses, and simplified processing pipelines maintain the security and privacy constraints applied to the full-capability mode.", "actions": [ "Review degraded-mode implementations for security control bypasses that may be introduced by fallback paths.", "Confirm that cached responses in degraded mode do not expose stale data that violates data residency or access control requirements.", "Ensure fallback models meet the same security review requirements as primary models." ], "failure_signals": [ "Degraded-mode fallback paths bypass security controls that are active in full-capability mode.", "Cached responses in degraded mode include data not authorized for the requesting user.", "Fallback models have not undergone security review." ] }, "it_operations": { "summary": "Operations teams must know which degraded-mode tier each AI system is in at all times, how to manually override tier transitions when automation fails, and what user-facing communications to initiate for each tier.", "actions": [ "Deploy monitoring dashboards that surface the current degraded-mode tier for all production AI systems.", "Maintain runbooks for manual degraded-mode tier transitions for scenarios where automatic activation fails.", "Define customer communication templates for each degraded-mode tier and integrate them into the incident communication workflow." ], "failure_signals": [ "Operations team cannot determine which degraded-mode tier a production AI system is in without reviewing application logs.", "No manual override procedure exists for degraded-mode activation.", "Customer-facing communications during degraded-mode incidents are improvised rather than following a defined template." ] }, "business_continuity": { "summary": "Degraded-mode operation is a resilience strategy that must be formally accepted by business stakeholders \u2014 it is not solely an engineering decision. Business continuity planning must document which degraded-mode tiers are acceptable for which business processes and under what time constraints.", "actions": [ "Facilitate business stakeholder review and formal acceptance of degraded-mode output quality for each AI system tier.", "Document degraded-mode acceptability windows: how long each degraded tier can be sustained before business impact becomes unacceptable.", "Include degraded-mode operating assumptions in business continuity plan scenarios for AI system disruptions." ], "failure_signals": [ "Degraded-mode tiers were defined by engineering without formal business stakeholder acceptance.", "No maximum acceptable duration is documented for sustained degraded-mode operation.", "Business continuity scenarios assume either full AI capability or full AI unavailability with no degraded-mode intermediate." ] }, "grc_auditor": { "summary": "Degraded-mode operation is a documented, approved operating state \u2014 regulators and auditors will ask who authorized reduced AI functionality and how users were informed. Verify definitions, approvals, and test evidence exist.", "actions": [ "Confirm each critical AI system has a documented, business-approved degraded-mode definition with acceptance sign-off.", "Sample degraded-mode test records for cadence and stakeholder validation of degraded outputs.", "Verify user-facing disclosures and internal notifications for degraded operation are defined and exercised." ], "failure_signals": [ "Degraded modes exist in code but have no documented business acceptance.", "Tests validate that degradation activates but never that its outputs remain acceptable.", "No evidence shows stakeholders were informed during past degraded-mode periods." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Graceful degradation is well-understood as a software engineering principle but is rarely implemented and tested specifically for AI inference pipelines, model fallback scenarios, and AI-specific capability tiers." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "multi-tenant" ], "implementers": [ "Site Reliability Engineering", "Product Engineering", "Platform Engineering" ], "frameworks": [ { "framework": "google_sre", "requirement_id": "Ch. 22", "fit": "direct", "rationale": "Google SRE (SRE Book, 2016) Chapter 22, Addressing Cascading Failures, treats load shedding and graceful degradation as first-class mechanisms for keeping a service useful when full capacity or functionality is unavailable (Chapter 26 covers data integrity). The degraded-mode testing this control requires validates exactly those mechanisms for AI systems.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "nist_800_160_v2", "requirement_id": "\u00a72.1.2", "fit": "direct", "rationale": "NIST SP 800-160 Vol 2 Rev 1 \u00a72.1.2 defines the cyber resiliency objective Continue \u2014 maximizing the duration and viability of essential mission and business functions during adversity. Explicitly defined and tested degraded-mode operation is how an AI system demonstrates that objective.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "REL 11", "fit": "direct", "rationale": "AWS Well-Architected Reliability Pillar REL 11 (Design your workload to withstand component failures) requires workloads to keep operating \u2014 possibly in a degraded state \u2014 when components fail, and for that behavior to be validated. Degraded-mode testing implements that validation for AI capability stacks (testing practices generally are REL 12).", "normative_force": "best-practice", "source_version": "2023", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "iso_22301", "requirement_id": "\u00a78.4.5", "fit": "partial", "rationale": "ISO 22301:2019 \u00a78.4.5 requires that continuity procedures address operating in a degraded mode until full capability is restored. While the standard addresses this at the business process level, the principle applies directly to AI systems that must maintain useful degraded operation during recovery periods.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "RC.RP-02", "fit": "adjacent", "rationale": "NIST CSF 2.0 RC.RP-02 requires that recovery actions be selected and scoped based on the nature and extent of the incident. Degraded-mode operation is a recovery action selection for partial capability loss, and its pre-defined, tested nature makes it available as a scoped response rather than requiring improvisation during incidents.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "microsoft_azure_resil", "requirement_id": "BCDR \u2013 Graceful degradation design and availability zone failover", "fit": "partial", "rationale": "RV-07 requires AI systems to have explicitly defined and tested degraded-mode operation for scenarios where full capability stacks are unavailable. Microsoft Azure BCDR and Resiliency guidance addresses graceful degradation through availability zone design, recommending that workloads define acceptable degraded states for partial zone failures and validate those states through controlled testing. Azure's availability zone guidance provides architectural patterns for implementing and testing the degraded modes that RV-07 requires AI systems to demonstrate.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "cloudflare_resilience", "requirement_id": "Rate limiting for API protection", "fit": "adjacent", "rationale": "Cloudflare rate limiting can throttle or shed traffic to an AI API at the edge during overload or upstream failure. This complements degraded-mode operation by constraining load while a reduced service runs, but the degraded behavior itself must be designed and validated in the AI application \u2014 the edge cannot define or enforce it.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RV-07", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "The AI system must have formally defined and stakeholder-accepted degraded-mode capability tiers with documented trigger conditions, and each degraded-mode tier must have been tested under its trigger conditions in a staging environment and validated for output acceptability within the last 12 months, with business stakeholder sign-off confirming each tier's outputs are acceptable for the contexts in which it will be used.", "evidence_required": [ "degraded_mode_capability_registry with ai_system_id, tier_name, trigger_conditions, fallback_mechanism (circuit_breaker|cached_response|fallback_model|human_escalation|service_unavailable), and stakeholder_acceptance_signoff_date for each defined tier", "degraded_mode_test_record with test_date, tier_name, environment, trigger_injected, activation_confirmed flag, output_sample_reviewed flag, and output_quality_within_bounds flag for each tier", "stakeholder_acceptance_record per degraded tier with business_owner, output_quality_description, acceptable_use_contexts, and maximum_acceptable_duration_hours", "degraded_mode_monitoring_evidence showing current_active_tier is observable in production dashboards with tier-specific SLO thresholds defined" ], "machine_tests": [ "Query degraded_mode_capability_registry \u2192 assert all tiers have a non-null stakeholder_acceptance_signoff_date within the last 12 months and at least one linked test_record with activation_confirmed=true", "Inject trigger condition for each degraded tier in staging environment \u2192 assert system transitions to correct tier within 30 seconds and degraded_mode_monitoring reflects the active tier", "Review degraded_mode_test_record where output_quality_within_bounds=false \u2192 assert each has a linked remediation item or updated tier design before production use", "Verify circuit_breaker configuration for each fallback_mechanism=circuit_breaker tier \u2192 assert open_threshold and close_threshold parameters are set and tested against the defined trigger condition" ], "human_review": [ "Confirm that stakeholder acceptance records capture specific output quality limitations for each degraded tier rather than generic approval, and that the accepting business owner understood what degraded outputs actually look like based on test evidence", "Assess whether the defined degraded-mode tiers cover the realistic partial failure scenarios the AI system is likely to encounter, or whether significant failure modes exist that would leave the system with no defined degraded response", "Evaluate whether security controls and data access constraints are explicitly verified as maintained in each degraded-mode tier, particularly for fallback model and cached-response modes that may use different code paths" ], "blocking_effect": "blocks-deployment", "normative_status": "voluntary-standard", "anti_patterns": [ "Declaring that the AI system 'supports graceful degradation' by returning HTTP 503 responses without defining intermediate degraded capability tiers that provide reduced but still useful service to callers", "Obtaining business stakeholder sign-off on degraded-mode tiers without showing stakeholders actual sample outputs from degraded operation, resulting in acceptance based on description rather than evidence", "Testing degraded-mode activation only by confirming the circuit breaker trips, without validating that the fallback path produces outputs of acceptable quality rather than degraded, misleading, or harmful outputs", "Implementing fallback model routes in degraded mode without applying the same security review, output filtering, and safety guardrails that are active in the primary inference path", "Defining degraded-mode tiers that require manual operator intervention to activate, removing the ability to respond within RTO when the failure occurs outside business hours or when operator capacity is constrained" ], "update_status": "current", "layer_code": "RV" }, { "id": "RV-08", "layer": "RV", "plane": "both", "name": "Resilience Verification Evidence Package", "plain": "An organization must compile, maintain, and produce on demand a current evidence package demonstrating that all RV-layer resilience verification controls are active, tested, and effective, providing auditors, regulators, and governance stakeholders with the artifacts required to assess the organization's AI system resilience posture.", "threat": { "tags": [ "evidence-gap", "audit-failure", "unverified-compliance", "control-attestation-gap" ], "desc": "Organizations that operate resilience controls without maintaining organized evidence packages face audit findings, regulatory penalties, and reputational damage when they cannot demonstrate control effectiveness on demand. Evidence that exists in disparate systems and is never aggregated cannot be quickly produced during regulatory examinations or after significant incidents. Gaps in evidence continuity suggest to auditors and regulators that controls may be operating inconsistently or may not have been operating at all." }, "standard": [ { "id": "iso_22301", "section": "\u00a79.1", "title": "Monitoring, measurement, analysis and evaluation" }, { "id": "dora", "section": "Art. 30", "title": "Key contractual provisions" }, { "id": "cobit_dss04", "section": "DSS04.05", "title": "Review, maintain and improve the continuity plans" }, { "id": "nist_csf", "section": "GV.RR-02", "title": "Roles, responsibilities, and authorities for risk management are established, communicated, and enforced" } ], "sources": [ { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RV-08 Resilience Verification Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RV-08 Resilience Verification Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RV-08 Resilience Verification Evidence Package control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RV-08 Resilience Verification Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RV-08 Resilience Verification Evidence Package control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Designate an RV evidence custodian, define the evidence package composition for each AI system (covering RV-01 through RV-07 artifacts), implement a collection cadence aligned to control testing schedules, store evidence in a tamper-evident repository, and review package completeness quarterly with sign-off from the business continuity program owner.", "steps": [ "Define the required evidence artifacts for each RV control (RV-01 through RV-07) covering recovery objectives documentation, test execution records, chaos experiment logs, post-incident reviews, third-party assessments, failover test results, and degraded-mode acceptance records.", "Designate an evidence custodian responsible for collecting, organizing, and maintaining RV-layer evidence for each production AI system scope.", "Implement an evidence collection workflow that automatically captures test results, exercise reports, and assessment outputs into a structured repository organized by AI system, control, and evidence date.", "Conduct quarterly evidence completeness reviews comparing collected artifacts against the required evidence manifest, documenting gaps and assigning remediation owners.", "Generate a periodic evidence summary report (minimum annually, or upon regulatory request) attesting to the completeness, currency, and effectiveness of RV-layer controls, with sign-off from the business continuity program owner." ], "grc_auditor": { "summary": "The RV evidence package is the primary artifact for demonstrating resilience verification compliance to auditors and regulators. Evidence packages that are incomplete, stale, or disorganized substantially increase audit findings risk.", "actions": [ "Review the evidence manifest for each production AI system scope and verify that all required RV-01 through RV-07 artifacts are present.", "Sample evidence artifacts for completeness, authenticity, and currency \u2014 confirm that evidence dates align with stated control execution schedules.", "Verify that evidence completeness review records exist and that gaps identified in prior reviews have been resolved." ], "metrics": [ "Evidence completeness rate: target 100% of required artifacts present for each AI system in scope.", "Evidence currency rate: target 100% of time-sensitive artifacts (test records, assessments) within their refresh interval.", "Quarterly review completion rate: target 100% of scheduled reviews completed with sign-off." ], "failure_signals": [ "Any required RV-layer evidence artifact is absent for a production AI system in scope.", "Evidence artifacts are more than 30 days past their required refresh date.", "Quarterly completeness review has not been conducted or signed off in the current review period." ] }, "business_continuity": { "summary": "The evidence package demonstrates that the organization's investment in AI system resilience is producing real, measurable, and verifiable protection \u2014 not just documented policy. Evidence package ownership belongs to the business continuity program.", "actions": [ "Own the evidence package manifest definition and ensure it reflects current regulatory and contractual obligations.", "Sign off quarterly on evidence completeness reviews as the business continuity program owner.", "Use the evidence package as the primary input for the annual business continuity program maturity assessment." ], "failure_signals": [ "Business continuity program owner has not reviewed or signed off on evidence completeness in the current quarter.", "Evidence package manifest has not been updated to reflect changes in regulatory obligations or in-scope AI systems.", "Annual maturity assessment was conducted without reference to the RV evidence package." ] }, "it_operations": { "summary": "IT operations teams are responsible for generating and submitting the raw evidence artifacts \u2014 test records, exercise reports, and operational metrics \u2014 that populate the RV evidence package.", "actions": [ "Ensure all resilience test executions produce structured output records that are automatically submitted to the evidence repository.", "Confirm that evidence repository access controls prevent modification of historical artifacts after submission.", "Flag any evidence collection failure to the GRC team immediately rather than waiting for the quarterly review cycle." ], "failure_signals": [ "Test executions do not produce structured evidence records suitable for the evidence repository.", "Evidence artifacts can be modified after submission without an audit trail.", "Evidence collection failures are not reported until discovered during a quarterly review." ] }, "site_reliability": { "summary": "SRE teams own the technical evidence artifacts (test results, chaos experiment logs, failover metrics) and must ensure they are accurate, complete, and automatically captured \u2014 not reconstructed from memory after the fact.", "actions": [ "Automate evidence capture at test execution time so that records are generated as an integral part of the test process, not as a manual follow-up step.", "Ensure chaos experiment and failover test logs include sufficient technical detail to demonstrate what was tested, what was observed, and whether the outcome met the hypothesis.", "Alert on test execution events that did not produce a corresponding evidence record." ], "failure_signals": [ "Evidence records are manually created after test completion rather than automatically generated.", "Test records lack sufficient technical detail to demonstrate what was tested and what outcome was observed.", "Evidence gaps are not detected until a quarterly GRC review." ] }, "security_architect": { "summary": "The verification evidence package must be trustworthy: integrity-protected, access-controlled, and complete enough to withstand adversarial scrutiny. Treat evidence tampering as a threat model.", "actions": [ "Require hash-chaining or signatures on evidence artifacts and verify them at package assembly.", "Enforce least-privilege write access to the evidence repository with immutable retention.", "Include security-control verification results (from RV-02/RV-04) in the package scope." ], "failure_signals": [ "Evidence artifacts can be edited after the fact without trace.", "The package omits security-relevant test failures.", "Repository access lists include accounts with no evidence-custodian role." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Most organizations can produce some resilience evidence on request but lack a systematic evidence package with defined composition, collection cadence, and completeness verification specific to AI system resilience verification controls." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai" ], "implementers": [ "GRC Team", "Business Continuity Team", "IT Operations" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a79.1", "fit": "direct", "rationale": "ISO 22301:2019 \u00a79.1 requires that the organization determine what evidence is needed to demonstrate conformity and evaluate performance, maintain documented information as evidence of monitoring and measurement results, and make this evidence available to interested parties. The RV evidence package directly fulfills these requirements for the resilience verification domain.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 30", "fit": "partial", "rationale": "EU DORA Article 30 prescribes the key contractual provisions financial entities must maintain with ICT third-party service providers, including full service level descriptions and monitoring and audit rights. The RV evidence package preserves the third-party resilience assessment evidence (RV-05) that exercising those rights presupposes; documentation duties toward authorities sit in Articles 5-16, not Article 30.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.05", "fit": "direct", "rationale": "COBIT 2019 DSS04.05 requires continuity plans and capabilities to be reviewed, maintained and improved on a defined cadence, with results reported to management. The quarterly-signed evidence package demonstrates that review discipline for AI resilience verification (DSS04.07 covers backup arrangements).", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.5", "fit": "direct", "rationale": "NIST SP 800-34 Rev 1 \u00a73.5 requires contingency plan tests and exercises to be documented, with results and lessons learned retained. The RV evidence package satisfies this by preserving test results and remediation records for every AI resilience verification activity.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "GV.RR-02", "fit": "partial", "rationale": "NIST CSF 2.0 GV.RR-02 requires that risk management roles, responsibilities, and authorities are established, communicated, understood, and enforced. The evidence custodian designation and quarterly sign-off process in this control implement that accountability for resilience verification evidence.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "REL 13 \u2013 Documented test results from reliability verification activities", "fit": "partial", "rationale": "RV-08 requires an evidence package demonstrating that all resilience verification controls are active and tested, producible on demand for auditors and governance stakeholders. AWS Well-Architected Reliability Pillar REL-13 requires that game days and fault injection testing produce documented results that demonstrate reliability control effectiveness. The test result artifacts from AWS reliability testing practices constitute a substantial portion of the evidence package that RV-08 requires organizations to maintain.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Postmortem practices and PRR \u2013 Resilience evidence from SRE operational artifacts", "fit": "direct", "rationale": "RV-08 requires a maintained evidence package demonstrating resilience verification control effectiveness, producible for auditors and regulators on demand. Google SRE postmortem documents, Production Readiness Review sign-offs, and error budget reports together constitute an operational evidence corpus that directly demonstrates resilience control function, test history, and continuous improvement. Organizing SRE artifacts per the RV-08 evidence package structure provides the auditability layer over existing SRE practices that regulators and governance stakeholders require.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "Business continuity planning \u2013 BCDR evidence documentation and audit readiness", "fit": "partial", "rationale": "RV-08 mandates that resilience verification evidence be compiled and maintained in an organized package for regulatory and governance review. Microsoft Azure BCDR guidance prescribes documentation standards for business continuity plans, test results, and recovery validation artifacts that demonstrate ongoing BCDR program effectiveness. For AI systems on Azure, the BCDR evidence artifacts \u2014 failover test reports, recovery objective validation records, continuity plan reviews \u2014 form the Azure-specific component of the evidence package RV-08 requires.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RV-08", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "The organization must maintain a structured RV evidence package that contains current, complete artifacts for every RV-01 through RV-07 control across all production AI systems in scope, with a completed quarterly completeness review signed off by the business continuity program owner, and the package must be producible on demand within 24 hours of a regulatory request.", "evidence_required": [ "RV evidence manifest listing each required artifact type per control per scoped AI system, with the name of the artifact, required refresh frequency, last collected date, and present/missing status", "Quarterly completeness review records showing percentage of required artifacts present, identified gaps with assigned remediation owners, and sign-off signature from the business continuity program owner", "Evidence custodian designation record naming the individual responsible for each AI system scope and the date of designation", "Repository access log confirming tamper-evident storage with no artifact modifications after submission", "Evidence summary report attesting to completeness and effectiveness of RV-layer controls, with program owner sign-off and generation date" ], "machine_tests": [ "Query evidence repository for all RV-01 through RV-07 required artifact types per each scoped AI system \u2192 assert zero missing required artifacts", "Request evidence package via API or export workflow \u2192 assert package assembles within 60 seconds and includes artifacts for all seven RV controls with non-expired timestamps", "Attempt to modify a submitted evidence artifact in the repository \u2192 assert modification is rejected or produces an immutable audit log entry", "Check artifact timestamp for each time-sensitive evidence item \u2192 assert no required artifact is older than its documented refresh interval" ], "human_review": [ "Review the evidence manifest definition to confirm it captures all required RV-01 through RV-07 artifact types and covers all production AI systems in scope", "Sample 20% of evidence artifacts for authenticity and completeness \u2014 confirm records represent actual control execution rather than post-hoc reconstruction", "Verify that quarterly completeness review records are present for each review period and that identified gaps from prior reviews have been closed" ], "blocking_effect": "advisory", "normative_status": "binding-law", "anti_patterns": [ "Reconstructing evidence records after test completion rather than capturing them automatically at execution time, producing records that cannot be independently verified", "Storing evidence artifacts in siloed team repositories without a central manifest, making package assembly for regulatory requests impossible within reasonable time", "Treating evidence collection as a pre-audit activity that begins when an examination is announced rather than as a continuous operational process", "Allowing evidence artifacts to be modified or deleted after submission without generating an immutable audit trail entry", "Conducting quarterly completeness reviews without a defined manifest of required artifacts, so gap identification depends on reviewer memory rather than systematic comparison" ], "update_status": "current", "layer_code": "RV" }, { "id": "RP-01", "layer": "RP", "plane": "lifecycle", "name": "AI-Specific Business Continuity Plan", "plain": "Business continuity plans must explicitly address AI system failure scenarios, including model service unavailability, training data corruption, inference pipeline disruption, and AI-specific recovery objectives that are distinct from conventional IT BCP scope.", "threat": { "tags": [ "model-unavailability", "data-corruption", "continuity-gap", "unplanned-downtime" ], "desc": "Generic IT business continuity plans fail to address AI-specific failure modes such as model degradation, embedding drift, or provider API outage. Without an AI-specific BCP, organizations default to improvised responses during AI failures, extending downtime and increasing the risk of unsafe fallback behaviors that bypass safety controls. RTO and RPO targets set for conventional IT systems are rarely calibrated to the complexity of AI pipeline recovery." }, "standard": [ { "id": "iso_22301", "section": "\u00a78.2.2", "title": "Business impact analysis" }, { "id": "nist_sp800_34", "section": "\u00a73.1", "title": "BCP development" }, { "id": "iso_27031", "section": "Cl. 8", "title": "IRBC recovery time and recovery point objectives (ISO/IEC 27031:2025)" }, { "id": "cobit_dss04", "section": "DSS04.01", "title": "Define business continuity policy" } ], "sources": [ { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RP-01 AI-Specific Business Continuity Plan control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RP-01 AI-Specific Business Continuity Plan control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RP-01 AI-Specific Business Continuity Plan control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RP-01 AI-Specific Business Continuity Plan control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RP-01 AI-Specific Business Continuity Plan control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Extend the enterprise BCP with an AI Annex that catalogs each AI system, defines AI-specific failure scenarios, documents manual fallback procedures, sets AI-specific RTO/RPO targets, and assigns AI system owners as named BCP stakeholders.", "steps": [ "Inventory all production AI systems and classify their business criticality and failure impact using a structured business impact analysis.", "Define AI-specific failure scenarios (model API outage, inference latency spike, embedding corruption, training data poisoning) and map each to a documented BCP response procedure.", "Establish AI-specific RTO and RPO targets distinct from general IT targets, validated against business impact analysis outputs.", "Assign AI system owners as named BCP stakeholders with defined roles and escalation authority during AI-related continuity events.", "Review and update the AI BCP annex at least annually and after any significant model, infrastructure, or provider change." ], "business_continuity": { "summary": "The AI BCP annex is the primary artifact demonstrating governance of AI failure scenarios within the enterprise continuity program.", "actions": [ "Lead the development and annual review of the AI BCP annex.", "Facilitate business impact analysis sessions with AI system owners to quantify downtime costs and tolerable outage periods.", "Ensure AI-specific RTO/RPO targets are approved by executive sponsors before publication." ], "failure_signals": [ "BCP does not contain an AI-specific annex or section.", "AI system owners are not listed as BCP stakeholders.", "RTO/RPO targets for AI systems have not been reviewed in over 12 months." ] }, "it_operations": { "summary": "IT Operations executes BCP procedures during AI continuity events and is responsible for technical fallback activation.", "actions": [ "Implement and test technical fallback procedures for each AI system listed in the BCP annex.", "Maintain runbooks for switching between primary and fallback AI inference paths.", "Report AI system availability metrics to BCP stakeholders monthly." ], "failure_signals": [ "Fallback procedures are undocumented or untested for any tier-1 AI system.", "Runbooks have not been updated to reflect current infrastructure within the last 6 months.", "AI system availability metrics are not tracked or reported to BCP stakeholders." ] }, "grc_auditor": { "summary": "GRC validates that the AI BCP annex meets regulatory and certification requirements and that evidence of annual review is available for audit.", "actions": [ "Request the current AI BCP annex and verify it addresses each major AI system failure scenario.", "Cross-reference AI-specific RTO/RPO targets against business impact analysis outputs.", "Verify that BCP review records exist and are dated within the last 12 months." ], "metrics": [ "AI systems covered by BCP annex: target 100% of production systems.", "BCP review currency: last review within 12 months.", "Named AI BCP stakeholders per tier-1 system: minimum 1." ], "failure_signals": [ "BCP annex missing for any tier-1 or tier-2 AI system.", "No evidence of annual review in the last audit cycle.", "AI RTO/RPO targets not validated against business impact analysis." ] }, "security_architect": { "summary": "The AI BCP must keep security invariants intact under continuity conditions: alternate processing paths, emergency vendors, and manual workarounds all need pre-assessed security postures.", "actions": [ "Review BCP alternate paths and workarounds for authentication, data protection, and logging parity before approval.", "Pre-assess and document the security posture of any emergency vendor or fallback service named in the plan.", "Define in the plan which security controls may never be suspended, even under continuity pressure." ], "failure_signals": [ "The BCP directs traffic to fallback services that were never security-assessed.", "Manual workarounds move regulated data over unapproved channels.", "Continuity activations suspend logging or review controls with no compensating measures." ] }, "site_reliability": { "summary": "The BCP is only executable if its assumptions match production reality. Keep the plan wired to current architecture, dependency maps, and measured recovery capabilities.", "actions": [ "Auto-generate the BCP's system inventory and dependency map from the service catalog and deployment metadata.", "Validate plan assumptions (capacity of fallback paths, restore durations) against measured data each review cycle.", "Trigger BCP review on every major architecture change via the change-management pipeline." ], "failure_signals": [ "The plan references decommissioned services or stale endpoints.", "Documented fallback capacity has never been load-tested.", "Architecture changed materially since the last plan revision." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most enterprises extend generic IT BCP without AI-specific failure scenarios or RTO/RPO targets calibrated to AI pipeline complexity." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "Business Continuity Team", "IT Operations", "AI Platform Team", "GRC" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a78.2.2", "fit": "direct", "rationale": "ISO 22301:2019 \u00a78.2.2 requires a business impact analysis that identifies activities, their dependencies, and prioritized timeframes for resumption. AI systems with real-time inference dependencies fall squarely within scope; the AI-specific BCP builds directly on that BIA output (\u00a78.3 covers strategy selection).", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.1", "fit": "direct", "rationale": "NIST SP 800-34 \u00a73.1 defines the contingency planning process that forms the structural basis for BCP development. Its system categorization and mission-essential function mapping applies directly to AI workloads when extended with an AI-specific failure mode taxonomy.", "normative_force": "best-practice", "source_version": "Rev 1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_27031", "requirement_id": "Cl. 8", "fit": "direct", "rationale": "ISO/IEC 27031:2025 Clause 8 requires recovery time and recovery point objectives to be established for ICT services supporting business continuity. AI inference pipelines are ICT services whose RTO/RPO must be defined in the AI business continuity plan.", "normative_force": "certification-standard", "source_version": "2025", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.01", "fit": "direct", "rationale": "COBIT 2019 DSS04.01 requires organizations to define continuity policies that establish scope, objectives, and governance responsibilities. AI systems must be explicitly scoped into this policy with AI-specific continuity objectives and named system owners.", "normative_force": "best-practice", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 11", "fit": "direct", "rationale": "EU DORA Article 11 requires financial entities to establish, maintain, and test ICT business continuity policies and plans. AI systems used in financial decision-making are explicitly within DORA scope and require dedicated AI-specific continuity planning documentation.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_csf", "requirement_id": "RC.RP-01", "fit": "direct", "rationale": "NIST CSF 2.0 RC.RP-01 requires that recovery plan execution is initiated and documented. The AI-specific BCP provides the documented plan baseline against which recovery execution is measured and audited for AI workloads.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "anthropic_rsp", "requirement_id": "Capability thresholds and ASL-3 standards \u2014 provider-initiated deployment restrictions", "fit": "adjacent", "rationale": "The Anthropic Responsible Scaling Policy (v3.3) binds Anthropic, not deployers: when capability thresholds are reached, ASL-3 deployment and security standards can lead Anthropic to restrict or condition model availability. An AI-specific BCP should treat such provider-initiated, safety-driven restrictions as a disruption scenario alongside conventional outages.", "normative_force": "best-practice", "source_version": "3.3", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "openai_preparedness", "requirement_id": "\u00a74 Safeguarding \u2014 safeguard-driven deployment decisions", "fit": "adjacent", "rationale": "The OpenAI Preparedness Framework v2 binds OpenAI: under \u00a74 (Safeguarding), deployments of models reaching High capability thresholds require safeguards sign-off, and safeguard decisions can restrict or modify model availability. Deployer BCPs should account for these safety-driven availability changes as a planning scenario.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "aws_reliability", "requirement_id": "REL 13", "fit": "direct", "rationale": "AWS Well-Architected Reliability Pillar REL 13 (Plan for disaster recovery) prescribes defining RTO and RPO and selecting a DR strategy as foundational inputs to continuity planning \u2014 directly applicable to AI workloads and their model artifacts, indexes, and serving infrastructure (REL 10 covers fault isolation, not DR).", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Production readiness review \u2014 Availability requirements", "fit": "partial", "rationale": "Google SRE Production Readiness Review practices require AI services to demonstrate availability readiness before launch and after significant changes, establishing that BCP-relevant information \u2014 service owners, escalation paths, and defined recovery targets \u2014 must be documented as launch prerequisites. The AI-Specific Business Continuity Plan's requirement to assign AI system owners as named BCP stakeholders and to define AI-specific failure scenarios maps directly to PRR availability and escalation requirement categories. Applying SRE PRR discipline to AI systems during BCP planning ensures that continuity assumptions are validated against measurable readiness criteria rather than assumed.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "BCDR design guidance \u2014 Business continuity planning methodology", "fit": "direct", "rationale": "Microsoft Azure Resiliency and BCDR design guidance provides a structured business continuity planning methodology directly applicable to AI workloads hosted on Azure OpenAI Service, Azure AI Services, and Azure Machine Learning, covering how to establish recovery objectives and map business function dependencies. The AI-Specific Business Continuity Plan requirement for an AI annex with AI-specific failure scenarios and named system owners aligns with Azure BCDR's recommended structure for service-scoped continuity plans. Organizations using Azure AI services should incorporate Azure BCDR's service-specific availability documentation as a reference for failure scenario identification within their AI BCP annex.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RP-01", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every production AI system must be explicitly named in a formally documented AI BCP annex with at least one named system owner, a defined set of AI-specific failure scenarios (including model service unavailability, training data corruption, and inference pipeline disruption), AI-specific RTO and RPO targets validated against business impact analysis findings, and an annual review record with executive sponsor sign-off.", "evidence_required": [ "AI BCP annex document listing all production AI systems with criticality tier, named system owner, and at least three AI-specific failure scenarios with mapped response procedures per system", "Business impact analysis output showing quantified downtime cost and tolerable outage period per AI system, serving as the basis for RTO/RPO target selection", "RTO/RPO target validation record per AI system showing alignment with BIA findings and confirmation that technical recovery capability has been measured against the target", "Annual review record with executive sponsor signature, review date, and a description of changes made since the prior review", "Named BCP stakeholder roster showing AI system owner assignments with role, contact information, and escalation authority" ], "machine_tests": [ "Query AI system inventory against BCP annex \u2192 assert every production AI system has a corresponding annex entry with non-null rto_minutes, rpo_minutes, owner_name, and at least three failure_scenarios", "Check BCP annex review date metadata \u2192 assert last_reviewed_on is within 365 days of current date for each AI system entry", "Cross-reference BCP annex AI system list against deployment registry \u2192 assert no production AI system deployed more than 30 days ago is absent from the annex" ], "human_review": [ "Review BCP annex failure scenarios for each AI system to confirm AI-specific failure modes (model API outage, embedding drift, inference latency spike) are addressed with concrete response procedures, not generic IT recovery steps", "Assess whether AI-specific RTO/RPO targets are meaningfully distinct from generic IT infrastructure targets and whether they account for model artifact restoration and pipeline dependency sequencing", "Verify that named stakeholders are current employees with the organizational authority described in the annex and that escalation paths reflect current reporting structures" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Applying generic IT RTO targets (such as a four-hour RTO for server restoration) to AI systems without accounting for model artifact rebuild time, vector index reconstruction, or provider API restoration dependencies", "Listing AI systems in the BCP annex without named system owners, making accountability ambiguous and escalation paths undefined during an actual continuity event", "Defining AI failure scenarios only as infrastructure failures (server down, network outage) without addressing AI-specific modes such as model performance degradation, embedding corruption, or provider-triggered model withdrawal", "Failing to update the AI BCP annex when AI systems are onboarded, retired, or significantly changed, causing the annex to reflect a stale system inventory", "Treating BCP annex development as an IT deliverable without requiring business stakeholder input on acceptable outage windows and degraded-mode tolerance" ], "update_status": "current", "layer_code": "RP" }, { "id": "RP-02", "layer": "RP", "plane": "lifecycle", "name": "AI System Disaster Recovery Plan", "plain": "Detailed disaster recovery plans must cover AI-specific recovery procedures including model rollback, inference pipeline restoration, knowledge base recovery, and integrity validation steps that must be completed before resuming production traffic to any recovered AI system.", "threat": { "tags": [ "inference-pipeline-failure", "model-corruption", "recovery-failure", "data-loss" ], "desc": "Standard IT DR plans do not account for AI-specific failure modes such as model weight corruption, vector index poisoning, or total inference cluster loss. Attempting to recover an AI system without a dedicated DR plan risks restoring a degraded or unsafe model state, reintroducing the failure condition, or resuming production traffic before integrity validation is complete \u2014 any of which may cause worse harm than the original failure." }, "standard": [ { "id": "nist_sp800_34", "section": "\u00a73.4", "title": "IT contingency plan development" }, { "id": "iso_27031", "section": "Cl. 10", "title": "IRBC plans and documentation (ISO/IEC 27031:2025)" }, { "id": "iso_22301", "section": "\u00a78.4", "title": "Recovery procedures" }, { "id": "nist_csf", "section": "RC.RP-02", "title": "Recovery plan execution" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RP-02 AI System Disaster Recovery Plan control.", "reviewed_on": "2026-07-01" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RP-02 AI System Disaster Recovery Plan control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RP-02 AI System Disaster Recovery Plan control.", "reviewed_on": "2026-07-02" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RP-02 AI System Disaster Recovery Plan control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RP-02 AI System Disaster Recovery Plan control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Develop an AI DR plan as a formal controlled document covering: pre-disaster asset inventory snapshot, step-by-step model and pipeline recovery procedures with explicit integrity validation gates, defined RTO/RPO targets per AI system tier, and responsible parties for each recovery phase.", "steps": [ "Document the full AI system asset inventory including model artifacts, inference endpoints, vector databases, and pipeline configurations used as the DR baseline.", "Define step-by-step recovery procedures for each AI component with explicit integrity validation gates that must pass before restoring production traffic.", "Establish RTO and RPO targets for each AI system tier and validate them against measured infrastructure recovery capability.", "Assign recovery roles and decision authority for each DR phase including technical lead, business sponsor, and communications owner.", "Integrate the AI DR plan into the enterprise DR program with coordinated activation triggers and joint testing schedules." ], "site_reliability": { "summary": "SRE owns the technical recovery runbooks that implement the AI DR plan and is responsible for validating system integrity before traffic restoration.", "actions": [ "Develop and maintain step-by-step recovery runbooks for each AI system component.", "Implement automated integrity validation checks that execute post-recovery before production traffic is restored.", "Track RTO achievement in DR tests and report gaps to the business continuity team." ], "failure_signals": [ "Recovery runbooks are absent for any tier-1 AI system.", "Post-recovery integrity validation is manual, absent, or untested.", "Last measured RTO in testing exceeded the documented target by more than 20%." ] }, "it_operations": { "summary": "IT Operations executes infrastructure recovery procedures and coordinates with AI platform teams during DR events.", "actions": [ "Maintain pre-tested restore procedures for AI inference infrastructure including container images, orchestration configurations, and network routing.", "Coordinate infrastructure recovery sequencing with AI platform teams to respect model dependency order.", "Document each DR activation with a post-recovery incident report within 5 business days." ], "failure_signals": [ "Infrastructure restore procedures have not been tested in the last 12 months.", "No documented dependency order exists for AI component recovery sequencing.", "Post-recovery incident reports are not produced following DR activations." ] }, "grc_auditor": { "summary": "GRC validates the AI DR plan exists as a formal document, covers all tier-1 AI systems, and has been reviewed and tested within the required period.", "actions": [ "Request the current AI DR plan and verify it covers all production AI systems by tier.", "Review RTO/RPO documentation and confirm targets were validated in the most recent DR test.", "Verify that recovery roles and decision authority are formally assigned and current." ], "metrics": [ "AI systems with documented DR procedures: target 100% of tier-1 and tier-2 systems.", "DR plan review currency: reviewed within last 12 months.", "RTO achievement rate in DR tests: target \u226590%." ], "failure_signals": [ "DR plan missing for any tier-1 AI system.", "RTO/RPO targets not validated in a DR test within the last 12 months.", "Recovery roles unassigned or not updated following personnel changes." ] }, "business_continuity": { "summary": "Business Continuity Manager ensures the AI DR plan is coordinated with the enterprise DR program and that business impact thresholds drive RTO/RPO targets.", "actions": [ "Review AI DR plan RTO/RPO targets against business impact analysis outputs annually.", "Coordinate AI DR test scheduling with the enterprise DR calendar.", "Escalate unresolved gaps between technical RTO capability and business impact thresholds to executive sponsors within 30 days of identification." ], "failure_signals": [ "AI DR plan is not integrated into the enterprise DR program.", "RTO/RPO targets are not traced to business impact analysis outputs.", "Capability gaps between technical recovery time and business tolerance are not escalated within 30 days." ] }, "security_architect": { "summary": "DR execution is a high-risk security window: restored environments, re-issued credentials, and bulk data movement all need pre-engineered secure paths in the DR plan.", "actions": [ "Embed secret/credential rehydration procedures with approvals into the DR plan rather than improvising at recovery time.", "Require encrypted, integrity-verified transport for model artifacts and data during DR restores.", "Review DR-site configurations for security parity with production on the same cadence as DR tests." ], "failure_signals": [ "DR runbooks tell operators to copy credentials manually from the primary site.", "Restored model artifacts are not integrity-verified before serving.", "The DR environment lags production hardening baselines." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "AI DR plans are typically absent or inadequate in enterprises that apply generic IT DR procedures to AI workloads without AI-specific recovery sequences or integrity validation gates." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "multi-tenant" ], "implementers": [ "IT Operations", "Platform Engineering", "Site Reliability Team", "AI Platform Team" ], "frameworks": [ { "framework": "nist_sp800_34", "requirement_id": "\u00a73.4", "fit": "direct", "rationale": "NIST SP 800-34 \u00a73.4 defines IT contingency plan development requirements including system description, notification procedures, recovery goals, and recovery procedures. AI systems require these elements extended with model-specific recovery sequences and integrity validation gates.", "normative_force": "best-practice", "source_version": "Rev 1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_27031", "requirement_id": "Cl. 10", "fit": "direct", "rationale": "ISO/IEC 27031:2025 Clause 10 requires documented IRBC plans covering how ICT services will be recovered to meet business continuity requirements. An AI-system DR plan is that documented plan for the model serving, retrieval, and data layers.", "normative_force": "certification-standard", "source_version": "2025", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_22301", "requirement_id": "\u00a78.4", "fit": "direct", "rationale": "ISO 22301:2019 \u00a78.4 requires documented recovery procedures for systems within BCP scope. AI system DR plans must address the unique recovery characteristics of model artifacts, inference pipelines, and knowledge bases beyond generic IT restore procedures.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "RC.RP-02", "fit": "direct", "rationale": "NIST CSF 2.0 RC.RP-02 requires recovery plan execution to be monitored and documented. The AI DR plan provides the structured execution framework against which recovery activities are tracked and reported.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 12", "fit": "direct", "rationale": "EU DORA Article 12 requires financial entities to maintain and test ICT disaster recovery plans covering data and system recovery capabilities. AI systems in financial services are in scope and require AI-specific DR documentation covering model artifact restoration.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "aws_reliability", "requirement_id": "REL 13", "fit": "direct", "rationale": "AWS Well-Architected Reliability Pillar REL 13 (Plan for disaster recovery) prescribes recovery objectives, DR strategy selection, and regular DR testing that apply directly to AI workloads on AWS, including model artifacts, vector stores, and serving endpoints.", "normative_force": "best-practice", "source_version": "2023", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Incident management and Production Readiness Review", "fit": "direct", "rationale": "Google SRE incident management practices define the on-call runbooks, severity classification, incident commander authority, and escalation procedures that form the operational execution backbone of any AI System Disaster Recovery Plan. The SRE Production Readiness Review framework requires services to demonstrate documented DR procedures as a launch criterion, directly establishing what an AI DR plan must contain before a system enters production. Adopting Google SRE incident management discipline as the operational layer of an AI DR plan ensures recovery procedures are designed for execution under pressure by teams who have rehearsed them, satisfying the control's requirement for documented and tested recovery sequences.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "BCDR design guidance \u2014 Recovery objectives and failover configuration", "fit": "direct", "rationale": "Microsoft Azure Resiliency BCDR design guidance provides DR planning methodology for Azure AI services including Azure OpenAI Service, Azure AI Search, and Azure Machine Learning, covering failover configuration and recovery objectives relevant to step-by-step AI pipeline recovery procedures. The AI System Disaster Recovery Plan control requires documented recovery sequences with explicit integrity validation gates before restoring production traffic \u2014 requirements that Azure BCDR's service-specific recovery documentation addresses for Azure-hosted AI workloads. Organizations using Azure AI services should derive their DR plan recovery sequences from Azure BCDR's service-specific guidance to ensure documented RTOs reflect actual platform recovery capabilities.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RP-02", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "For every tier-1 and tier-2 AI system, a versioned AI disaster recovery plan document must exist with step-by-step recovery procedures for each AI component, explicit integrity validation gates that must pass before production traffic is restored, named recovery roles with decision authority, and RTO/RPO targets validated against measured recovery capability in at least one DR test within the past 12 months.", "evidence_required": [ "AI DR plan document per production AI system with version history, step-by-step component recovery procedures, and an integrity validation checklist specifying each gate and its pass criteria", "DR test execution record showing measured actual RTO against target, pass/fail status of each integrity validation gate, and a post-test summary signed by the technical lead", "Recovery role assignment matrix showing named individuals for technical lead, business sponsor, and communications owner per AI system with confirmation that each individual has participated in at least one DR drill", "Asset inventory baseline document listing model artifacts, inference endpoints, vector databases, and pipeline configurations used as the DR restore target with integrity checksums", "Post-DR test report or post-incident review within five business days of the most recent DR activation documenting deviation from plan and corrective actions" ], "machine_tests": [ "Restore model artifact from backup to isolated environment \u2192 assert hash of restored artifact matches the registered baseline integrity hash before the traffic restoration gate is cleared", "Execute DR runbook in staging with simulated infrastructure failure \u2192 assert all integrity validation checks pass and the system returns to functional state within the documented RTO window", "Check DR plan metadata \u2192 assert plan_last_reviewed within 365 days, last_dr_test_date within 365 days, and rto_achievement_pct from most recent test is greater than or equal to 90" ], "human_review": [ "Walk through recovery procedure steps with an on-call engineer who did not author the runbook to identify ambiguities, missing prerequisite steps, or references to infrastructure that no longer exists", "Review integrity validation gates for technical adequacy \u2014 confirm they would detect a model weight corruption, vector index poisoning, or partial pipeline restore where downstream dependencies are not yet healthy", "Assess whether recovery role assignments are current and whether each named individual has demonstrated they can execute the runbook under pressure by participating in a timed drill" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Restoring production traffic before all integrity validation gates have passed because of stakeholder pressure to meet the RTO target, reintroducing a potentially unsafe or degraded model state", "Using a DR plan written at initial system deployment without updating it after model version upgrades, infrastructure migrations, or provider changes", "Defining recovery procedures at such a high level (e.g., 'restore the AI system from backup') that an on-call engineer cannot execute them without additional investigation during an incident", "Omitting the dependency order for AI component recovery, causing engineers to attempt to restore inference endpoints before model artifacts or supporting data infrastructure is available", "Setting RTO targets based on regulatory requirements or stakeholder expectations without first measuring actual infrastructure recovery capability, guaranteeing RTO failures in real incidents" ], "update_status": "current", "layer_code": "RP" }, { "id": "RP-03", "layer": "RP", "plane": "lifecycle", "name": "Model Rollback and Previous Version Recovery Planning", "plain": "Organizations must maintain documented plans for rolling back AI models to previously validated versions when the current deployed version exhibits performance degradation, safety failures, or unexpected behavioral changes, with defined triggers, rollback procedures, authority chains, and post-rollback validation criteria.", "threat": { "tags": [ "model-degradation", "silent-regression", "rollback-failure", "version-mismatch" ], "desc": "Deploying an updated model without a tested rollback plan creates an irreversible forward commitment under pressure. Silent regressions \u2014 where a model produces subtly incorrect or unsafe outputs without triggering hard failures \u2014 may go undetected until significant harm has occurred. Without a rollback plan, recovery requires emergency redeployment under incident conditions, increasing the probability of additional errors and extending exposure to the degraded model state." }, "standard": [ { "id": "nist_sp800_34", "section": "\u00a73.4", "title": "Recovery procedures and sequences" }, { "id": "anthropic_rsp", "section": "ASL-3 Deployment Standard", "title": "Provider-side deployment restrictions and de-deployment (binds Anthropic)" }, { "id": "openai_preparedness", "section": "\u00a74", "title": "Safeguarding \u2014 safeguard decisions affecting deployed models (binds OpenAI)" }, { "id": "nist_csf", "section": "RC.RP-01", "title": "Recovery plan documentation" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RP-03 Model Rollback and Previous Version Recovery Planning control.", "reviewed_on": "2026-07-01" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RP-03 Model Rollback and Previous Version Recovery Planning control.", "reviewed_on": "2026-07-02" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RP-03 Model Rollback and Previous Version Recovery Planning control.", "reviewed_on": "2026-07-02" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RP-03 Model Rollback and Previous Version Recovery Planning control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RP-03 Model Rollback and Previous Version Recovery Planning control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Establish an immutable model version registry, define rollback triggers and authority chains, document step-by-step rollback procedures for each production model, and validate rollback capability through scheduled technical rehearsals at least semi-annually.", "steps": [ "Maintain an immutable model version registry with artifact hashes, deployment timestamps, and validation records for all production models and at least the three most recent prior versions.", "Define explicit rollback triggers including performance degradation thresholds, safety evaluation failures, anomalous output rate thresholds, and business stakeholder escalation criteria.", "Document step-by-step rollback procedures for each production model covering artifact retrieval, endpoint reconfiguration, cache invalidation, and downstream dependency notification.", "Assign rollback decision authority with defined escalation paths and maximum time-to-decision targets for each tier of AI system.", "Test rollback procedures for each tier-1 model at least semi-annually with recorded test outcomes including actual rollback duration compared to RTO target." ], "site_reliability": { "summary": "SRE owns the technical rollback runbooks and is responsible for executing rollbacks within defined RTO targets during model regression events.", "actions": [ "Maintain tested rollback runbooks for each production model with step-by-step execution procedures.", "Implement automated rollback capability for models where the rollback RTO is under 15 minutes.", "Monitor rollback trigger metrics in real-time and alert the on-call team when thresholds are breached." ], "failure_signals": [ "Rollback runbooks are missing or untested for any tier-1 model.", "Automated rollback is not implemented where SLA requires sub-15-minute recovery.", "Rollback trigger metrics are not monitored or alerting is not configured." ] }, "security_architect": { "summary": "Security Architect ensures model artifact integrity is protected in the version registry and that rollback procedures cannot be subverted by unauthorized parties.", "actions": [ "Verify that the model version registry enforces immutability and records artifact hashes at ingestion.", "Ensure rollback decision authority requires multi-party authorization for tier-1 production models.", "Review rollback procedures for steps that could introduce supply chain risk during artifact retrieval." ], "failure_signals": [ "Model artifact registry does not enforce immutability or verify hashes at retrieval.", "Rollback can be initiated by a single individual without peer authorization for tier-1 models.", "Artifact retrieval during rollback bypasses integrity verification checks." ] }, "grc_auditor": { "summary": "GRC validates that rollback plans are documented, current, and have been tested within the required interval for all tier-1 production models.", "actions": [ "Request rollback plan documentation for all tier-1 production models and verify completeness against the required elements.", "Review rollback test records to confirm testing within the last 6 months for tier-1 models.", "Verify that rollback decision authority is formally assigned and documented." ], "metrics": [ "Tier-1 models with documented rollback plans: target 100%.", "Rollback test completion rate (semi-annual): target 100%.", "Mean rollback duration in tests vs. RTO target: tracked and within target." ], "failure_signals": [ "Rollback plan absent for any tier-1 production model.", "No rollback test evidence for any tier-1 model in the last 6 months.", "Rollback decision authority is undefined or not updated after personnel changes." ] }, "business_continuity": { "summary": "Model rollback is a continuity mechanism: when a model version produces harmful or degraded output, reverting to a known-good version is often the fastest path to restoring the business function. Ensure rollback plans carry business decision criteria, not just technical steps.", "actions": [ "Define business-impact criteria and decision authority for invoking model rollback in the plan.", "Include model-rollback scenarios (including provider-forced version deprecation) in continuity exercises.", "Document the business functions affected by each model version and their tolerance for rollback-induced capability loss." ], "failure_signals": [ "Rollback decisions stall because no business owner is authorized to accept capability loss.", "Provider version deprecations arrive with no continuity playbook.", "Rollback restores the model but downstream business processes were never re-validated." ] }, "it_operations": { "summary": "Operations owns the rollback machinery: version registries, artifact retention, and the runbooks that swap versions under pressure. Keep prior versions genuinely restorable, not just archived.", "actions": [ "Retain N-2 model versions with configs, prompts, and dependency pins in the artifact registry, restore-tested quarterly.", "Maintain step-by-step rollback runbooks with pre-staged approvals and validation checkpoints.", "Monitor registry integrity and alert when a production system's rollback target is missing or unverifiable." ], "failure_signals": [ "A rollback target exists in the registry but fails to load in staging.", "Rollback requires engineers to reconstruct configuration from memory.", "Version metadata does not record which prompt/config set shipped with each model." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Most enterprises lack formal model rollback plans; rollback is ad hoc and dependent on individual engineer knowledge rather than documented procedures." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "eu-high-risk-ai" ], "implementers": [ "ML Engineering", "Platform Engineering", "Site Reliability Team", "GRC" ], "frameworks": [ { "framework": "nist_sp800_34", "requirement_id": "\u00a73.4", "fit": "direct", "rationale": "NIST SP 800-34 \u00a73.4 covers recovery procedures and sequences which form the structural basis for model rollback planning. The sequencing requirements and decision criteria guidance apply directly to model version management and rollback execution under incident conditions.", "normative_force": "best-practice", "source_version": "Rev 1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "anthropic_rsp", "requirement_id": "ASL-3 Deployment Standard \u2014 deployment restrictions and de-deployment", "fit": "adjacent", "rationale": "The Responsible Scaling Policy (v3.3) binds Anthropic: models that reach capability thresholds without adequate safeguards face deployment restrictions, and previously deployed models can be restricted or de-deployed. Consumer-side rollback and version-pinning plans must anticipate these provider-initiated changes in addition to self-initiated rollbacks.", "normative_force": "best-practice", "source_version": "3.3", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "openai_preparedness", "requirement_id": "\u00a74 Safeguarding \u2014 safeguard decisions affecting deployed models", "fit": "adjacent", "rationale": "The OpenAI Preparedness Framework v2 \u00a74 (Safeguarding) governs OpenAI's own deployment decisions: models reaching High capability require adequate safeguards before deployment continues, which can change or withdraw model versions consumers depend on. Rollback plans should therefore include provider-version deprecation as a trigger scenario. (PF v2 \u00a73 covers measuring capabilities, not rollback.)", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "nist_csf", "requirement_id": "RC.RP-01", "fit": "direct", "rationale": "NIST CSF 2.0 RC.RP-01 requires documented recovery plans for each identified system. Model rollback plans are the recovery plan artifact for AI model deployments and must be maintained as living documents updated after each test.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "REL 8", "fit": "partial", "rationale": "AWS Well-Architected Reliability Pillar REL 08 (Use Version Control) and associated deployment recovery practices provide operational guidance for maintaining versioned model artifacts and implementing blue/green or canary rollback strategies on AWS infrastructure.", "normative_force": "best-practice", "source_version": "2023", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "dora", "requirement_id": "Art. 12", "fit": "partial", "rationale": "EU DORA Article 12 requires backup policies and documented restoration and recovery procedures and methods. Versioned model registries and tested rollback plans apply that requirement to AI model artifacts as a class of ICT assets (Article 13 addresses learning and evolving, not backup).", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "google_sre", "requirement_id": "Production Readiness Review \u2014 Rollback readiness criteria", "fit": "partial", "rationale": "Google SRE Production Readiness Review launch criteria include assessing rollback capability as a deployment gate, establishing that a tested rollback path is a readiness prerequisite rather than an afterthought to model deployment. The Model Rollback and Previous Version Recovery Planning control requires pre-defined rollback triggers, authority chains, and semi-annually tested rollback procedures \u2014 elements that map directly to SRE PRR rollback readiness criteria assessed before each production deployment. Incorporating SRE PRR rollback readiness checks into the AI model deployment lifecycle ensures rollback plans are validated before model updates reach production and that model version registries are maintained in a state that supports rapid rollback.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "Failover configuration \u2014 Blue/green deployment and deployment slot management", "fit": "partial", "rationale": "Microsoft Azure Resiliency documentation covers blue/green deployment patterns and Azure App Service deployment slot management, which are the infrastructure mechanisms enabling model version rollback for AI workloads hosted on Azure OpenAI Service and Azure Machine Learning. The Model Rollback and Previous Version Recovery Planning control requires documented step-by-step rollback procedures for each production model, and Azure's deployment slot and staged rollout documentation provides the technical basis for those procedures on Azure-hosted AI infrastructure. Organizations running AI workloads on Azure should align their model rollback procedures with Azure's deployment pattern guidance to ensure rollback steps are consistent with platform-level capabilities and can meet defined RTO targets.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RP-03", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "For every production AI model, a documented rollback plan must exist specifying at least one previously validated version registered with an integrity hash and available for immediate rollback, quantitative trigger criteria that define when rollback evaluation must begin, step-by-step rollback procedures, the authority required to approve rollback execution, and post-rollback validation criteria that confirm the prior version is performing within acceptable bounds.", "evidence_required": [ "Model rollback plan document per production model specifying rollback trigger thresholds (error rate, output quality degradation metric, safety failure count), approved rollback target version with integrity hash, step-by-step rollback procedure, and named rollback authority", "Model version registry showing all production models with at least one prior validated version, integrity hash per version, validation test results that qualified it for rollback use, and storage location", "Rollback drill execution record from within the last 12 months showing rehearsed execution of the rollback procedure in staging, measured rollback time, and post-rollback validation test results", "Monitoring alert configuration showing trigger thresholds that fire rollback evaluation (e.g., error rate > X%, safety refusal rate change > Y%, output quality score below Z)", "Rollback authority approval log or decision matrix showing who may approve rollback under what conditions and at what urgency level" ], "machine_tests": [ "Deploy a known-degraded model version in staging and inject trigger-threshold conditions \u2192 assert rollback alert fires within 5 minutes and executing the rollback procedure returns the system to the registered prior version with hash-verified integrity", "Query model version registry \u2192 assert every production model has at least one prior validated version registered with a non-null integrity hash and validation test result", "Trigger rollback monitoring alert threshold in observability platform \u2192 assert alert reaches on-call channel within the documented notification SLA and references the rollback runbook" ], "human_review": [ "Review rollback trigger definitions for coverage of silent regression scenarios \u2014 confirm triggers include soft signals such as behavioral drift or output quality degradation, not only hard failures like error rate spikes", "Assess whether the rollback authority chain is appropriate for the time-sensitivity of safety-critical rollback scenarios, where a multi-tier approval process could delay action during active harm", "Verify that post-rollback validation criteria are sufficient to confirm the prior version performs correctly in the current production environment, including any prompt, tool, or data context changes since that version was last deployed" ], "blocking_effect": "blocks-deployment", "normative_status": "best-practice", "anti_patterns": [ "Treating model rollback as an emergency improvisation with no pre-documented procedure, forcing engineers to determine rollback steps under incident pressure without a tested sequence", "Retaining only one previous model version for rollback, leaving no recovery path if the immediate prior version is itself degraded or was the cause of the original failure", "Defining rollback triggers exclusively for hard failures (inference error rate spikes, system crashes) without including behavioral drift indicators or output quality metrics that detect silent regressions", "Requiring a lengthy multi-stakeholder approval chain for rollback execution that introduces dangerous delays when the active model is producing safety failures", "Completing rollback without executing post-rollback validation before restoring full production traffic, reintroducing uncertainty about whether the prior version is performing correctly in the current context" ], "update_status": "current", "layer_code": "RP" }, { "id": "RP-04", "layer": "RP", "plane": "data", "name": "RAG Index and Knowledge Base Recovery Plan", "plain": "Organizations deploying retrieval-augmented generation systems must maintain documented recovery procedures for vector databases, retrieval indexes, and knowledge bases, including backup schedules, index rebuild procedures, freshness validation, and RTO/RPO targets specific to knowledge infrastructure.", "threat": { "tags": [ "index-corruption", "knowledge-staleness", "retrieval-failure", "data-integrity-loss" ], "desc": "Vector databases and retrieval indexes are specialized infrastructure with recovery characteristics distinct from relational databases. Index corruption can cause RAG systems to return semantically incorrect results without triggering hard failures, making degraded operation difficult to detect at the application layer. Without a dedicated recovery plan, restoring a RAG knowledge base requires manual reconstruction that may take days and produces unvalidated output quality, silently degrading AI responses." }, "standard": [ { "id": "iso_22301", "section": "\u00a78.4", "title": "Recovery procedures" }, { "id": "nist_sp800_34", "section": "\u00a73.4", "title": "Backup and recovery strategies" }, { "id": "cis_controls_v8", "section": "CIS Control 11", "title": "Data recovery" }, { "id": "nist_csf", "section": "RC.RP-02", "title": "Recovery plan execution" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RP-04 RAG Index and Knowledge Base Recovery Plan control.", "reviewed_on": "2026-07-01" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RP-04 RAG Index and Knowledge Base Recovery Plan control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RP-04 RAG Index and Knowledge Base Recovery Plan control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Implement scheduled snapshots of all vector databases and knowledge base source documents, document index rebuild procedures with freshness validation steps, define RTO/RPO targets specific to RAG infrastructure accounting for embedding regeneration time, and test recovery procedures at least annually.", "steps": [ "Implement automated snapshot schedules for all vector databases and retrieval indexes with off-site backup storage and verified integrity hashing of each snapshot.", "Document index rebuild procedures covering source document retrieval, embedding regeneration, index construction, and freshness validation benchmarks against a known-good baseline.", "Define RTO and RPO targets for RAG knowledge infrastructure, accounting for embedding regeneration time at current production data scale.", "Establish post-recovery validation criteria including retrieval accuracy benchmarks, freshness timestamps, and document coverage metrics that must pass before restoring production RAG traffic.", "Test full knowledge base recovery annually and partial incremental recovery semi-annually, recording actual rebuild times and validation outcomes against defined criteria." ], "site_reliability": { "summary": "SRE owns the RAG infrastructure backup schedules and recovery runbooks and is responsible for meeting RTO targets during knowledge base recovery events.", "actions": [ "Implement and monitor automated snapshot jobs for all vector databases with alerting on backup failures.", "Maintain recovery runbooks for full and partial index rebuild procedures including embedding regeneration steps.", "Benchmark embedding regeneration throughput to validate RTO feasibility at current production data scale." ], "failure_signals": [ "Snapshot job failures are not alerted or not resolved within SLA.", "Recovery runbooks do not include embedding regeneration procedures.", "Embedding regeneration throughput has not been benchmarked at current production scale." ] }, "it_operations": { "summary": "IT Operations manages backup storage infrastructure and ensures off-site copies of RAG knowledge infrastructure are available and recoverable on demand.", "actions": [ "Configure and monitor off-site backup storage for vector database snapshots and knowledge base source documents.", "Perform quarterly backup restoration tests to verify recoverability of RAG knowledge infrastructure.", "Maintain storage capacity planning to ensure the backup retention policy can be met without compromise." ], "failure_signals": [ "Off-site backup storage is not configured for vector database snapshots.", "Backup restoration tests have not been performed in over 12 months.", "Backup retention period does not meet the documented RPO target." ] }, "grc_auditor": { "summary": "GRC validates that RAG recovery plans exist, cover all production knowledge bases, and have been tested and updated within required intervals.", "actions": [ "Request RAG recovery plan documentation and verify coverage of all production vector databases.", "Review backup test records and confirm restoration was validated within the last 12 months.", "Verify post-recovery validation criteria are defined and include quantified retrieval accuracy benchmarks." ], "metrics": [ "Production vector databases with documented recovery plans: target 100%.", "Backup restoration test completion (annual): target 100%.", "Post-recovery retrieval accuracy vs. baseline: target \u226595%." ], "failure_signals": [ "Recovery plan missing for any production vector database.", "No restoration test evidence within the last 12 months.", "Post-recovery validation criteria are absent or not quantified." ] }, "business_continuity": { "summary": "RAG knowledge bases power business-critical answers; their loss quietly corrupts decisions. Treat index and source-corpus recovery as a continuity requirement with defined tolerable staleness.", "actions": [ "Set business-approved RPO/staleness limits for each knowledge base based on decision impact.", "Include knowledge-base loss and rebuild scenarios in continuity exercises for AI-assisted processes.", "Document interim operating procedures for when retrieval quality is degraded during rebuilds." ], "failure_signals": [ "Business users unknowingly act on answers from a stale or partially restored index.", "No one has defined how long the business can tolerate a rebuilt-from-source index lag.", "Continuity plans cover the model but ignore the retrieval layer entirely." ] }, "security_architect": { "summary": "Index recovery is a data-integrity and access-control problem: a poisoned or permission-stripped restore is worse than an outage. Bake integrity verification and ACL restoration into the recovery plan.", "actions": [ "Require checksums/signatures on index snapshots and source corpora, verified at restore time.", "Restore and re-validate document-level access controls and tenant isolation before reopening retrieval.", "Scan rebuilt indexes for unauthorized or unexpected content changes against the source-of-record." ], "failure_signals": [ "Restored indexes serve documents to users who lost access rights since the snapshot.", "Snapshot integrity is assumed, never verified.", "A rebuild silently ingests content that was quarantined before the incident." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "RAG knowledge base recovery is an emerging gap \u2014 most enterprises apply generic database backup policies without accounting for embedding regeneration complexity or retrieval accuracy validation." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "multi-tenant" ], "implementers": [ "Data Engineering", "IT Operations", "ML Engineering", "Platform Engineering" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a78.4", "fit": "direct", "rationale": "ISO 22301:2019 \u00a78.4 requires documented recovery procedures for all systems within BCP scope. RAG knowledge bases are business-critical ICT assets requiring dedicated recovery procedures that address their unique rebuild and validation characteristics.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.4", "fit": "direct", "rationale": "NIST SP 800-34 \u00a73.4 requires backup and recovery strategies for all system components. Vector databases require backup strategies that account for snapshot consistency during active ingestion and embedding index rebuild time at recovery scale.", "normative_force": "best-practice", "source_version": "Rev 1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cis_controls_v8", "requirement_id": "CIS Control 11", "fit": "direct", "rationale": "CIS Controls v8 Control 11 (Data Recovery) requires automated backup of enterprise data and regular testing of recovery procedures. Vector databases and knowledge base source documents are enterprise data assets requiring this protection with AI-specific recovery validation criteria.", "normative_force": "industry-framework", "source_version": "v8", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "RC.RP-02", "fit": "direct", "rationale": "NIST CSF 2.0 RC.RP-02 requires recovery plan execution to be monitored and documented. RAG knowledge base recovery plans must include monitoring of rebuild progress and validation of retrieval accuracy after recovery completes.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "REL 9", "fit": "direct", "rationale": "AWS Well-Architected Reliability Pillar REL 09 (Back Up Data) prescribes backup policies for managed data stores. Amazon OpenSearch, Pinecone on AWS, and other vector database services require backup configurations aligned with this pillar guidance.", "normative_force": "best-practice", "source_version": "2023", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "Azure Well-Architected RE:09", "fit": "partial", "rationale": "Microsoft Azure Well-Architected Reliability recommendation RE:09 (disaster recovery) covers backup and recovery planning for Azure data services, including Azure AI Search and Azure Cosmos DB vector stores. Organizations running RAG infrastructure on Azure should align knowledge-base recovery plans with this guidance.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Production Readiness Review \u2014 Data infrastructure recovery readiness", "fit": "partial", "rationale": "Google SRE Production Readiness Review practices require that backup and restoration procedures for production data stores are documented and tested as a launch criterion, directly applicable to vector databases and retrieval indexes that underpin RAG systems. The RAG Index and Knowledge Base Recovery Plan control requires backup schedules, index rebuild procedures with freshness validation, and post-recovery accuracy benchmarks \u2014 elements that correspond to the SRE PRR data infrastructure readiness assessment requirements. Applying SRE PRR discipline to RAG knowledge infrastructure ensures recovery procedures are validated before the vector database is relied upon in production, and that the PRR gate is re-evaluated after material changes to corpus scale or embedding model selection.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RP-04", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every production RAG system must have a documented knowledge base recovery plan specifying vector database backup schedules aligned to the system's RPO, step-by-step index rebuild procedures with measured time estimates, freshness validation queries that confirm restored content is semantically current and not corrupted, and RTO/RPO targets that explicitly account for index rebuild duration, validated in at least one recovery drill within the past 12 months.", "evidence_required": [ "RAG knowledge base recovery plan document per production deployment specifying backup schedule, index rebuild procedure steps, freshness validation query set, RTO/RPO targets inclusive of rebuild time, and responsible roles", "Vector database backup execution logs covering the last 90 days showing each scheduled backup completed with a record of embedding count, backup size, and checksum", "Index rebuild test results from a drill within the last 12 months showing measured rebuild time against RTO target, freshness validation query results, and pass/fail determination per validation check", "Knowledge base content integrity report showing source-to-index mapping is current, no documents are indexed beyond their freshness window, and no corrupted or null embedding vectors exist", "RPO validation record confirming the gap between last successful backup and recovery point is within the documented RPO for the system" ], "machine_tests": [ "Restore vector database from most recent backup to isolated environment \u2192 assert embedding count matches pre-backup count within 0.1% and a set of representative freshness validation queries return expected documents with correct metadata and embedding dimensions", "Trigger full index rebuild from backup in staging \u2192 assert rebuild completes within the documented RTO window and freshness validation pass rate is 100% on the standard validation query set", "Query backup execution log for the last 30 days \u2192 assert no scheduled backup was skipped and the most recent backup timestamp is no older than the documented RPO interval" ], "human_review": [ "Review freshness validation procedures to confirm they would detect semantically stale content \u2014 outdated source documents indexed without refresh \u2014 as well as structurally missing entries or dimension-mismatched embeddings", "Assess whether RTO targets account for the full end-to-end rebuild time including source document re-ingestion, embedding generation, and index population, not only infrastructure provisioning", "Evaluate the recovery plan's coverage of a corrupted-backup scenario, confirming that the plan addresses how far back the organization would need to go in the backup history to find a clean restore point" ], "blocking_effect": "requires-review", "normative_status": "certification-standard", "anti_patterns": [ "Setting backup schedules based on generic data retention policy without accounting for the rate of knowledge base content updates in the specific RAG system, causing RPO targets to be unachievable", "Treating vector database recovery as equivalent to relational database restore without including embedding freshness validation procedures that confirm semantic accuracy of the restored index", "Omitting index rebuild time from RTO calculations, causing the organization to systematically underestimate actual recovery duration and miss RTO targets during real incidents", "Storing vector database backups in the same availability zone as the primary index, creating a co-located failure scenario where the backup and primary are lost in the same event", "Declaring knowledge base recovery complete based on database connectivity or row count without executing freshness validation queries that confirm semantically correct retrieval results" ], "update_status": "current", "layer_code": "RP" }, { "id": "RP-05", "layer": "RP", "plane": "control", "name": "Recovery Priority Classification for AI Systems", "plain": "AI systems must be classified into recovery priority tiers based on business criticality, regulatory obligation, and downstream dependency, with tier assignments used to sequence recovery resources and efforts during enterprise-wide disaster events where simultaneous AI system failures exceed available recovery capacity.", "threat": { "tags": [ "unordered-recovery", "resource-contention", "critical-system-delay", "triage-failure" ], "desc": "During an enterprise-wide disaster, simultaneous recovery attempts across all AI systems create resource contention that delays recovery of the most critical systems. Without a formal priority classification, recovery sequencing defaults to whoever requests resources first, routinely resulting in non-critical AI systems consuming recovery capacity ahead of AI systems that block revenue-generating or safety-critical operations. Absent classification, triage decisions are made under pressure by individuals without visibility into the full business impact picture." }, "standard": [ { "id": "iso_22301", "section": "\u00a78.2.2", "title": "Business impact analysis \u2014 prioritization" }, { "id": "nist_sp800_34", "section": "\u00a73.2", "title": "Business impact analysis \u2014 criticality and recovery priorities" }, { "id": "cobit_dss04", "section": "DSS04.02", "title": "Manage continuity strategies" }, { "id": "nist_csf", "section": "ID.AM-05", "title": "Assets are prioritized based on classification, criticality, resources, and impact on the mission" } ], "sources": [ { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RP-05 Recovery Priority Classification for AI Systems control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RP-05 Recovery Priority Classification for AI Systems control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RP-05 Recovery Priority Classification for AI Systems control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RP-05 Recovery Priority Classification for AI Systems control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Define a four-tier AI system recovery classification (Critical, High, Medium, Low), assign each production AI system to a tier based on BIA outputs, publish a recovery sequencing policy that allocates resources by tier, and review tier assignments annually and after material AI portfolio changes.", "steps": [ "Define recovery priority tiers with explicit, measurable criteria including revenue impact per hour of outage, regulatory obligation, safety criticality, and downstream system dependency count.", "Conduct a classification workshop with AI system owners and business stakeholders to assign each production AI system to a tier using the defined criteria.", "Publish a recovery sequencing policy specifying resource allocation rules, escalation triggers, and deconfliction procedures when multiple Critical-tier systems fail simultaneously.", "Integrate tier assignments into DR and BCP documentation so recovery teams can identify priority systems during an event without requiring business stakeholder consultation under pressure.", "Review and update tier assignments annually and after any material change to AI system business criticality, regulatory scope, or dependency architecture." ], "business_continuity": { "summary": "Business Continuity Manager owns the recovery priority classification process and ensures tier assignments reflect current business impact analysis.", "actions": [ "Lead annual tier classification reviews with AI system owners and business unit representatives.", "Maintain the published recovery sequencing policy and ensure it is distributed to all recovery teams.", "Validate that BIA outputs are used as the primary input to tier assignment decisions." ], "failure_signals": [ "Recovery priority tiers are not defined or not published.", "Tier assignments have not been reviewed in the last 12 months.", "BIA outputs are not linked to tier assignment decisions." ] }, "it_operations": { "summary": "IT Operations uses tier assignments to sequence infrastructure recovery and allocate resources during disaster events without requiring real-time business stakeholder input.", "actions": [ "Maintain a current tier assignment register accessible to all recovery teams without requiring business stakeholder approval during an event.", "Implement resource allocation procedures that reserve infrastructure capacity for Critical and High-tier AI system recovery before initiating lower-tier recovery.", "Report actual recovery sequencing adherence in post-event reviews against the published sequencing policy." ], "failure_signals": [ "Tier assignment register is not accessible to recovery teams during an event or is out of date.", "Lower-tier systems have been recovered ahead of Critical-tier systems in a DR event.", "Post-event reviews do not assess sequencing adherence against the published policy." ] }, "grc_auditor": { "summary": "GRC validates that all production AI systems have documented tier assignments, that the classification methodology is defensible, and that assignments are reviewed on schedule.", "actions": [ "Request the tier assignment register and verify all production AI systems are classified.", "Review the tier classification methodology for alignment with BIA outputs and regulatory obligations.", "Confirm tier assignments were reviewed within the last 12 months with documented rationale." ], "metrics": [ "Production AI systems with documented tier assignments: target 100%.", "Tier assignment review currency: reviewed within last 12 months.", "Critical-tier systems with RTO targets shorter than non-critical systems: target 100%." ], "failure_signals": [ "Any production AI system lacks a documented tier assignment.", "Tier classification methodology is undocumented.", "Tier assignments not reviewed in the last annual cycle." ] }, "security_architect": { "summary": "Recovery priority tiers drive where security effort concentrates during incidents: top-tier AI systems need pre-hardened recovery paths and priority credential restoration. Align security recovery sequencing to the tier model.", "actions": [ "Map credential, key, and certificate restoration order to the recovery tier classification.", "Pre-harden and pre-authorize recovery paths for Tier 1 AI systems.", "Review tier assignments for systems whose compromise would have security blast radius beyond availability." ], "failure_signals": [ "Security restoration order contradicts the published recovery sequence.", "Tier 1 systems depend on secrets that only one engineer can restore.", "Security-critical systems sit in low tiers because only availability impact was scored." ] }, "site_reliability": { "summary": "Tier classifications must translate into engineering reality: capacity reservations, failover automation, and alert priorities should all derive from the tier model.", "actions": [ "Encode tiers as machine-readable service metadata consumed by alert routing and capacity planning.", "Verify through game days that Tier 1 systems actually recover before lower tiers.", "Reconcile tier assignments against measured criticality signals (traffic, dependency fan-in) each quarter." ], "failure_signals": [ "Tier labels exist in documents but not in the service catalog.", "Recovery exercises restore convenient systems first regardless of tier.", "Alerting treats Tier 1 and Tier 3 AI systems identically." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "AI systems are rarely tiered separately from general IT systems; recovery priority for AI workloads is typically improvised during events." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "federated-enterprise", "eu-high-risk-ai" ], "implementers": [ "Business Continuity Team", "IT Operations", "GRC", "AI Platform Team" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a78.2.2", "fit": "direct", "rationale": "ISO 22301:2019 \u00a78.2.2 requires business impact analysis to determine prioritization of activities and recovery time objectives. AI system tier classification operationalizes this BIA output into an actionable recovery sequencing policy with explicit resource allocation rules.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.2", "fit": "direct", "rationale": "NIST SP 800-34 Rev 1 \u00a73.2 requires a business impact analysis that identifies critical business processes, their supporting systems, and the impact of outages over time \u2014 the analytic basis for classifying AI systems into recovery priority tiers and allocating recovery resources (there is no \u00a72.3 in the document's process chapters).", "normative_force": "best-practice", "source_version": "Rev 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.02", "fit": "direct", "rationale": "COBIT 2019 DSS04.02 requires organizations to identify and select continuity strategies based on risk and cost. Recovery priority classification is the mechanism by which continuity strategy choices are translated into operational recovery sequencing rules.", "normative_force": "best-practice", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 11(5)", "fit": "direct", "rationale": "EU DORA Article 11(5) requires financial entities to determine ICT systems and services to prioritize in recovery plans. AI system tier classification directly satisfies this requirement when applied to AI workloads in financial services entities.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_csf", "requirement_id": "ID.AM-05", "fit": "direct", "rationale": "NIST CSF 2.0 ID.AM-05 requires assets to be prioritized based on classification, criticality, resources, and impact on the mission. AI recovery tier classification is that prioritization applied to AI systems, and it feeds recovery sequencing and resource allocation during incidents (RC.RP-03 concerns backup integrity verification, not tiering).", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_800_160_v2", "requirement_id": "Appendix D \u2014 Focus on common critical assets", "fit": "partial", "rationale": "NIST SP 800-160 Vol 2 Rev 1's strategic design principle 'Focus on common critical assets' (Appendix D) directs resilience investment toward the assets most critical to mission and business functions. Recovery priority classification identifies exactly which AI systems those critical assets are.", "normative_force": "best-practice", "source_version": "Rev 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "openai_preparedness", "requirement_id": "Capability thresholds \u2014 High and Critical capability levels", "fit": "adjacent", "rationale": "The OpenAI Preparedness Framework v2 classifies frontier capabilities into High and Critical threshold tiers that gate OpenAI's own deployment decisions. It is a capability-risk tiering discipline analogous to \u2014 but distinct from \u2014 the recovery-priority tiering of deployed AI systems this control requires.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "aws_reliability", "requirement_id": "REL 10", "fit": "partial", "rationale": "AWS Well-Architected Reliability Pillar REL 10 (Use fault isolation to protect your workload) requires fault-isolation boundaries that presuppose knowing each workload's criticality and blast radius. Tier classification supplies that criticality input for AI workloads.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Error budget policies \u2014 Service tier definition by SLO", "fit": "partial", "rationale": "Google SRE error budget policies are defined per service tier, with SLOs establishing quantitative availability targets that differentiate critical services from lower-priority workloads \u2014 a discipline that directly informs the Recovery Priority Classification for AI Systems control's requirement for measurable tier assignment criteria. The control requires tier definitions with explicit, measurable criteria; SRE-derived SLO targets provide exactly this quantitative basis for tier differentiation, where higher-SLO services warrant higher recovery priority and shorter RTO targets. Organizations applying SRE practices to AI workloads should use existing SLO tier assignments as the primary input to recovery priority classification rather than creating a parallel classification system.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "Recovery objectives \u2014 Business continuity tiering by service criticality", "fit": "partial", "rationale": "Microsoft Azure Resiliency documentation provides guidance on setting tiered recovery objectives by service criticality, with Azure BCDR's recommended approach of differentiating RTO and RPO targets by business impact tier directly applicable to the Recovery Priority Classification for AI Systems control. The control requires tier assignments to drive differential recovery resource allocation and RTO targets, which Azure BCDR's business continuity tiering guidance operationalizes for Azure-hosted workloads. Organizations deploying AI workloads on Azure should align their AI system recovery tier assignments with Azure BCDR's recommended recovery objective tiers to ensure platform-level recovery capabilities are consistent with declared tier requirements.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RP-05", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every production AI system must be assigned a formally documented recovery priority tier (tier 1 through tier N) based on quantified business impact analysis, regulatory obligation mapping, and downstream dependency assessment, with tier assignments approved by named business stakeholders and explicitly used to sequence recovery resources in the enterprise DR plan when concurrent AI system failures exceed available recovery capacity.", "evidence_required": [ "Recovery priority tier register listing every production AI system with tier assignment, business impact justification citing downtime cost, regulatory obligation notes, downstream dependency list, and date of last tier review", "Business impact analysis output per AI system showing quantified downtime cost per hour, maximum tolerable outage period, and dependency chain analysis supporting the tier assignment", "Stakeholder approval records for each tier assignment showing business owner, compliance owner, and date of approval", "Enterprise DR sequencing plan or runbook section showing how tier assignments drive recovery resource allocation during concurrent AI system failures", "Tier review records confirming assignments were reviewed within the last 12 months or within 30 days of any significant AI system change" ], "machine_tests": [ "Query AI system inventory \u2192 assert every production AI system has a recovery_priority_tier value in the set {1, 2, 3} and a non-null business_impact_justification and stakeholder_approved_by field in the tier register", "Simulate concurrent tier-1, tier-2, and tier-3 failures in tabletop tool \u2192 assert DR sequencing plan assigns recovery resources to all tier-1 systems before any tier-2 system receives resources", "Check tier review date metadata \u2192 assert last_tier_review_date for all tier-1 systems is within 365 days and within 30 days for any system added or substantially changed in the last quarter" ], "human_review": [ "Review tier assignments for AI systems with known regulatory obligations to confirm that compliance-driven recovery priority (e.g., a regulatory-reporting AI system) is captured in the tier justification and not subordinated to lower-priority internal systems", "Assess downstream dependency mapping completeness \u2014 verify that an AI system whose output feeds another AI system or business process is tiered appropriately for the downstream criticality, not only its direct user impact", "Evaluate whether tier assignments reflect current business priorities, particularly for AI systems that have expanded in scope, gained new downstream consumers, or changed regulatory status since the last review" ], "blocking_effect": "requires-review", "normative_status": "certification-standard", "anti_patterns": [ "Assigning all AI systems to tier 1 to guarantee resource priority, defeating the purpose of tiering and creating resource contention that delays recovery of genuinely critical systems during a concurrent failure event", "Basing tier assignments solely on technical complexity or system age rather than quantified business criticality, regulatory obligation, and downstream dependency impact", "Failing to update tier assignments when downstream AI consumers are added or removed, causing the tier register to misrepresent the actual blast radius of an AI system failure", "Maintaining tier assignments in a static spreadsheet disconnected from the DR execution plan, so recovery teams fall back to negotiating priority verbally during an incident rather than following a documented sequence", "Completing tier assignment without regulatory obligation input, causing compliance-critical AI systems that support reporting or risk functions to be under-prioritized relative to internal productivity tools" ], "update_status": "current", "layer_code": "RP" }, { "id": "RP-06", "layer": "RP", "plane": "control", "name": "Third-Party AI Provider Outage Response Plan", "plain": "Organizations that depend on external AI model API providers must maintain documented contingency plans for provider outages, including fallback routing to alternative providers, graceful degradation procedures for AI-dependent business functions, and defined criteria for activating each response tier.", "threat": { "tags": [ "provider-dependency", "api-outage", "single-point-of-failure", "vendor-concentration-risk" ], "desc": "Enterprise AI systems built on third-party model APIs inherit the full availability risk of those providers. A major provider outage (Anthropic, OpenAI, Google, AWS Bedrock) can instantly disable all AI-dependent business functions with no warning. Without a pre-documented contingency plan, organizations default to waiting for provider restoration, which may exceed their business continuity tolerance. Concentration of critical AI workflows on a single provider creates a systemic single point of failure with no internal mitigation path." }, "standard": [ { "id": "iso_22301", "section": "\u00a78.3.4", "title": "Resource requirements" }, { "id": "dora", "section": "Art. 28", "title": "Third-party ICT risk management" }, { "id": "nist_sp800_34", "section": "\u00a73.5", "title": "Alternate site and vendor considerations" } ], "sources": [ { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RP-06 Third-Party AI Provider Outage Response Plan control.", "reviewed_on": "2026-07-02" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RP-06 Third-Party AI Provider Outage Response Plan control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RP-06 Third-Party AI Provider Outage Response Plan control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RP-06 Third-Party AI Provider Outage Response Plan control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RP-06 Third-Party AI Provider Outage Response Plan control.", "reviewed_on": "2026-07-01" }, { "id": "cloudflare_resilience_2024", "title": "Cloudflare DDoS Protection", "authority": "Cloudflare, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.cloudflare.com/ddos/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "cloudflare_resilience_2024", "relationship": "informative_reference", "rationale": "Establishes Cloudflare DDoS Protection requirements informing the apeiris://resilience/controls/RP-06 Third-Party AI Provider Outage Response Plan control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Map all AI API provider dependencies, define provider SLA breach thresholds that trigger contingency activation, document fallback routing rules for each provider outage scenario, implement automated provider health monitoring, and test fallback procedures at least semi-annually for all tier-1 AI workflows.", "steps": [ "Inventory all third-party AI provider dependencies, documenting which business functions depend on each provider and the maximum tolerable outage duration per function.", "Define provider outage trigger criteria for each response tier: degraded-mode activation, fallback provider routing, and graceful degradation to non-AI workflows.", "Document fallback procedures for each major provider outage scenario, including prompt compatibility considerations when switching between model providers.", "Implement provider health monitoring with automated alerting and, where feasible, automated failover routing to alternate providers for tier-1 AI workflows.", "Test fallback routing to at least one alternate provider for each tier-1 AI workflow semi-annually and document results including latency changes and output quality delta." ], "site_reliability": { "summary": "SRE implements provider health monitoring, automated failover routing, and maintains fallback runbooks for each major provider outage scenario.", "actions": [ "Deploy provider health monitoring dashboards with alerting to the on-call rotation when provider error rates exceed defined thresholds.", "Implement and test automated failover routing for tier-1 AI workflows with documented fallback provider mappings.", "Maintain prompt compatibility documentation for cross-provider failover scenarios including known behavioral differences." ], "failure_signals": [ "No provider health monitoring exists for any critical AI API provider.", "Automated failover has not been tested for any tier-1 AI workflow in the last 6 months.", "Prompt compatibility between primary and fallback providers has not been validated." ] }, "it_operations": { "summary": "IT Operations manages activation of provider contingency procedures and coordinates with the business continuity team during provider outage events.", "actions": [ "Maintain current contact information and escalation procedures for each major AI provider's enterprise support team.", "Execute provider outage response procedures and communicate status updates to business stakeholders on defined intervals.", "Document provider outage events with timeline, business impact, and recovery actions for post-event review." ], "failure_signals": [ "Provider support contact information is outdated or unavailable to operations staff.", "No outage response procedure is documented for any of the top three AI API providers.", "Provider outage events are not documented with timelines for post-event learning." ] }, "business_continuity": { "summary": "Business Continuity Manager ensures provider outage contingency plans are integrated into the enterprise BCP and that graceful degradation procedures are approved by business stakeholders in advance.", "actions": [ "Facilitate agreement with business stakeholders on acceptable degraded-mode procedures for each AI-dependent business function.", "Include provider outage scenarios in enterprise BCP review cycles and tabletop exercises.", "Review provider concentration risk annually and escalate to executive leadership when critical functions are single-provider dependent." ], "failure_signals": [ "Graceful degradation procedures for AI-dependent functions are not pre-approved by business stakeholders.", "Provider concentration risk has not been reviewed in the last 12 months.", "Provider outage scenarios are not represented in any BCP exercise in the last 12 months." ] }, "grc_auditor": { "summary": "GRC validates that provider outage contingency plans exist for all critical AI provider dependencies and that third-party risk management requirements are satisfied.", "actions": [ "Request provider dependency inventory and contingency plan documentation for all AI API providers.", "Review provider contracts for SLA commitments and verify internal contingency trigger criteria are aligned with contractual terms.", "Verify DORA or equivalent third-party ICT risk requirements are met for AI provider dependencies in regulated entities." ], "metrics": [ "Critical AI provider dependencies with documented contingency plans: target 100%.", "Fallback test completion (semi-annual) for tier-1 AI workflows: target 100%.", "Provider concentration risk review currency: reviewed annually." ], "failure_signals": [ "Any critical AI provider dependency lacks a documented contingency plan.", "Fallback tests have not been performed for tier-1 workflows in the last 6 months.", "Provider contracts do not include SLA commitments for AI API availability in regulated entities." ] }, "security_architect": { "summary": "Provider failover changes your trust boundary: an alternate model API means different data-handling terms, auth models, and content controls. Pre-assess the security of every fallback provider before the outage forces the decision.", "actions": [ "Complete security and data-protection assessments of alternate providers before listing them in the outage plan.", "Pre-provision scoped credentials and egress controls for fallback endpoints; forbid ad-hoc key creation mid-incident.", "Verify guardrails and content filters are re-applied when traffic shifts to an alternate provider." ], "failure_signals": [ "Failover sends regulated data to a provider with no signed data-processing agreement.", "Incident responders mint new API keys with broad scopes under pressure.", "Safety filters configured for the primary provider silently vanish on the fallback path." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most enterprises have no documented provider outage contingency plans; operational resilience depends entirely on provider uptime with no internal mitigation." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "eu-high-risk-ai" ], "implementers": [ "IT Operations", "Platform Engineering", "Business Continuity Team", "GRC" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a78.3.4", "fit": "direct", "rationale": "ISO 22301:2019 \u00a78.3.4 requires organizations to determine the resource requirements \u2014 including partners and suppliers \u2014 needed to implement chosen business continuity solutions. Third-party AI model providers are exactly such supplier dependencies, and outage response plans document the alternative resources this clause requires.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 28", "fit": "direct", "rationale": "EU DORA Article 28 imposes binding requirements for managing third-party ICT risk including cloud service providers and AI API vendors. Financial entities using third-party model APIs must maintain contingency plans for provider outages as a regulatory obligation.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "anthropic_rsp", "requirement_id": "Capability thresholds and ASL-3 standards \u2014 provider-initiated restrictions", "fit": "adjacent", "rationale": "The Responsible Scaling Policy (v3.3) binds Anthropic: capability-threshold determinations and ASL-3 safeguard requirements can restrict or condition model availability. Provider outage response plans should treat safety-driven provider restrictions as an outage class alongside infrastructure failures.", "normative_force": "best-practice", "source_version": "3.3", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "openai_preparedness", "requirement_id": "\u00a74 Safeguarding \u2014 safeguard-driven availability changes", "fit": "adjacent", "rationale": "The OpenAI Preparedness Framework v2 \u00a74 (Safeguarding) governs whether and how OpenAI continues deploying models that reach capability thresholds; safeguard decisions can curtail API availability. Enterprise outage plans for third-party model providers should include this safety-driven scenario.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.5", "fit": "direct", "rationale": "NIST SP 800-34 \u00a73.5 addresses alternate site and vendor considerations in contingency planning. AI provider failover planning is a direct application of this guidance to modern API-based AI dependency chains.", "normative_force": "best-practice", "source_version": "Rev 1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "REL 5", "fit": "partial", "rationale": "AWS Well-Architected Reliability Pillar REL 5 (Design interactions in a distributed system to mitigate or withstand failures) requires graceful degradation, throttling, retry limits, and fail-fast behavior when dependencies fail. Provider-outage failover routing implements these interaction patterns for external AI APIs.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Cascading failure prevention", "fit": "partial", "rationale": "Google SRE cascading failure prevention practices address external service dependencies as a primary source of reliability risk, requiring load shedding, fallback paths, and timeout/retry discipline to prevent provider unavailability from propagating into broader service failures. The Third-Party AI Provider Outage Response Plan control requires tier-based activation criteria and graceful degradation procedures for AI-dependent business functions \u2014 a direct operational implementation of SRE cascading failure prevention for AI provider dependency chains. Applying Google SRE load shedding and fallback routing practices to AI provider dependencies reduces the blast radius of provider outage events and ensures that graceful degradation procedures are designed to engage before cascading failure thresholds are reached.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "BCDR design guidance \u2014 Geo-redundancy and failover configuration", "fit": "partial", "rationale": "Microsoft Azure Resiliency BCDR design guidance covers geo-redundancy and failover configuration patterns applicable to routing AI inference requests away from failed providers to alternative regional endpoints or alternative provider services, relevant to organizations using Azure API Management or Azure Front Door to mediate AI provider API calls. The Third-Party AI Provider Outage Response Plan control requires documented fallback procedures including prompt compatibility considerations when switching between model providers \u2014 design concerns addressed by Azure BCDR's multi-region failover guidance. Organizations using Azure to front their AI provider API traffic should apply Azure BCDR failover configuration patterns to implement provider-level failover with health-check-driven routing.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "cloudflare_resilience", "requirement_id": "Load balancing and failover \u2014 Health-based origin routing", "fit": "partial", "rationale": "Cloudflare's load balancing and failover capabilities \u2014 including health-check-based origin failover and priority-ordered origin pools \u2014 provide an infrastructure mechanism for routing AI inference API requests away from unavailable providers to fallback providers automatically, reducing mean time to failover below what manual activation procedures could achieve. The Third-Party AI Provider Outage Response Plan control requires automated provider health monitoring and failover routing for tier-1 AI workflows, requirements that Cloudflare's load balancing platform directly addresses for organizations using Cloudflare to front their AI API traffic at the edge. Implementing Cloudflare health-based origin failover as the automated routing layer of a provider outage response plan operationalizes the control's automated failover requirement for API-layer provider dependencies.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RP-06", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "For every AI system that depends on an external model API provider, a documented provider outage contingency plan must exist specifying quantitative activation criteria, fallback routing or graceful degradation procedures for each tier of provider unavailability, stakeholder communication templates, and evidence that the plan has been validated through at least one tabletop exercise covering a multi-hour primary provider outage scenario within the past 12 months.", "evidence_required": [ "Provider outage contingency plan document per AI system showing activation trigger criteria (e.g., provider API error rate > X% for Y minutes), fallback routing configuration or graceful degradation procedures per outage tier, and stakeholder communication templates for internal and customer-facing audiences", "Fallback routing or alternative provider configuration evidence showing the fallback endpoint, credentials, and rate limit are in place, tested, and producing acceptable outputs for the AI system's use case", "Tabletop exercise record from within the last 12 months covering at least one scenario of primary provider unavailability lasting more than four hours with no estimated restoration time, including plan activation decision, fallback routing execution, and stakeholder communication steps", "Provider incident notification subscription record confirming the organization receives real-time status alerts from each critical external AI API provider via status page webhook or email", "Post-exercise action item list with owners and resolution status, confirming findings from the tabletop are tracked to remediation" ], "machine_tests": [ "Block all outbound calls to primary AI API provider in staging \u2192 assert traffic automatically routes to the fallback provider or graceful degradation activates within the documented activation time, and the system returns a defined response rather than an unhandled error", "Query provider notification configuration \u2192 assert a valid status page webhook or email subscription exists for each critical external AI API provider named in the contingency plan", "Check contingency plan document structure \u2192 assert plan contains non-null activation_criteria, fallback_procedure, and communication_templates fields and tabletop_last_run_date is within 365 days" ], "human_review": [ "Review fallback routing configuration for operational readiness \u2014 verify that alternative provider API contracts are signed, credentials are stored in secrets management, rate limits are confirmed adequate for production traffic volume, and at least one end-to-end test has been run against the fallback endpoint", "Assess whether graceful degradation procedures for AI-dependent business functions have been reviewed and formally accepted by business stakeholders as tolerable during a provider outage lasting hours or days", "Evaluate the tabletop exercise scenario used to validate the plan \u2014 confirm it modeled a realistic multi-hour outage with no ETA rather than an optimistic short-duration scenario, and that the exercise surfaced genuine gaps rather than confirming existing assumptions" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Building AI systems with a single external model API provider and no technical fallback path, making the contingency plan purely reactive documentation with no ability to rapidly restore service", "Maintaining contingency plans only for infrastructure-layer provider failures without addressing provider-initiated capability restrictions, model deprecation, or safety-driven model withdrawal scenarios", "Assuming provider outages will be short-lived and testing only five-minute outage scenarios in tabletop exercises, leaving the organization unprepared for sustained multi-hour or multi-day unavailability", "Designating a single alternative provider for all fallback scenarios without validating that the alternative provider's model produces outputs of acceptable quality and format for each specific AI use case", "Omitting customer and partner communication procedures from the contingency plan, resulting in improvised and inconsistent stakeholder messaging during a high-pressure provider outage event" ], "update_status": "current", "layer_code": "RP" }, { "id": "RP-07", "layer": "RP", "plane": "lifecycle", "name": "Recovery Plan Testing and Tabletop Exercises", "plain": "Recovery plans for AI systems must be validated through regular tabletop exercises and simulated disaster scenarios, with exercise outcomes used to identify plan gaps, update procedures, and confirm that recovery teams can execute plans under pressure without relying on individual knowledge not captured in documentation.", "threat": { "tags": [ "untested-plans", "exercise-gap", "plan-staleness", "team-capability-gap" ], "desc": "Recovery plans that have never been tested provide false assurance. Team members who have never practiced recovery procedures under simulated pressure fail to execute them accurately during an actual event, particularly when AI-specific complexity (model integrity validation, embedding regeneration, provider failover) is involved. Untested plans accumulate staleness as infrastructure and personnel change, resulting in procedures that no longer match the actual system state and that will fail at the worst possible moment." }, "standard": [ { "id": "iso_22301", "section": "\u00a78.5", "title": "Exercising and testing" }, { "id": "nist_sp800_34", "section": "\u00a73.5", "title": "Contingency plan testing" }, { "id": "iso_27031", "section": "Cl. 11", "title": "Testing, exercising and audit of ICT readiness (ISO/IEC 27031:2025)" }, { "id": "cobit_dss04", "section": "DSS04.04", "title": "Exercise, test and review the BCP and DRP" } ], "sources": [ { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RP-07 Recovery Plan Testing and Tabletop Exercises control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RP-07 Recovery Plan Testing and Tabletop Exercises control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RP-07 Recovery Plan Testing and Tabletop Exercises control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RP-07 Recovery Plan Testing and Tabletop Exercises control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Establish an annual AI recovery exercise program with at minimum one full tabletop exercise and one technical runbook walkthrough per year per tier-1 AI system, incorporating scenario injection, role-specific execution, structured after-action review, and mandatory plan update within 30 days of exercise completion.", "steps": [ "Define an annual AI recovery exercise calendar covering tabletop exercises, technical runbook walkthroughs, and partial live recovery tests for each AI system tier.", "Design exercise scenarios that cover AI-specific failure modes: model provider outage, inference pipeline corruption, vector database failure, and supply chain compromise of a model artifact.", "Execute tabletop exercises with AI system owners, recovery team members, and business stakeholders present, using scenario inject cards to simulate evolving conditions.", "Conduct structured after-action reviews following each exercise to document identified gaps, assign remediation owners, and set deadlines no longer than 60 days for critical findings.", "Update recovery plan documentation within 30 days of exercise completion to incorporate all identified gaps, and verify closure of remediation items before the next exercise cycle." ], "business_continuity": { "summary": "Business Continuity Manager owns the AI recovery exercise program, designs scenarios, facilitates tabletop exercises, and tracks after-action remediation to closure.", "actions": [ "Develop exercise scenarios tailored to AI-specific failure modes for each tier-1 system.", "Facilitate tabletop exercises with multi-stakeholder participation including business unit representatives.", "Track after-action remediation items to closure and report status to executive sponsors quarterly." ], "failure_signals": [ "No tabletop exercise was conducted for any tier-1 AI system in the last 12 months.", "Exercise scenarios do not include AI-specific failure modes.", "After-action remediation items are not tracked or have exceeded their deadlines without escalation." ] }, "it_operations": { "summary": "IT Operations participates in exercises as technical recovery executors and owns the technical runbook walkthrough component of the exercise program.", "actions": [ "Conduct technical runbook walkthroughs for all tier-1 AI system recovery procedures semi-annually.", "Identify and report infrastructure changes that invalidate recovery procedures during exercise preparation.", "Execute post-exercise infrastructure validation to confirm recovery runbook accuracy against current system state." ], "failure_signals": [ "Technical runbook walkthroughs have not been conducted for tier-1 systems in the last 6 months.", "Infrastructure changes are not reflected in recovery runbooks within 30 days of the change.", "Post-exercise validation is not performed to confirm runbook accuracy." ] }, "site_reliability": { "summary": "SRE contributes to exercise design for AI-specific technical failure scenarios and validates automated recovery tooling during exercise events.", "actions": [ "Contribute AI-specific failure scenario inputs (chaos injection, API mock failures) to tabletop exercise design.", "Validate automated failover and rollback tooling during technical exercise components.", "Report exercise findings on automated recovery tool gaps to the platform engineering team within 5 business days." ], "failure_signals": [ "SRE is not included in AI recovery exercise planning.", "Automated recovery tooling is not validated during any exercise in a 12-month period.", "Exercise findings on tooling gaps are not routed to platform engineering within SLA." ] }, "grc_auditor": { "summary": "GRC validates the AI recovery exercise program exists, is executed on schedule, and that after-action remediation is completed within defined timeframes.", "actions": [ "Request exercise records for the last 12 months and verify all tier-1 AI systems were included in at least one exercise.", "Review after-action reports for completeness of gap identification and documented remediation tracking.", "Verify that recovery plan updates were completed within 30 days of each exercise." ], "metrics": [ "Tier-1 AI systems exercised annually: target 100%.", "After-action remediation closure rate within deadline: target \u226590%.", "Recovery plan update latency post-exercise: target \u226430 days." ], "failure_signals": [ "Any tier-1 AI system not covered by a recovery exercise in the last 12 months.", "After-action reports are absent or lack documented remediation items and owners.", "Recovery plan documentation not updated within 30 days of exercise completion." ] }, "security_architect": { "summary": "Recovery exercises are the only safe place to discover that security and recovery procedures conflict. Participate in tabletops to test credential restoration, emergency access, and secure communications under recovery conditions.", "actions": [ "Inject security complications (revoked credentials, compromised backup suspicion) into recovery exercise scenarios.", "Validate emergency-access and break-glass procedures during exercises, including their revocation.", "Review exercise findings for security-relevant gaps and track them in the security backlog." ], "failure_signals": [ "Exercises assume all credentials and access work perfectly during recovery.", "Break-glass procedures have never been rehearsed.", "Security findings from exercises are not tracked to closure." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "AI recovery exercises are rarely conducted; most organizations rely on untested plans that have not been updated since initial drafting and contain procedures that no longer match current infrastructure." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "federated-enterprise", "eu-high-risk-ai" ], "implementers": [ "Business Continuity Team", "IT Operations", "GRC", "Site Reliability Team" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a78.5", "fit": "direct", "rationale": "ISO 22301:2019 \u00a78.5 requires organizations to exercise and test their business continuity plans to verify they are fit for purpose. This is a mandatory certification requirement and directly governs the minimum frequency and scope of AI recovery exercises.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.5", "fit": "direct", "rationale": "NIST SP 800-34 \u00a73.5 describes contingency plan testing methods including tabletop exercises, functional exercises, and full-scale tests. This taxonomy maps directly to the AI recovery exercise program structure and provides guidance on selecting appropriate test types per system tier.", "normative_force": "best-practice", "source_version": "Rev 1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_27031", "requirement_id": "Cl. 11", "fit": "direct", "rationale": "ISO/IEC 27031:2025 Clause 11 requires ICT readiness capabilities to be tested and exercised, with results evaluated and fed into improvement. AI-specific recovery tests and tabletop exercises fall directly within this clause.", "normative_force": "certification-standard", "source_version": "2025", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 26", "fit": "direct", "rationale": "EU DORA Article 26 requires financial entities to conduct advanced ICT risk testing including threat-led penetration testing and recovery exercises. AI system recovery tabletop exercises satisfy the exercise requirement for systems below the advanced testing threshold and contribute to DORA compliance evidence.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.04", "fit": "direct", "rationale": "COBIT 2019 DSS04.04 requires the BCP and DRP to be exercised and tested regularly, with results reviewed and improvement actions tracked. After-action remediation tracking in the AI recovery exercise program directly satisfies this practice (testing is DSS04.04; DSS04.05 is review and maintenance).", "normative_force": "best-practice", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "ID.IM-02", "fit": "direct", "rationale": "NIST CSF 2.0 ID.IM-02 requires improvements to be identified from security tests and exercises. AI recovery exercises are the mechanism that generates those improvements for recovery plans, closing the after-action loop this control requires.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "REL 13 \u2014 Testing reliability: chaos engineering and disaster recovery testing", "fit": "direct", "rationale": "AWS Well-Architected Reliability Pillar REL 13 requires organizations to conduct regular reliability tests including chaos engineering, fault injection, and disaster recovery testing \u2014 the same technical exercise methods that the Recovery Plan Testing and Tabletop Exercises control formalizes into an annual AI exercise program. The control's requirement for scenario-based testing covering AI-specific failure modes (provider outage, vector database failure, supply chain compromise) directly maps to the chaos engineering and fault injection methodologies prescribed by REL 13. Organizations running AI workloads on AWS should use AWS Fault Injection Simulator and Well-Architected Review reliability findings as inputs to their AI recovery exercise scenario design, ensuring technical exercises test real infrastructure failure paths.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Disaster Recovery Testing (DiRT) and game days", "fit": "direct", "rationale": "Google SRE's Disaster Recovery Testing (DiRT) methodology and game days practice define the industry standard for structured recovery plan validation, requiring that both technical recovery capability and organizational response procedures are tested under realistic simulated conditions with documented outcomes. The Recovery Plan Testing and Tabletop Exercises control adopts the same structure \u2014 scenario injection, role-specific execution, and structured after-action review \u2014 that Google SRE DiRT exercises operationalize at scale. Applying Google SRE DiRT principles to AI recovery exercises ensures that exercise scenarios test the full sociotechnical system including human decision-making under pressure, not just technical runbook execution in isolation.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "BCDR design guidance \u2014 Testing and validation program", "fit": "partial", "rationale": "Microsoft Azure Resiliency BCDR design guidance includes testing and validation requirements for continuity plans, prescribing tabletop exercises, functional tests, and failover tests as components of an Azure workload BCDR program that maps to the Recovery Plan Testing and Tabletop Exercises control's exercise program structure. The control requires an annual exercise calendar covering each AI system tier with structured after-action reviews, which aligns with Azure BCDR's recommendation for tiered testing programs that scale exercise scope to system criticality. Organizations with AI workloads on Azure should integrate Azure BCDR testing recommendations into their AI recovery exercise program, including Azure-specific scenarios such as availability zone failover and Azure OpenAI Service endpoint restoration.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RP-07", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "All tier-1 AI systems have documented evidence of at least one tabletop exercise in the preceding 12 months, with exercise records containing AI-specific failure scenarios, multi-stakeholder participation, and a closed after-action report with critical remediation items resolved within 60 days of exercise completion.", "evidence_required": [ "exercise_schedule_record showing an annual calendar with at least one tabletop exercise per tier-1 AI system, signed by the Business Continuity Manager", "tabletop_exercise_record per covered system containing scenario inject cards, participant attendance list (AI system owner, recovery team, and business stakeholder), and structured facilitation notes", "after_action_report per exercise documenting identified gaps, assigned remediation owners, and deadlines not exceeding 60 days for critical findings", "plan_update_acknowledgment confirming recovery plan documentation was updated within 30 days of each exercise completion" ], "machine_tests": [ "Query exercise management system for all tier-1 AI systems \u2192 assert each has at least one exercise record with a date within the last 365 days", "Parse after-action reports for tier-1 AI systems \u2192 assert each report contains at least one AI-specific failure scenario tag (model-provider-outage, vector-db-failure, or supply-chain-compromise)", "Query plan document repository last-modified timestamps for tier-1 AI recovery runbooks \u2192 assert each runbook was updated within 30 days of its most recent exercise completion date" ], "human_review": [ "Review exercise scenario design for at least one AI-specific failure mode (model provider outage, inference pipeline corruption, vector database failure) per covered system", "Assess after-action reports for quality of gap identification and whether remediation items have assigned owners with realistic deadlines that have been tracked to closure", "Verify that business stakeholders \u2014 not only technical recovery team members \u2014 participated in tabletop exercises as evidenced by signed attendance records" ], "blocking_effect": "requires-review", "normative_status": "certification-standard", "anti_patterns": [ "Conducting only technical runbook walkthroughs without multi-stakeholder tabletop exercises, leaving organizational decision-making gaps and escalation paths untested under pressure", "Using generic IT disaster recovery scenarios instead of AI-specific failure modes, failing to exercise recovery from model provider outage, vector database corruption, or supply chain compromise of a model artifact", "Completing exercises without structured after-action reviews, producing no documented gap list, no remediation ownership, and no evidence that the exercise improved plan quality", "Allowing after-action remediation items to remain unresolved past their deadlines without escalation, permitting identified plan weaknesses to persist into the next exercise cycle", "Treating exercise completion as the endpoint rather than updating recovery plan documentation within 30 days, leaving plans stale relative to exercise findings" ], "update_status": "current", "layer_code": "RP" }, { "id": "RP-08", "layer": "RP", "plane": "control", "name": "Recovery Planning Evidence Package", "plain": "The Recovery Planning layer must produce a compiled evidence package demonstrating that AI recovery plans are documented, current, tested, and owned, drawing artifacts from RP-01 through RP-07 to support regulatory compliance attestation, certification audits, and internal governance reviews.", "threat": { "tags": [ "evidence-gap", "compliance-deficiency", "audit-failure", "plan-incompleteness" ], "desc": "Individual recovery planning controls may each produce artifacts, but without a compiled and reviewed evidence package, an auditor or regulator cannot efficiently verify the completeness of the recovery planning program. Fragmented evidence stored across siloed teams creates compliance risk \u2014 an audit finding that a required artifact is missing can result in certification loss, regulatory action, or reputational harm even when the underlying capability exists but is undocumented or inaccessible." }, "standard": [ { "id": "iso_22301", "section": "\u00a79.1", "title": "Monitoring, measurement, analysis and evaluation" }, { "id": "nist_sp800_34", "section": "\u00a73.6", "title": "Contingency plan maintenance" }, { "id": "dora", "section": "Art. 11(10)", "title": "Estimation of aggregated annual costs and losses from major ICT incidents (reported on request)" }, { "id": "cobit_dss04", "section": "DSS04.06", "title": "Conduct continuity plan training" } ], "sources": [ { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RP-08 Recovery Planning Evidence Package control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Compile a structured Recovery Planning Evidence Package at least annually from artifacts produced by RP-01 through RP-07, store the package in a designated compliance repository with version control and access controls, and present it during scheduled governance reviews and on demand for regulatory inquiries.", "steps": [ "Define the Recovery Planning Evidence Package structure with required artifact types mapped to each RP control (RP-01 through RP-07), including document references, last-review dates, and owner attestations.", "Assign a named evidence package owner (Business Continuity Manager or GRC lead) responsible for compiling and updating the package on the annual review cycle.", "Collect artifacts from each RP control: BCP annex (RP-01), DR plan (RP-02), model rollback plans (RP-03), RAG recovery procedures (RP-04), tier classification register (RP-05), provider contingency plans (RP-06), and exercise records with after-action reports (RP-07).", "Review the compiled package for completeness, cross-reference artifact currency against defined review intervals, and document any identified gaps with assigned remediation owners and deadlines.", "Store the final evidence package in a version-controlled compliance repository with access controls appropriate to its sensitivity, and register it in the audit evidence management system with a document reference number." ], "business_continuity": { "summary": "Business Continuity Manager is the primary owner of the Recovery Planning Evidence Package and is responsible for its completeness, currency, and presentation to auditors.", "actions": [ "Compile and review the evidence package annually and following material changes to any AI recovery plan.", "Attest to the completeness of the package and sign off prior to audit submissions.", "Coordinate with GRC to ensure the package is registered in the audit evidence management system." ], "failure_signals": [ "Evidence package is not compiled or has not been updated in the last 12 months.", "Package owner attestation is absent for the current review cycle.", "Artifacts from one or more RP controls are missing from the package without a documented gap and remediation plan." ] }, "grc_auditor": { "summary": "GRC uses the evidence package as the primary audit artifact for RP-layer compliance assessments and regulatory submissions, and tracks evidence currency metrics.", "actions": [ "Review the evidence package against the required artifact checklist prior to each audit cycle.", "Identify and escalate gaps between documented package contents and regulatory evidence requirements.", "Track evidence package currency metrics and report to executive governance committees." ], "metrics": [ "RP control artifact coverage in evidence package: target 100% (RP-01 through RP-07).", "Evidence package review currency: compiled within last 12 months.", "Identified gaps with assigned remediation owners and deadlines: target 100%." ], "failure_signals": [ "Evidence package is missing artifacts for more than one RP control.", "Package has not been reviewed or updated in the last 12 months.", "Gaps identified in a previous review have no remediation owner or deadline." ] }, "it_operations": { "summary": "IT Operations contributes technical evidence artifacts to the package including recovery test records, system availability reports, and current runbook versions.", "actions": [ "Provide current recovery runbooks, test execution records, and system availability reports to the evidence package owner on the annual compilation schedule.", "Maintain technical artifacts in a location accessible to the evidence package owner.", "Notify the evidence package owner within 5 business days of any material change to recovery procedures." ], "failure_signals": [ "Technical artifacts are not provided to the evidence package owner on the compilation schedule.", "Recovery runbooks held by IT Operations do not match the version referenced in the evidence package.", "Material changes to recovery procedures are not communicated to the evidence package owner." ] }, "security_architect": { "summary": "The recovery-planning evidence package will be examined by auditors and, after a breach, by forensics and counsel. Ensure its integrity, access control, and completeness \u2014 including security aspects of recovery plans.", "actions": [ "Apply integrity protection (hashes, signatures) and immutable retention to evidence artifacts.", "Include security review sign-offs of recovery plans in the package's required contents.", "Restrict and log access to the evidence repository." ], "failure_signals": [ "Package contents can be silently altered after compilation.", "Security review of recovery plans is absent from the evidence set.", "Evidence access is broader than the custodian list." ] }, "site_reliability": { "summary": "Evidence compilation should be a pipeline, not a scramble: generate exercise reports, plan versions, and test results into the package automatically so it is always current.", "actions": [ "Automate collection of exercise reports, plan versions, and restore-test results into the evidence store.", "Alert when required evidence artifacts approach staleness thresholds.", "Track package compilation time and completeness as operational metrics." ], "failure_signals": [ "Producing the package takes days of manual document hunting.", "Evidence artifacts are missing for systems added in the last year.", "Staleness is discovered by auditors rather than monitoring." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Compiled recovery planning evidence packages are rare; most organizations present fragmented artifacts during audits, increasing audit preparation time and the risk of missing critical evidence." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "GRC", "Business Continuity Team", "IT Operations", "AI Platform Team" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a79.1", "fit": "direct", "rationale": "ISO 22301:2019 \u00a79.1 requires organizations to monitor, measure, analyze, and evaluate their business continuity management system. The Recovery Planning Evidence Package is the primary artifact demonstrating this evaluation has been performed for AI recovery planning.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.6", "fit": "direct", "rationale": "NIST SP 800-34 \u00a73.6 requires contingency plan maintenance including scheduled reviews and updates. The evidence package demonstrates that maintenance requirements are being met across all documented AI recovery plans with version-controlled records.", "normative_force": "best-practice", "source_version": "Rev 1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 11(10)", "fit": "partial", "rationale": "EU DORA Article 11(10) requires financial entities (other than microenterprises) to report, at the request of competent authorities, an estimation of aggregated annual costs and losses caused by major ICT-related incidents. A maintained recovery-planning evidence package is what makes such on-request reporting producible and defensible.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.06", "fit": "partial", "rationale": "COBIT 2019 DSS04.06 addresses training and awareness for continuity plans. The evidence package includes exercise and training records as evidence that recovery teams have been prepared, supporting the DSS04.06 governance requirement.", "normative_force": "best-practice", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "RC.RP-05", "fit": "direct", "rationale": "NIST CSF 2.0 RC.RP-05 requires recovery plan updates after events or exercises. The evidence package provides the audit trail of plan updates triggered by exercise findings and incident post-mortems, satisfying this continuous improvement requirement.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_27031", "requirement_id": "Cl. 10", "fit": "direct", "rationale": "ISO/IEC 27031:2025 Clause 10 requires documented IRBC plans and supporting documentation to be maintained as evidence that ICT recovery is planned and current. The Recovery Planning Evidence Package aggregates exactly that documentation for AI systems.", "normative_force": "certification-standard", "source_version": "2025", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://resilience/controls/RP-08", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "A compiled Recovery Planning Evidence Package exists, has been reviewed and attested by the named owner within the last 12 months, contains artifacts for all seven RP controls (RP-01 through RP-07), and is stored in a version-controlled compliance repository with access controls and a registered audit evidence management system reference number.", "evidence_required": [ "evidence_package_index listing all required artifact types mapped to RP-01 through RP-07, with document reference numbers, artifact owners, and last-review dates for each control", "package_owner_attestation signed by the Business Continuity Manager or GRC lead confirming completeness and currency, dated within the last 12 months", "compliance_repository_record confirming the package is stored in a version-controlled system with access controls and a registered document reference number in the audit evidence management system", "gap_register documenting any identified artifact gaps with assigned remediation owners and resolution deadlines, or explicit confirmation that no gaps exist" ], "machine_tests": [ "Query the audit evidence management system \u2192 assert an evidence package record exists with a package reference ID, owner attestation attached, and a review date within the last 365 days", "Enumerate the evidence package index \u2192 assert artifact entries exist for all seven RP control IDs (RP-01 through RP-07) with non-null, non-placeholder artifact document references", "Check compliance repository version history \u2192 assert the package has at least one version update or owner attestation refresh within the last 12 months" ], "human_review": [ "Review the package index for substantive completeness: confirm each RP control has a current artifact (not a placeholder) and that each artifact's last-review date is within its defined review interval", "Assess the gap register to verify that identified gaps have assigned owners with deadlines and that prior-cycle gaps have been resolved before the current attestation was signed", "Verify the package owner is a named individual rather than a shared team account and that the attestation is authentic and scoped to the current evidence package version" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Storing RP control artifacts in siloed team SharePoint folders rather than a unified compliance repository, making complete package assembly impossible under audit time pressure", "Relying on a shared team mailbox or unowned GRC account for evidence package attestation rather than a named individual, removing clear accountability for completeness", "Compiling the evidence package only when an audit is imminent rather than maintaining it on the annual review cycle, introducing gaps when artifacts have changed since last compilation", "Including placeholder document references for RP controls without verifying the referenced artifacts are current and accessible, creating the appearance of coverage without substance", "Storing the evidence package alongside the underlying artifacts in the same uncontrolled system, making it susceptible to the same failure modes as the artifacts it is meant to govern" ], "update_status": "current", "layer_code": "RP" }, { "id": "RO-01", "layer": "RO", "plane": "control", "name": "AI System Failover Execution Procedures", "plain": "Enterprises must maintain tested, step-by-step procedures for executing failover when primary AI system infrastructure fails, ensuring that the transition to standby or backup systems is deterministic, role-assigned, and completed within defined recovery time objectives.", "threat": { "tags": [ "failover-failure", "undocumented-recovery", "single-point-of-failure", "uncoordinated-response" ], "desc": "Without documented and rehearsed failover procedures, infrastructure failures devolve into improvised responses that exceed RTO targets and introduce additional risk. Ad hoc failover creates gaps in authorization, configuration drift between primary and standby systems, and undetected data loss. In high-risk AI deployments, an uncontrolled failover may activate a model instance operating on stale or inconsistent state, producing downstream harm before the anomaly is detected." }, "standard": [ { "id": "iso_22301", "section": "\u00a78.4.2", "title": "Incident response structure" }, { "id": "nist_sp800_34", "section": "\u00a73.4.3", "title": "Recovery procedures \u2014 alternate site activation" }, { "id": "nist_csf", "section": "RC.RP-01", "title": "Recovery plan execution" }, { "id": "aws_reliability", "section": "REL 8", "title": "Implement change \u2014 automated, tested failover procedures" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RO-01 AI System Failover Execution Procedures control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RO-01 AI System Failover Execution Procedures control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RO-01 AI System Failover Execution Procedures control.", "reviewed_on": "2026-07-01" }, { "id": "cloudflare_resilience_2024", "title": "Cloudflare DDoS Protection", "authority": "Cloudflare, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.cloudflare.com/ddos/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "cloudflare_resilience_2024", "relationship": "informative_reference", "rationale": "Establishes Cloudflare DDoS Protection requirements informing the apeiris://resilience/controls/RO-01 AI System Failover Execution Procedures control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RO-01 AI System Failover Execution Procedures control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Maintain a version-controlled runbook per AI system component (inference endpoint, orchestration layer, data pipeline) with role-assigned steps, decision checkpoints, and rollback triggers. Integrate runbooks into an on-call platform so the current version is always reachable during an incident.", "steps": [ "Inventory all AI system components requiring failover coverage and assign a primary and backup responsible team for each.", "Author a structured runbook for each component covering: trigger criteria, pre-failover health checks, traffic drain and redirect steps, standby system activation, and post-failover state verification.", "Integrate runbooks into the incident management platform (PagerDuty, OpsGenie, or equivalent) so they surface automatically when the relevant alert fires.", "Execute a tabletop test of each runbook quarterly and a live failover drill at least annually; record results and update runbooks within five business days of each exercise.", "Gate runbook changes through a pull-request review process requiring sign-off from both the owning engineering team and the business continuity manager." ], "site_reliability": { "summary": "Failover runbooks are operational contracts. They must be machine-accessible, versioned, and linked to alert definitions so engineers never have to hunt for them during an outage.", "actions": [ "Link each runbook to its corresponding alerting rule in the monitoring platform so it surfaces automatically on page.", "Automate the pre-failover health checks where possible to reduce human error under pressure.", "Track mean-time-to-failover per component and set an improvement target each quarter." ], "failure_signals": [ "Failover takes longer than the documented RTO in a real incident.", "Engineers cannot locate the runbook within two minutes of an incident declaration.", "Post-incident review identifies steps that were skipped or performed out of sequence." ] }, "it_operations": { "summary": "Operations teams execute failover runbooks during incidents. Runbooks must be unambiguous, current, and pre-authorized so execution is not delayed by approval chains.", "actions": [ "Ensure all runbook-specified accounts and credentials are pre-authorized and accessible from the break-glass vault.", "Validate that standby infrastructure configurations match production at the start of each on-call rotation.", "Document any deviations from the runbook during an actual failover for post-incident review." ], "failure_signals": [ "Credentials or access paths called out in the runbook are unavailable at incident time.", "Standby systems are found to be out of configuration sync during pre-failover checks.", "Runbook last-tested date is older than six months." ] }, "grc_auditor": { "summary": "Failover procedures are a primary artifact for demonstrating operational resilience commitments to regulators. Auditors need evidence that procedures exist, are tested, and are improved after each exercise.", "actions": [ "Request the runbook version history and confirm review cadence matches policy.", "Obtain exercise records from the last four quarters and verify documented outcomes map to identified gaps.", "Cross-reference actual incident records against runbook activation logs to confirm procedures were used." ], "metrics": [ "Runbook coverage: 100% of Tier-1 AI components have a current runbook.", "Exercise frequency: all runbooks tested at least annually.", "Post-exercise update rate: 100% of gaps resolved within five business days." ], "failure_signals": [ "Runbooks exist but have never been tested.", "Exercise records show repeated failures in the same steps without remediation.", "Audit sample reveals runbook version in use differs from the approved version." ] }, "business_continuity": { "summary": "Failover procedures must align with BCP objectives and be kept current as AI system architecture evolves.", "actions": [ "Maintain a dependency map between AI system components and business processes so each runbook addresses the correct recovery priority.", "Ensure RTO targets written into the BCP are reflected as explicit time checkpoints in runbook steps.", "Sponsor at least one enterprise-wide AI failover exercise annually to validate cross-team coordination." ], "failure_signals": [ "Business process impact is not reflected in failover priority ordering.", "RTO targets in the BCP do not match the time allowances in runbook checkpoints.", "No cross-team exercise has been completed in the past twelve months." ] }, "security_architect": { "summary": "Failover execution must not become a privilege-escalation event: runbooks need pre-scoped credentials, and standby infrastructure must enforce the same security controls as primary.", "actions": [ "Pre-provision least-privilege runbook credentials for failover steps; eliminate shared admin accounts from procedures.", "Verify standby endpoints enforce the same authN/authZ and network policy as primary before traffic lands.", "Log and review every manual override used during failover executions." ], "failure_signals": [ "Failover runbooks require root/admin credentials fetched from a wiki.", "Standby environments accept traffic with relaxed authentication.", "Manual overrides during failover leave no audit trail." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most enterprises have generic DR runbooks that do not account for AI-specific concerns such as model state, inference cache, and feature store consistency." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "eu-high-risk-ai" ], "implementers": [ "Site Reliability Engineering", "IT Operations", "Business Continuity" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a78.4.2", "fit": "direct", "rationale": "ISO 22301:2019 \u00a78.4.2 requires an incident response structure with defined roles, authorities, and documented procedures that can be activated when needed. Failover runbooks with named executor roles are the operational expression of this requirement for AI systems (\u00a78.4.3 covers warning and communication).", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.4.3", "fit": "direct", "rationale": "NIST SP 800-34 \u00a73.4.3 specifies that contingency plans must include recovery procedures covering the activation and transition to alternate sites or systems. AI failover runbooks satisfy this requirement by providing step-by-step transition procedures with defined checkpoints. The guide requires procedures be tested and updated annually.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "RC.RP-01", "fit": "direct", "rationale": "NIST CSF 2.0 RC.RP-01 requires that recovery plan execution is initiated during or after a cybersecurity incident. For AI systems, this directly maps to the activation of failover runbooks in response to infrastructure failures. The CSF emphasizes that recovery activities must be communicated to relevant parties.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "REL 8", "fit": "partial", "rationale": "AWS Well-Architected Reliability Pillar REL 8 (Implement change) requires changes \u2014 including failover activations \u2014 to be executed through tested, automated procedures rather than ad-hoc manual action. Automated, rehearsed failover runbooks apply this discipline to AI system recovery.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "dora", "requirement_id": "Art. 12(1)", "fit": "direct", "rationale": "EU DORA Article 12(1) requires financial entities to have documented response and recovery plans for ICT incidents. For AI systems operating in financial services, failover execution procedures are a compliance obligation, not merely a best practice. DORA mandates these procedures be regularly tested and updated.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "google_sre", "requirement_id": "Production Readiness Review \u2014 Failover procedures and incident response runbooks", "fit": "partial", "rationale": "The AI System Failover Execution Procedures control requires tested, step-by-step runbooks for deterministic transition to standby infrastructure within defined RTO constraints. Google SRE's Production Readiness Review process includes explicit evaluation of failover and escalation runbooks before a service is certified for production, and the SRE Workbook's incident management chapter documents the role of pre-defined failover scripts during active incidents. While advisory rather than prescriptive, the SRE checklist structure and testing cadence directly inform the runbook format and drill frequency required by this control.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "BCDR \u2014 Failover configuration and availability zone design", "fit": "direct", "rationale": "The AI System Failover Execution Procedures control requires deterministic, role-assigned failover to backup infrastructure within defined recovery time objectives. Microsoft Azure's BCDR design guidance provides prescriptive failover configuration patterns including availability zone pairing, Traffic Manager failover policies, and Azure Site Recovery orchestration that directly implement the traffic and compute transition steps this control requires. The Azure reliability documentation explicitly covers failover execution checkpoints, role assignments during transition, and post-failover verification steps for services hosted on Azure.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "cloudflare_resilience", "requirement_id": "Load balancing and failover \u2014 Health-based steering policies and failover DNS", "fit": "direct", "rationale": "The AI System Failover Execution Procedures control requires tested mechanisms that transition AI inference traffic to backup systems within RTO constraints. Cloudflare's load balancing and failover DNS capabilities provide health-check-driven origin failover and pool steering that implement the traffic-layer component of this control for AI inference endpoints fronted by Cloudflare. The Edge resilience patterns documentation specifically addresses automated failover execution for API services, reducing reliance on manual runbook steps that introduce timing variance under incident pressure.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RO-01", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every tier-1 AI system component has a version-controlled failover runbook linked to its incident management alert definition, tested within the last six months through a tabletop walkthrough or live drill, and all runbook-specified credentials and access paths are confirmed accessible from the break-glass vault at the start of each on-call rotation.", "evidence_required": [ "runbook_registry listing all tier-1 AI system components with their runbook document ID, last-modified date, and the alert definition ID to which each runbook is linked in the incident management platform", "runbook_test_record per component showing test date, test type (tabletop or live drill), participants, documented outcomes, and identified gaps with remediation status", "credential_validation_log from the break-glass vault showing all runbook-specified accounts were verified accessible at the start of each on-call rotation within the last 30 days", "runbook_change_review_record confirming pull-request approval with sign-off from both the owning engineering team and Business Continuity Manager for any runbook changes in the past 12 months" ], "machine_tests": [ "Query incident management platform alert definitions for all tier-1 AI system alerts \u2192 assert each alert has a linked runbook document ID with a last-tested date within 180 days", "Query break-glass vault for accounts referenced in tier-1 runbooks \u2192 assert all accounts have a successful access verification timestamp within the last 30 days", "Query runbook repository for tier-1 components \u2192 assert no runbook version in active use differs from the approved version recorded in the runbook registry" ], "human_review": [ "Review a sample of runbook test records to assess whether exercise outcomes were documented with sufficient detail to identify recurring failure patterns and whether remediation items were tracked to closure", "Verify that runbook steps are unambiguous enough for execution by on-call engineers who did not author them, assessing readability, decision checkpoint clarity, and rollback trigger definitions", "Assess whether AI-specific recovery concerns \u2014 model state consistency, inference cache draining, feature store synchronization \u2014 are explicitly addressed rather than treating AI systems as generic application workloads" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Storing failover runbooks in engineers' personal notes or team chat pinned messages rather than a centralized, versioned runbook library linked to alert definitions", "Writing AI system failover procedures as generic infrastructure runbooks without AI-specific steps for model state validation, inference cache draining, and feature store consistency verification after failover", "Listing credentials and access paths in runbooks without pre-validating that they are accessible from the break-glass vault, causing critical delays when credentials have been rotated since the runbook was authored", "Allowing runbooks to go untested for longer than six months as infrastructure and personnel change, accumulating staleness that causes procedure failures during real incidents", "Permitting single-engineer runbook ownership with no peer review gate, concentrating procedural knowledge in one person and creating key-person risk for recovery execution" ], "update_status": "current", "layer_code": "RO" }, { "id": "RO-02", "layer": "RO", "plane": "lifecycle", "name": "Model Rollback Execution and Validation", "plain": "When a deployed AI model version produces harmful, degraded, or non-compliant outputs, the enterprise must be able to execute a controlled rollback to a known-good prior version and validate that the rollback was successful before restoring full production traffic.", "threat": { "tags": [ "model-regression", "rollback-failure", "degraded-inference", "version-confusion" ], "desc": "A model rollback that is not fully validated before traffic restoration may leave production serving a partially reverted or configuration-mismatched model, producing inconsistent or harmful outputs that are harder to diagnose than the original failure. Without version-pinned artifacts and automated validation steps, teams may inadvertently roll back to an incompatible model-pipeline combination, corrupting inference results silently. In regulated sectors, an unvalidated rollback that continues to produce non-compliant outputs creates compounded legal exposure." }, "standard": [ { "id": "nist_800_160_v2", "section": "\u00a72.1.3", "title": "Substantiated Integrity (cyber resiliency technique)" }, { "id": "nist_csf", "section": "RC.RP-02", "title": "Recovery plan execution \u2014 system restoration" }, { "id": "iso_22301", "section": "\u00a78.4.5", "title": "Recovery \u2014 restoring business activities" }, { "id": "google_sre", "section": "Ch. 8", "title": "Release Engineering \u2014 safe rollback practices" } ], "sources": [ { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RO-02 Model Rollback Execution and Validation control.", "reviewed_on": "2026-07-02" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RO-02 Model Rollback Execution and Validation control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RO-02 Model Rollback Execution and Validation control.", "reviewed_on": "2026-07-01" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RO-02 Model Rollback Execution and Validation control.", "reviewed_on": "2026-07-01" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RO-02 Model Rollback Execution and Validation control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Maintain an immutable model artifact registry with version-pinned snapshots including weights, configuration, and serving pipeline. Rollback procedures reference registry artifact IDs, execute through the same CI/CD pipeline as forward deployments, and gate traffic restoration on automated smoke tests and behavioral regression checks.", "steps": [ "Maintain an immutable artifact registry that stores each deployed model version with its weights, serving configuration, feature pipeline binding, and dependency manifest.", "Implement a rollback command in the deployment pipeline that re-deploys a specified artifact ID through the same promotion gates used for forward deployments, ensuring consistency of configuration.", "Define a rollback validation suite covering: inference availability, output schema compliance, behavioral regression against a reference dataset, and latency within SLA thresholds.", "Require the validation suite to pass fully before directing production traffic to the rolled-back version; implement traffic shadowing as an intermediate step for high-stakes systems.", "Record all rollback actions in the incident log with the artifact ID deployed, validation results, approver identity, and timestamp of traffic restoration." ], "site_reliability": { "summary": "Model rollback must be as automated and reliable as application rollback. Treat model versions as first-class deployment artifacts with the same pipeline protections.", "actions": [ "Integrate model rollback into the same deployment automation used for forward releases, with artifact ID as the sole input.", "Build and maintain a regression test suite that validates behavioral correctness, not just uptime, after rollback.", "Define SLOs for rollback execution time and track them as reliability metrics alongside normal deployment SLOs." ], "failure_signals": [ "Rollback requires manual file transfers or out-of-band configuration changes.", "Validation suite does not include behavioral regression checks against a reference dataset.", "Rollback execution time consistently exceeds the target RTO." ] }, "security_architect": { "summary": "The artifact registry and rollback pathway are high-value attack surfaces. An adversary who can inject a malicious artifact or manipulate the rollback target can weaponize the recovery process.", "actions": [ "Enforce cryptographic signing on all artifacts in the model registry; rollback commands must verify the artifact signature before deployment.", "Apply least-privilege access to rollback commands, requiring dual authorization for Tier-1 models.", "Audit the rollback pipeline for injection points where an attacker could substitute a different artifact ID." ], "failure_signals": [ "Artifact signatures are not verified as part of the rollback execution path.", "Any single engineer can initiate and complete a rollback of a Tier-1 model without a second approver.", "The artifact registry has write access paths that bypass the signing workflow." ] }, "grc_auditor": { "summary": "Regulators and certification bodies expect evidence that model rollbacks are controlled, validated, and documented. Uncontrolled rollbacks indicate immature MLOps governance.", "actions": [ "Request artifact registry export confirming all deployed versions are version-pinned and signed.", "Review a sample of past rollback events and verify validation suite pass/fail records accompany each.", "Confirm that rollback authorization is documented with approver identity for all Tier-1 model rollbacks." ], "metrics": [ "Rollback validation pass rate before traffic restoration: target 100%.", "Artifact registry coverage: 100% of production models have immutable version records.", "Dual-authorization compliance: 100% of Tier-1 rollbacks have documented second approver." ], "failure_signals": [ "Any production rollback occurred without a validation suite execution record.", "Artifact registry contains entries lacking cryptographic signatures.", "Rollback events are logged only in unstructured chat logs rather than the incident management system." ] }, "business_continuity": { "summary": "Model rollback is a recovery action with direct business impact. Business continuity plans must account for rollback scenarios and their effect on service level commitments.", "actions": [ "Identify business processes dependent on each model version and assess rollback impact on those processes before executing.", "Include model rollback scenarios in BCP exercises so stakeholders understand the trade-offs of reverting to an older model version.", "Establish communication templates for notifying impacted business units when a rollback is executed." ], "failure_signals": [ "Business impact of rolling back to a prior model version has never been assessed.", "BCP exercises do not include any model rollback scenario.", "No communication process exists to inform business units that a rollback has changed model behavior." ] }, "it_operations": { "summary": "Operations executes the rollback: staging the prior version, swapping traffic, and validating before full restoration. The procedure must be rehearsed, timed, and monitored.", "actions": [ "Rehearse model rollback end-to-end in staging on a defined cadence, recording execution time against the RTO.", "Maintain pre-staged rollback artifacts (model, config, prompts) so execution needs no ad-hoc retrieval.", "Monitor post-rollback validation checkpoints and halt traffic ramp if checks fail." ], "failure_signals": [ "Rollback has never been executed outside of documentation.", "Artifacts needed for rollback must be located during the incident.", "Traffic is fully restored before validation checks complete." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Many organizations can perform model rollbacks manually but lack automated validation gates that confirm behavioral correctness, not just infrastructure availability, before restoring traffic." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "federated-enterprise", "eu-high-risk-ai" ], "implementers": [ "ML Engineering", "Site Reliability Engineering", "Platform Engineering" ], "frameworks": [ { "framework": "nist_800_160_v2", "requirement_id": "\u00a72.1.3", "fit": "direct", "rationale": "NIST SP 800-160 Vol 2 Rev 1 \u00a72.1.3 defines Substantiated Integrity \u2014 ascertaining that critical system elements have not been corrupted \u2014 among its cyber resiliency techniques. Rolling back to an immutable, versioned, known-good model and validating it before restoring traffic is a direct application of that technique to AI deployments.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "RC.RP-02", "fit": "direct", "rationale": "NIST CSF 2.0 RC.RP-02 requires that systems are restored to normal operations after an incident. Model rollback is the primary restoration mechanism for AI inference systems when a deployed version has caused harm or degradation. The CSF requires restoration actions to be validated before normal operations resume.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "google_sre", "requirement_id": "Ch. 8", "fit": "direct", "rationale": "Google SRE (SRE Book, 2016) Chapter 8, Release Engineering, treats reproducible builds and the ability to revert to a known-good release as core release-engineering capabilities. Model rollback execution is release engineering applied to AI model versions, including validation before full traffic restoration.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "anthropic_rsp", "requirement_id": "ASL-3 Deployment Standard \u2014 provider-initiated de-deployment", "fit": "adjacent", "rationale": "The Responsible Scaling Policy (v3.3) binds Anthropic: capability-threshold and safeguard determinations can lead Anthropic to restrict or withdraw deployed model versions. Enterprise rollback execution procedures are the consumer-side mechanism for absorbing such provider-initiated version changes as well as self-initiated reverts.", "normative_force": "best-practice", "source_version": "3.3", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "dora", "requirement_id": "Art. 9(4)(e)", "fit": "partial", "rationale": "EU DORA Article 9(4)(e) requires documented ICT change management policies, procedures and controls \u2014 covering changes to software and systems \u2014 based on a risk-assessment approach and including fallback procedures. Controlled model rollback with validation is the fallback half of change management for AI model deployments.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "aws_reliability", "requirement_id": "REL 8 \u2014 Deploy changes with automation and safe deployment practices", "fit": "partial", "rationale": "The Model Rollback Execution and Validation control requires that enterprises can execute a controlled rollback to a known-good prior AI model version and validate successful restoration before full traffic resumes. The AWS Reliability Pillar's REL 08 guidance on safe deployment practices recommends automated deployment pipelines with automated rollback triggers and defines health check patterns that validate whether a rollback was successful prior to traffic restoration. Although REL 08 addresses application deployments generally, the immutable artifact management, canary validation, and automated rollback patterns it prescribes apply directly to AI model version management.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "BCDR design guidance \u2014 Deployment rollback and blue-green recovery patterns", "fit": "partial", "rationale": "The Model Rollback Execution and Validation control requires executing a controlled revert to a prior model version when harmful or non-compliant outputs are detected, with validation before full traffic restoration. Microsoft Azure's BCDR design guidance covers deployment recovery patterns including blue-green and canary rollback approaches for AI model services hosted on Azure, and the reliability documentation addresses artifact version pinning and integrity verification as components of safe rollback. The Azure guidance on staged traffic restoration informs this control's requirement for validation gates before declaring rollback complete.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "openai_preparedness", "requirement_id": "\u00a74 Safeguarding \u2014 safeguard-driven deployment changes", "fit": "adjacent", "rationale": "The OpenAI Preparedness Framework v2 \u00a74 (Safeguarding) can change which model versions OpenAI keeps deployed once capability thresholds are reached. Tested rollback and version-pinning procedures let deployers revert cleanly when a provider withdraws or modifies a model version.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RO-02", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every production AI model has an immutable, signed artifact registry entry with version-pinned weights, serving configuration, and dependency manifest; rollback commands reference artifact IDs and verify signatures before deployment; and a validation suite covering inference availability, output schema compliance, behavioral regression against a golden dataset, and latency SLA must pass fully before production traffic is restored to the rolled-back version.", "evidence_required": [ "artifact_registry_export listing all production model versions with artifact ID, cryptographic signature, weights checksum, serving configuration hash, and dependency manifest reference for each entry", "rollback_execution_log per rollback event showing: triggering incident ID, artifact ID deployed, validation suite pass/fail per category, authorizer identity, and traffic restoration timestamp relative to validation suite completion", "rollback_validation_suite_report showing behavioral regression test results against the golden reference dataset with quantitative deviation metric, latency measurements, and output schema compliance check results", "dual_authorization_record for all tier-1 model rollbacks showing two distinct approver identities and their authorization timestamps before rollback execution began" ], "machine_tests": [ "Query artifact registry for all production models \u2192 assert each entry has a non-null cryptographic signature and all required fields: weights_checksum, serving_config_hash, dependency_manifest_ref", "Trigger a test rollback to a designated staging model version \u2192 assert rollback command verifies artifact signature before deployment, validation suite executes automatically, and traffic restoration is blocked until all suite categories pass", "Review all rollback event logs in the last 12 months \u2192 assert no traffic restoration timestamp precedes the validation suite completion timestamp for any rollback event" ], "human_review": [ "Review the rollback validation suite to assess whether the behavioral regression golden dataset is sufficiently representative of production input distribution to detect subtle output degradation, not only catastrophic failures or schema errors", "Inspect the dual-authorization procedure for tier-1 model rollbacks to verify the second approver is independent of the initiating engineer and that the approval path cannot be circumvented by a single actor under incident pressure", "Assess artifact registry access controls to confirm all write paths require cryptographic signing and that no bypass path exists for submitting unsigned artifacts" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Rolling back model versions by manually copying weight files rather than using a versioned artifact registry with signed references, creating risk of file corruption, version confusion, and inability to audit what was actually deployed", "Validating rollback success only by checking that the inference endpoint returns HTTP 200 without any behavioral regression test against a reference dataset that would detect degraded output quality", "Permitting a single engineer to both initiate and complete a tier-1 model rollback without a second approver, enabling a rogue or coerced insider to revert to a less safe model version", "Storing model artifacts in the same mutable location as the serving configuration, allowing the serving environment to diverge from the pinned artifact without detection", "Restoring production traffic to a rolled-back model version before the behavioral validation suite completes, under the assumption that infrastructure health checks are sufficient evidence of behavioral correctness" ], "update_status": "current", "layer_code": "RO" }, { "id": "RO-03", "layer": "RO", "plane": "data", "name": "Data Recovery and Integrity Verification Post-Incident", "plain": "After any incident affecting AI system data stores \u2014 including training datasets, feature stores, vector databases, and inference logs \u2014 the enterprise must execute a structured data recovery process and verify integrity before resuming AI operations that depend on the recovered data.", "threat": { "tags": [ "data-corruption", "integrity-bypass", "incomplete-recovery", "poisoned-recovery-data" ], "desc": "Data corruption or partial recovery that goes undetected before AI operations resume causes models to operate on inconsistent or attacker-influenced data, producing incorrect or harmful outputs that are difficult to trace back to the data incident. An adversary who can contaminate backup data before recovery executes a persistent poisoning attack that survives the recovery process. In federated or multi-tenant architectures, data from one tenant's corrupted store may cross-contaminate shared feature pipelines if integrity checks are not tenant-scoped." }, "standard": [ { "id": "cis_controls_v8", "section": "Control 11", "title": "Data Recovery \u2014 backup and restoration procedures" }, { "id": "iso_22301", "section": "\u00a78.4.5", "title": "Recovery \u2014 restoring ICT services and data" }, { "id": "nist_sp800_34", "section": "\u00a73.4.2", "title": "Recovery procedures \u2014 data restoration and validation" }, { "id": "nist_csf", "section": "RC.RP-03", "title": "Integrity of backups and restoration assets is verified before use" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RO-03 Data Recovery and Integrity Verification Post-Incident control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RO-03 Data Recovery and Integrity Verification Post-Incident control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RO-03 Data Recovery and Integrity Verification Post-Incident control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Maintain versioned, cryptographically hashed backups of all AI data stores. Recovery procedures reference the most recent clean snapshot predating the incident, execute a full restore, compute post-restore hashes, and compare against the pre-incident baseline before AI workloads are allowed to read from the recovered store.", "steps": [ "Catalog all data stores used by AI systems (training corpora, feature stores, vector DBs, model registries, inference logs) and assign each a backup tier with a defined RPO.", "Implement automated integrity hashing on all protected stores at each backup interval; record hashes in an append-only integrity log separate from the backup storage system.", "During data recovery, restore from the most recent snapshot predating the incident boundary, compute hashes on the restored data, and compare against the integrity log before releasing the store for AI use.", "Implement a quarantine state for recovered data stores: AI workloads receive a read-block signal until the store transitions from quarantined to verified-clean.", "Log all recovery actions \u2014 backup ID used, hash comparison result, authorizing engineer, verification timestamp \u2014 in the incident record for post-recovery audit." ], "site_reliability": { "summary": "Data integrity verification must be automated and gate AI workload access to recovered stores. Manual verification steps are too slow and error-prone for high-frequency AI operations.", "actions": [ "Deploy an automated hash verification job that runs immediately after each data restore and reports pass/fail to the incident management platform.", "Implement store-level read-block enforcement so AI workloads cannot access unverified recovered data regardless of manual errors in the recovery process.", "Monitor RPO compliance for each data store and alert when backup staleness approaches the RPO threshold." ], "failure_signals": [ "Data restore completes without any hash verification step in the execution log.", "AI workloads resumed on a recovered store before the verification job completed.", "Integrity log and backup storage reside in the same system, making both susceptible to the same failure." ] }, "security_architect": { "summary": "Backup stores are high-value targets for pre-positioning poisoned data. Integrity verification must be designed to detect both accidental corruption and deliberate tampering.", "actions": [ "Ensure integrity hashes are stored in a write-once system that is logically and physically separate from backup storage.", "Extend integrity checks to detect statistical anomalies in recovered datasets that may indicate data poisoning even when byte-level hashes match.", "Apply network segmentation to quarantined recovery stores so AI workloads cannot reach them via any path until quarantine is explicitly lifted." ], "failure_signals": [ "Integrity log is stored alongside backups, meaning a compromised backup system could also alter hashes.", "Integrity checks verify only byte-level hashes without any statistical or semantic validation of data content.", "Network controls do not enforce quarantine state independently of the application-layer read-block." ] }, "grc_auditor": { "summary": "Data recovery integrity is a core control for demonstrating that AI systems operate on trustworthy data following an incident. Auditors must verify that integrity checks are automated, documented, and effective.", "actions": [ "Request the integrity log and confirm hashes are recorded at each backup interval and compared during every recovery event.", "Review a sample of past recovery events and verify each has a documented hash comparison result before workload resumption.", "Confirm the backup inventory covers all AI data stores identified in the system data flow diagram." ], "metrics": [ "Backup coverage: 100% of Tier-1 AI data stores have automated backups meeting their defined RPO.", "Integrity verification completion rate: 100% of data recoveries include a documented hash comparison.", "Quarantine enforcement: 0 instances of AI workloads accessing unverified recovered data." ], "failure_signals": [ "Backup inventory does not include vector databases or feature stores used by AI systems.", "Any recovery event lacks a corresponding integrity verification record.", "RPO compliance has not been measured or reported in the past quarter." ] }, "business_continuity": { "summary": "Data recovery timelines and integrity gates must be reflected in BCP RTO/RPO targets. Business units must understand that data verification adds time to recovery that cannot be bypassed.", "actions": [ "Document the expected time for data integrity verification in BCP recovery timelines so RTO commitments account for the verification step.", "Identify AI systems where data freshness is critical to business output and assign them to the highest backup tier.", "Include data recovery scenarios in BCP exercises to validate that verification processes do not become bottlenecks under real incident conditions." ], "failure_signals": [ "BCP RTO targets do not account for the time required to complete data integrity verification.", "Business-critical AI systems have backup tiers that do not meet their implicit RPO requirements.", "No data recovery scenario has been tested in a BCP exercise." ] }, "it_operations": { "summary": "Operations runs the restore and owns the integrity gate: no AI data store re-enters service until checksums, record counts, and consistency checks pass and are recorded.", "actions": [ "Execute restores from runbooks with built-in integrity checkpoints (checksums, counts, referential checks) per data store.", "Quarantine restored data sets until validation results are recorded and approved.", "Keep restore tooling and credentials tested and current for every AI data store class." ], "failure_signals": [ "Restores are declared complete without recorded integrity results.", "Vector stores return online while embedding consistency is still unverified.", "Restore procedures fail because tooling drifted from current store versions." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Organizations frequently back up AI data stores but rarely implement automated integrity verification or AI-workload quarantine gates after restore, leaving recovered systems open to operating on corrupted data." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "multi-tenant", "eu-high-risk-ai" ], "implementers": [ "Data Engineering", "Site Reliability Engineering", "Security Architecture" ], "frameworks": [ { "framework": "cis_controls_v8", "requirement_id": "Control 11", "fit": "direct", "rationale": "CIS Controls v8 Control 11 (Data Recovery) requires enterprises to establish and maintain a data recovery capability including regular backups, integrity testing, and recovery verification. For AI systems, this control extends to all data stores that influence model behavior, not just traditional application databases. CIS explicitly requires testing that recovered data is usable before declaring recovery complete.", "normative_force": "best-practice", "source_version": "v8", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_22301", "requirement_id": "\u00a78.4.5", "fit": "direct", "rationale": "ISO 22301:2019 \u00a78.4.5 (Recovery) requires documented procedures for restoring business activities after a disruption. For AI-dependent processes, data restoration with integrity verification is the recovery step that makes resumed operations trustworthy.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.4.2", "fit": "direct", "rationale": "NIST SP 800-34 \u00a73.4.2 specifies that recovery procedures must include steps for data restoration and validation to ensure data integrity before resuming normal operations. The guidance explicitly requires that organizations test data recovery procedures, including verifying that restored data accurately reflects the pre-incident state.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 12(1)-(2)", "fit": "direct", "rationale": "EU DORA Article 12(1)-(2) requires backup policies, backup systems and restoration and recovery procedures that safeguard the security and integrity of data when restoring, including restoration from segregated backup systems. Post-incident data recovery with integrity verification implements those requirements for AI training data, feature stores, and vector databases.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_csf", "requirement_id": "RC.RP-03", "fit": "direct", "rationale": "NIST CSF 2.0 RC.RP-03 requires that the integrity of backups and other restoration assets is verified before they are used for restoration. This control extends that verification through restoration and into post-restore validation for AI data stores.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "REL 9 \u2014 Back up data and test recovery procedures including integrity verification", "fit": "direct", "rationale": "The Data Recovery and Integrity Verification Post-Incident control requires a structured recovery process for AI system data stores \u2014 including training datasets, feature stores, vector databases, and inference logs \u2014 with hash-based integrity verification before operations resume. The AWS Reliability Pillar's REL 09 guidance on data backup and recovery is directly applicable, covering backup policies, point-in-time restoration, and integrity verification steps that align with this control's quarantine gate requirements. The Failure management section extends these patterns specifically to the scenario of recovering data stores following incidents that affect production AI systems.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Incident management \u2014 Verification of data consistency before incident resolution", "fit": "partial", "rationale": "The Data Recovery and Integrity Verification Post-Incident control requires that all AI system data stores be verified as uncorrupted before resuming operations that depend on them, with a quarantine gate blocking resumption until verification passes. Google SRE incident management practices document the systematic steps required to confirm that all affected data has returned to a known-good state before an incident is resolved, including data consistency checks as a resolution prerequisite. The SRE Workbook's incident management chapter addresses the verification steps that must complete before declaring recovery done, directly mapping to this control's quarantine gate and hash verification requirements.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "BCDR \u2014 Data backup, geo-redundant recovery, and RPO verification for AI workloads", "fit": "direct", "rationale": "The Data Recovery and Integrity Verification Post-Incident control requires executing a structured data recovery process and verifying integrity for AI-specific data stores \u2014 including vector databases and feature stores \u2014 after incidents affecting production AI systems. Microsoft Azure's BCDR documentation provides comprehensive guidance on Azure Backup configuration, geo-redundant storage recovery procedures, and data integrity verification steps for workloads hosted on Azure including AI and ML data stores. The recovery objectives section specifically addresses how to measure and verify that restored data meets RPO commitments before resuming dependent AI operations, directly implementing this control's structured recovery and integrity gate requirements.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RO-03", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "All tier-1 AI data stores \u2014 including training datasets, feature stores, vector databases, and inference logs \u2014 have backup intervals meeting their defined RPO; post-restore integrity hashes are computed and compared against a separate append-only integrity log before AI workloads are permitted to read from the restored store; and no AI workload accessed unverified recovered data in any recovery event within the review period.", "evidence_required": [ "data_store_inventory listing all AI data stores with assigned backup tier, defined RPO, and last backup timestamp confirming compliance within the RPO window for each store", "integrity_hash_log from the append-only integrity store showing hashes computed at each backup interval for all tier-1 data stores, with the log stored in a system separate from backup storage", "recovery_event_record per data recovery showing: backup ID used, pre-restore hash from the integrity log, post-restore hash comparison result (pass/fail), authorizing engineer, quarantine-lift timestamp, and any AI workload access attempts blocked during quarantine", "quarantine_enforcement_log confirming read-block signals were active and no AI workload accessed the data store before verification completed for each recovery event" ], "machine_tests": [ "Simulate a data restore on a non-production AI data store \u2192 assert post-restore hash computation runs automatically, result is compared against the integrity log, and the store remains in quarantine (read-blocked) until comparison returns pass", "Query backup management system for all tier-1 AI data stores \u2192 assert last backup timestamp for each store is within its defined RPO window", "Attempt AI workload read access to a store in quarantine state \u2192 assert the request is denied with a quarantine-state error, not bypassed based on application-layer credentials" ], "human_review": [ "Assess whether the integrity log is genuinely write-once and stored in a system logically and physically separate from backup storage, not merely a different folder in the same storage account", "Review the data store inventory for completeness, verifying it includes vector databases and feature stores used by AI systems \u2014 not only relational databases that would appear in a standard CMDB", "Evaluate whether statistical or semantic integrity checks are applied to recovered datasets in addition to byte-level hash verification, particularly for training corpora and feature stores where data poisoning could pass hash checks if backups were poisoned before the incident boundary" ], "blocking_effect": "blocks-runtime-action", "normative_status": "binding-law", "anti_patterns": [ "Storing integrity hashes in the same backup storage system as the data they protect, allowing a compromised backup to have its hash altered to match the corrupted content", "Performing byte-level hash verification only, without statistical anomaly detection on recovered training corpora or feature stores where data poisoning could alter values while keeping file sizes and checksums plausible", "Relying on application-layer read-block logic without network-level quarantine enforcement, allowing AI workloads to bypass quarantine through direct database connections or alternative access paths outside the application", "Treating the backup inventory as covering only relational databases and omitting vector databases, feature stores, and inference log archives that AI systems depend on", "Resuming AI workloads on recovered data stores before hash comparison completes, under recovery time pressure, without any documented bypass authorization record" ], "update_status": "current", "layer_code": "RO" }, { "id": "RO-04", "layer": "RO", "plane": "control", "name": "Recovery Operations Communication Protocol", "plain": "During AI system recovery operations, a defined communication protocol must specify who is notified, at what intervals, through which channels, and with what content, ensuring that decision-makers receive timely and accurate information and that external stakeholders are informed in compliance with regulatory obligations.", "threat": { "tags": [ "communication-failure", "stakeholder-misalignment", "delayed-notification", "information-asymmetry" ], "desc": "Recovery operations that lack a defined communication protocol produce information asymmetry between technical responders, business leadership, and regulatory authorities. Decision-makers acting on stale or incomplete information may approve premature traffic restoration, escalate unnecessarily, or fail to meet mandatory breach notification windows. In federated or multi-tenant architectures, a communication failure in one recovery team can block dependent teams from acting on correct information about shared infrastructure state." }, "standard": [ { "id": "iso_22301", "section": "\u00a78.4.3", "title": "Warning and communication" }, { "id": "nist_sp800_34", "section": "\u00a74.2.2", "title": "Notification procedures" }, { "id": "dora", "section": "Art. 19", "title": "ICT incident reporting to authorities" }, { "id": "cobit_dss04", "section": "DSS04.04", "title": "Exercise, test and review the BCP and DRP \u2014 communication procedures" } ], "sources": [ { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RO-04 Recovery Operations Communication Protocol control.", "reviewed_on": "2026-07-02" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RO-04 Recovery Operations Communication Protocol control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RO-04 Recovery Operations Communication Protocol control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RO-04 Recovery Operations Communication Protocol control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Maintain a recovery communication matrix defining notification tiers (technical responders, management, executive, regulatory), trigger conditions for each tier, update interval, channel, and pre-authored message templates. Automate initial notifications through the incident management platform; require human-authored status updates at defined intervals.", "steps": [ "Define a stakeholder communication matrix with five tiers: technical response team, engineering management, executive leadership, customer/partner contacts, and regulatory authorities.", "For each tier, document: trigger condition for initial notification, maximum time from incident declaration to first notification, update interval, communication channel, and the owner responsible for sending updates.", "Author pre-approved message templates for common recovery scenarios (infrastructure failover, model rollback, data recovery) covering: incident description, current status, estimated resolution time, and any customer-visible impacts.", "Automate initial notifications through the incident management platform; require status updates from the incident commander at defined intervals (e.g., every 30 minutes for Sev-1 incidents).", "Maintain a regulatory notification register listing applicable jurisdiction-specific breach and incident notification windows (e.g., EU DORA major-incident initial notification due within 4 hours of classifying the incident as major, and no later than 24 hours after becoming aware of it) and configure automated alerts when an incident's elapsed time approaches those windows." ], "it_operations": { "summary": "IT operations teams must know exactly who to notify, when, and through which channel from the first moment of an incident. Confusion about communication responsibilities delays decision-making and increases recovery time.", "actions": [ "Ensure the communication matrix is embedded in the incident management platform and surfaces automatically at incident declaration.", "Conduct a communication drill as part of each tabletop exercise, verifying that all stakeholders receive notifications within defined windows.", "Maintain an out-of-band communication channel (e.g., SMS bridge) for use when primary communication infrastructure is part of the incident." ], "failure_signals": [ "Executive stakeholders learn of an incident through informal channels before receiving a formal notification.", "Communication matrix has not been updated after an organizational change in the past six months.", "No out-of-band communication channel exists for use when primary systems are degraded." ] }, "grc_auditor": { "summary": "Communication during recovery is a compliance obligation under multiple regulatory regimes. Auditors must verify that notification timelines are documented, met, and evidenced.", "actions": [ "Request the regulatory notification register and confirm all applicable notification windows are captured for the jurisdiction.", "Review a sample of past incident records and verify notification timestamps fall within documented SLAs for each stakeholder tier.", "Confirm automated regulatory notification alerts are configured and have been tested." ], "metrics": [ "Initial stakeholder notification compliance rate: 100% of Sev-1 incidents notify all Tier-1 stakeholders within 15 minutes.", "Regulatory notification compliance: 100% of notifiable incidents meet applicable jurisdiction deadlines.", "Communication matrix currency: reviewed and updated within the last six months." ], "failure_signals": [ "Past incident records show notification timestamps that exceed defined SLA windows.", "Regulatory notification windows are not documented for all jurisdictions in which the enterprise operates.", "Communication matrix was last reviewed more than six months ago." ] }, "business_continuity": { "summary": "The communication protocol bridges technical recovery operations and business decision-making. Business continuity managers must ensure the protocol enables executives to make informed decisions about service continuity trade-offs.", "actions": [ "Ensure executive communication templates include a business-impact summary that translates technical status into revenue and customer-service terms.", "Include communication protocol exercises in BCP drills to validate that escalation paths work under real conditions.", "Define decision authority clearly: who can approve continued degraded operation versus who must authorize full recovery before traffic is restored." ], "failure_signals": [ "Executive status updates are written in technical language that obscures business impact.", "No BCP exercise has included a communication drill in the past twelve months.", "Decision authority for traffic restoration during recovery is undefined or disputed." ] }, "security_architect": { "summary": "Communication channels used during recovery are themselves attack surfaces. An adversary who can intercept or inject into recovery communications can manipulate recovery decisions.", "actions": [ "Ensure all recovery communication channels are end-to-end encrypted and authenticated.", "Define an out-of-band channel that is independent of the primary communication infrastructure, used when that infrastructure is suspected to be compromised.", "Train incident responders to verify the identity of all participants in recovery communication bridges before sharing sensitive incident details." ], "failure_signals": [ "Primary recovery communication channel is unencrypted or uses shared credentials.", "No out-of-band channel exists for use when primary communication infrastructure is compromised.", "Incident communication bridges have no identity verification step for participants." ] }, "site_reliability": { "summary": "During recovery, communication runs on the same discipline as incident command: automated status triggers, defined intervals, and a single source of truth that on-callers update as part of the runbook.", "actions": [ "Wire recovery-state transitions to automated stakeholder notifications with runbook-defined intervals.", "Keep a single live status record (incident channel/status page) updated at each recovery checkpoint.", "Include communication checkpoints in recovery runbooks so updates are steps, not afterthoughts." ], "failure_signals": [ "Stakeholders learn recovery status by asking engineers directly.", "Update cadence collapses as soon as recovery work intensifies.", "Post-incident review finds conflicting status messages across channels." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most organizations have informal communication norms during incidents. Formal communication matrices with regulatory notification triggers are rare outside heavily regulated sectors." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "Incident Management", "Business Continuity", "Legal / Compliance" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a78.4.3", "fit": "direct", "rationale": "ISO 22301:2019 \u00a78.4.3 (Warning and communication) requires documented procedures for communicating with internal stakeholders, external parties, and authorities during a disruption. The recovery communication protocol is the AI-specific implementation of those procedures (\u00a78.4.2 defines the incident response structure).", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 19", "fit": "direct", "rationale": "EU DORA Article 19 imposes mandatory reporting of major ICT-related incidents on financial entities; under the incident-reporting technical standards, the initial notification is due within 4 hours of classifying an incident as major, and no later than 24 hours after becoming aware of it. This control's regulatory notification register and automated alerts ensure organizations know their deadlines and are warned before breaching them.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a74.2.2", "fit": "direct", "rationale": "NIST SP 800-34 Rev 1 \u00a74.2.2 defines notification procedures for plan activation \u2014 who is notified, in what sequence, by what means, and with what message content. The communication protocol's notification matrix directly implements this guidance for AI recovery operations.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.04", "fit": "partial", "rationale": "COBIT 2019 DSS04.04 requires continuity plans to be exercised and tested, which includes validating that communication procedures work under pressure. Communication-path testing in recovery exercises satisfies this practice; role training remains under DSS04.06 (Conduct continuity plan training).", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "iso_27031", "requirement_id": "Cl. 11", "fit": "partial", "rationale": "ISO/IEC 27031:2025 Clause 11 requires ICT readiness plans to be exercised and tested, including the communication procedures they contain. Regular exercises of the AI recovery communication protocol keep notification paths and templates functional.", "normative_force": "voluntary-standard", "source_version": "2025", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "google_sre", "requirement_id": "Incident management \u2014 Incident Commander role and structured stakeholder communication cadence", "fit": "direct", "rationale": "The Recovery Operations Communication Protocol control requires a defined protocol specifying who is notified, at what intervals, through which channels, and with what content during AI system recovery operations. Google SRE's incident management practices are built around a structured communication model with an Incident Commander responsible for stakeholder updates, defined status update intervals, and role-specific communication responsibilities \u2014 making the SRE incident management framework the canonical reference for operationalizing this control. The SRE Workbook's incident command chapter provides the communication cadence, escalation path structure, and status update templates that this control requires organizations to adapt for AI system recovery scenarios.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "Business continuity planning \u2014 Communication procedures and escalation trees during disruptions", "fit": "partial", "rationale": "The Recovery Operations Communication Protocol control requires pre-defined communication runbooks with channel-specific templates, regulatory notification registers, and automated alert configurations for AI system recovery operations. Microsoft Azure's BCDR design guidance includes business continuity communication planning as a required component of continuity plans, covering internal escalation paths, management notification procedures, and external stakeholder communication templates. The Azure reliability framework's emphasis on pre-defined escalation trees and communication templates tested through BCDR drills directly supports this control's requirements for documented runbooks and drill-validated communication procedures.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RO-04", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "A documented recovery communication matrix exists covering all five stakeholder tiers with defined trigger conditions, maximum initial notification windows, update intervals, assigned sender roles, communication channels, and pre-approved message templates; mandatory regulatory notification windows for all applicable jurisdictions are captured with automated pre-deadline alerts configured; and the communication protocol has been validated through a drill at least once in the preceding 12 months.", "evidence_required": [ "communication_matrix_document covering all five notification tiers (technical responders, engineering management, executive leadership, customer/partner contacts, regulatory authorities) with: trigger condition, maximum initial notification window, update interval, communication channel, and named sender role for each tier", "regulatory_notification_register listing all applicable jurisdictions, their mandatory incident notification windows, and evidence that automated pre-deadline alerts are configured and have been tested", "incident_communication_audit_log for a sample of past Sev-1 incidents showing notification timestamps per tier, confirming all timestamps fall within defined SLA windows", "communication_drill_record from at least one BCP exercise in the last 12 months showing that all stakeholder tiers received communications within their defined windows during the simulated exercise" ], "machine_tests": [ "Inject a synthetic Sev-1 incident into the incident management platform \u2192 assert automated notifications fire to technical response team and engineering management tiers within the defined window (e.g., 5 minutes) without manual trigger required", "Query the regulatory notification register for active incident tracking \u2192 assert an automated alert is configured to fire at 75% of each applicable jurisdiction's notification deadline when an active incident matches notifiable criteria", "Review past Sev-1 incident notification logs \u2192 assert 100% of events show a notification timestamp for the engineering management tier within 15 minutes of incident declaration" ], "human_review": [ "Review executive communication templates to assess whether they translate technical incident status into business-impact language \u2014 revenue exposure, customer-visible impact, estimated resolution time \u2014 rather than technical jargon that obscures decision-making", "Verify the out-of-band communication channel is genuinely independent of primary communication infrastructure, documented with tested contact lists, and accessible when primary systems are part of the incident", "Assess whether decision authority during recovery is explicitly documented: which role can approve continued degraded operation versus which role must authorize full traffic restoration, and whether this matches actual organizational authority" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Relying on informal Slack messages or chat channels as the primary recovery communication mechanism without documented notification windows, message templates, or compliance logging that demonstrates regulatory notification deadlines were met", "Maintaining a communication matrix that has not been updated after organizational changes \u2014 team restructuring, personnel departures \u2014 leaving notification paths pointing to incorrect or former recipients", "Omitting mandatory regulatory notification windows from the communication matrix, causing missed breach reporting deadlines that result in regulatory penalties even when the technical recovery was successful", "Writing executive status updates in technical infrastructure language without business-impact translation, causing executives to make uninformed decisions about service restoration trade-offs under pressure", "Having no out-of-band communication channel available when primary communication infrastructure is degraded by the same incident affecting AI systems" ], "update_status": "current", "layer_code": "RO" }, { "id": "RO-05", "layer": "RO", "plane": "control", "name": "Pre-Restoration Validation Before Traffic Resumption", "plain": "Before restoring production traffic to an AI system following any recovery operation, a defined set of validation gates must pass \u2014 covering infrastructure health, model behavioral correctness, data integrity, security posture, and compliance status \u2014 with traffic resumption blocked until all gates report clean.", "threat": { "tags": [ "premature-restoration", "validation-bypass", "cascading-failure", "undetected-degradation" ], "desc": "Restoring traffic to an AI system before validating its full recovery state exposes downstream business processes to models operating on corrupted data, incomplete configuration, or residual security vulnerabilities from the incident. In high-stakes domains such as financial decisioning or clinical AI, a premature restoration that introduces even subtle behavioral changes can cause significant harm before the anomaly is detected through downstream monitoring. Pressure to restore service quickly creates systematic incentives to skip or abbreviate validation gates unless they are automated and enforced as hard blockers." }, "standard": [ { "id": "nist_csf", "section": "RC.RP-05", "title": "System restoration \u2014 verification and validation before return to operation" }, { "id": "iso_22301", "section": "\u00a78.4.5", "title": "Recovery \u2014 validation before resuming normal operations" }, { "id": "google_sre", "section": "Ch. 14", "title": "Managing Incidents \u2014 return to service verification" }, { "id": "aws_reliability", "section": "REL 13", "title": "Test resiliency \u2014 verify recovery" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RO-05 Pre-Restoration Validation Before Traffic Resumption control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RO-05 Pre-Restoration Validation Before Traffic Resumption control.", "reviewed_on": "2026-07-02" }, { "id": "cloudflare_resilience_2024", "title": "Cloudflare DDoS Protection", "authority": "Cloudflare, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.cloudflare.com/ddos/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "cloudflare_resilience_2024", "relationship": "informative_reference", "rationale": "Establishes Cloudflare DDoS Protection requirements informing the apeiris://resilience/controls/RO-05 Pre-Restoration Validation Before Traffic Resumption control.", "reviewed_on": "2026-07-02" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RO-05 Pre-Restoration Validation Before Traffic Resumption control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RO-05 Pre-Restoration Validation Before Traffic Resumption control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RO-05 Pre-Restoration Validation Before Traffic Resumption control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Implement a multi-gate pre-restoration checklist enforced by the deployment platform. Each gate maps to a validation domain (infrastructure, model behavior, data integrity, security, compliance) with automated pass/fail signals. Traffic resumption requires all gates to pass; any gate failure blocks resumption and triggers a defined escalation path.", "steps": [ "Define a pre-restoration validation framework with five mandatory gate categories: (1) infrastructure health, (2) model behavioral correctness, (3) data integrity, (4) security posture, and (5) compliance status.", "For each gate, define: the specific test or check, the automated tool or script that executes it, the pass/fail threshold, and the escalation path if it fails.", "Implement a gate orchestrator in the deployment platform that executes all gates in parallel where possible and blocks traffic resumption until all gates report pass; record gate results with timestamps in the incident log.", "Define a bypass procedure for use only in declared crisis scenarios where continued downtime causes greater harm than the unvalidated risk; require CISO and business continuity manager dual authorization and mandatory post-restoration validation within two hours.", "Review and update gate definitions after each incident or significant architecture change to ensure they remain effective against the current threat model." ], "site_reliability": { "summary": "Pre-restoration gates must be automated and integrated into the deployment pipeline so they are enforced consistently regardless of incident pressure. Manual checklists are routinely skipped under time pressure.", "actions": [ "Automate all infrastructure health gates using existing monitoring stack probes.", "Build model behavioral correctness gates as a regression test suite that runs against a golden dataset and reports pass/fail with a quantitative deviation threshold.", "Integrate the gate orchestrator with the traffic management layer so traffic restoration is technically blocked, not just procedurally required, until all gates pass." ], "failure_signals": [ "Any gate is implemented as a manual checklist item rather than an automated test.", "Traffic can be restored by any engineer without a gate orchestrator pass signal.", "Gate execution time is not tracked; gates are suspected to be taking too long and being bypassed under pressure." ] }, "security_architect": { "summary": "Security posture gates must verify that the incident did not leave residual vulnerabilities in the restored system, including configuration drift, exposed credentials, and injection of malicious artifacts.", "actions": [ "Include a configuration integrity check in the security posture gate that compares restored system configuration against the last known-good baseline.", "Add a credential rotation verification step to confirm that any credentials potentially exposed during the incident have been rotated before traffic resumes.", "Require a vulnerability scan gate for high-severity incidents that may have involved exploitation of system components." ], "failure_signals": [ "Security posture gate does not include configuration integrity verification.", "No credential rotation check is included in pre-restoration validation.", "Security gates have never been updated after a security-category incident." ] }, "grc_auditor": { "summary": "Pre-restoration validation gates provide the primary documentary evidence that AI systems were in a compliant state before returning to production. Auditors must verify gates are enforced, not advisory.", "actions": [ "Request gate orchestrator logs for a sample of recovery events and verify all five gate categories show pass results before traffic resumption timestamps.", "Review any bypass events and confirm dual-authorization records exist for each.", "Confirm compliance status gates include checks relevant to applicable regulations (e.g., data residency, model card currency)." ], "metrics": [ "Gate completion rate: 100% of traffic resumption events have a corresponding complete gate orchestrator pass log.", "Bypass rate: target less than 2% of recovery events use the bypass procedure.", "Post-bypass validation completion: 100% of bypassed restorations complete full validation within two hours." ], "failure_signals": [ "Traffic resumption events exist in logs without a corresponding gate orchestrator pass record.", "Bypass procedure has been used without dual-authorization documentation.", "Compliance gates have not been updated after a regulatory change in the applicable jurisdiction." ] }, "business_continuity": { "summary": "Pre-restoration validation gates add time to recovery. Business continuity plans must account for gate execution time in RTO calculations and communicate clearly to stakeholders why gates cannot be skipped.", "actions": [ "Include gate execution time in RTO calculations for each AI system to ensure commitments are realistic.", "Document the business rationale for each gate in language accessible to executive stakeholders so bypass pressure is met with informed context.", "Test the full gate suite in BCP exercises to identify which gates are bottlenecks and to set accurate expectations for recovery timelines." ], "failure_signals": [ "RTO targets do not account for gate execution time, creating systematic pressure to skip gates.", "Executive stakeholders are unaware of why traffic restoration is gated after recovery.", "Gate execution time has never been measured in a BCP exercise." ] }, "it_operations": { "summary": "Operations owns the gate discipline: no traffic returns until every validation gate passes and is recorded. Pre-stage the checks so validation is fast enough that no one is tempted to skip it.", "actions": [ "Automate gate checks (infrastructure health, model behavior, data integrity, security posture) as a pre-restoration pipeline.", "Record gate results and approver identity for each restoration event.", "Rehearse the validation pipeline during recovery exercises so its runtime is known and budgeted." ], "failure_signals": [ "Traffic is restored with gates skipped under time pressure.", "Gate results exist only in chat scrollback.", "Validation takes so long that bypassing it has become normalized." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Pre-restoration validation for AI systems is rarely systematized. Infrastructure health checks are common; behavioral correctness and compliance gates are almost universally absent." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "eu-high-risk-ai", "multi-tenant" ], "implementers": [ "Site Reliability Engineering", "ML Engineering", "Security Architecture" ], "frameworks": [ { "framework": "nist_csf", "requirement_id": "RC.RP-05", "fit": "direct", "rationale": "NIST CSF 2.0 RC.RP-05 explicitly requires that systems are verified and validated before returning to normal operation following recovery. This control is the operational implementation of that outcome, providing the specific gate categories and enforcement mechanisms that make verification meaningful rather than nominal.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_22301", "requirement_id": "\u00a78.4.5", "fit": "direct", "rationale": "ISO 22301:2019 \u00a78.4.5 (Recovery) requires documented procedures for restoring and returning business activities from the measures adopted during disruption \u2014 which entails confirming the restored service is actually fit to resume. The gate-based validation framework is that confirmation step for AI systems.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "google_sre", "requirement_id": "Ch. 14", "fit": "direct", "rationale": "Google SRE Chapter 14 on managing incidents describes the return-to-service verification process and emphasizes that automated checks are more reliable than manual procedures under incident pressure. The SRE workbook's guidance on progressive traffic restoration as a risk mitigation strategy directly informs the gate orchestrator and traffic shadowing recommendations in this control.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "dora", "requirement_id": "Art. 12(2)", "fit": "partial", "rationale": "EU DORA Article 12(2) requires backup systems and restoration procedures whose activation does not jeopardise the security of network and information systems or the availability, authenticity, integrity or confidentiality of data. Pre-restoration validation gates are how an AI operator demonstrates a restored system is safe before production traffic resumes.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "aws_reliability", "requirement_id": "REL 13", "fit": "partial", "rationale": "AWS Well-Architected REL 13 on testing resiliency recommends verifying recovery effectiveness after each failure, including confirming that systems meet their functional and non-functional requirements before returning to full operation. The AWS guidance on canary deployments and health checks informs the infrastructure and behavioral correctness gate designs.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "cloudflare_resilience", "requirement_id": "Load balancing and failover \u2014 Health checks and origin validation before traffic steering", "fit": "direct", "rationale": "The Pre-Restoration Validation Before Traffic Resumption control requires that infrastructure health gates pass and origin health checks confirm recovery before production traffic is restored to an AI system. Cloudflare's load balancing health check framework provides configurable health monitors for AI inference endpoints that keep traffic diverted to healthy origins until the recovered system passes health verification, directly implementing the infrastructure gate for traffic restoration. The Edge resilience patterns documentation specifically addresses using health check states and origin pool steering to enforce the traffic resumption gate this control requires for AI API backends exposed through Cloudflare.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "Failover configuration \u2014 Return-to-service validation and progressive traffic restoration", "fit": "partial", "rationale": "The Pre-Restoration Validation Before Traffic Resumption control requires multiple validation gates \u2014 covering infrastructure health, model behavioral correctness, data integrity, security posture, and compliance status \u2014 with traffic blocked until all gates pass. Microsoft Azure's BCDR design guidance and failover configuration patterns include return-to-service verification steps that align with these gate categories, including health endpoint validation, capacity confirmation, and functional smoke tests before production traffic is accepted. The Azure reliability documentation's guidance on staged and canary traffic restoration patterns directly informs the traffic shadowing and incremental restoration approach recommended by this control.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RO-05", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Before production traffic is restored to any AI system following a recovery operation, a gate orchestrator must execute and record pass results for all five gate categories \u2014 infrastructure health, model behavioral correctness, data integrity, security posture, and compliance status \u2014 with each gate's pass timestamp preceding the traffic restoration timestamp; any bypass must carry dual-authorization documentation and be followed by full gate completion within two hours.", "evidence_required": [ "gate_orchestrator_pass_log per recovery event showing gate category, test executed, pass/fail result, and timestamp for each of the five categories, with all gates showing pass before the traffic restoration timestamp", "behavioral_regression_test_report from the model behavioral correctness gate showing results against the golden reference dataset with quantitative deviation metric below the defined threshold and latency measurements within SLA", "security_posture_check_record confirming: configuration integrity comparison against last known-good baseline, credential rotation verification for credentials in scope of the incident, and vulnerability scan completion for high-severity incidents", "bypass_authorization_record (if bypass was used) showing dual-authorization approver identities and timestamps, and post-bypass validation completion confirmation within two hours of traffic restoration" ], "machine_tests": [ "Simulate a recovery event in a staging environment \u2192 assert gate orchestrator executes all five gate categories, traffic management layer holds traffic in divert-to-backup state until all gates emit pass, and no requests reach the primary origin before gate completion", "Attempt to manually issue a traffic restoration command while a gate is in progress \u2192 assert the command is rejected with a gate-in-progress error, not accepted based on operator credentials alone", "Review all traffic resumption events in the last 12 months \u2192 assert 100% have a gate orchestrator pass log with a completion timestamp that precedes the traffic restoration timestamp for all five gate categories" ], "human_review": [ "Review the behavioral correctness gate definition to assess whether the golden reference dataset is representative of production input distribution and whether the deviation threshold is calibrated to detect harmful output degradation rather than only catastrophic or availability failures", "Verify the security posture gate's credential rotation check covers all credential types exposed to the incident scope \u2014 service account passwords, API keys, signing certificates, and OAuth client secrets \u2014 not only the most visible credential class", "Assess the bypass procedure to confirm dual-authorization requirements are genuinely enforceable and that the two-hour post-bypass validation window is monitored by automated alerting that fires if it expires without confirmed completion" ], "blocking_effect": "blocks-runtime-action", "normative_status": "binding-law", "anti_patterns": [ "Implementing pre-restoration gates as a manual checklist that engineers complete and self-certify rather than an automated gate orchestrator that technically blocks traffic restoration, making gate completion dependent on human discipline under incident pressure", "Including only infrastructure health checks (ping, HTTP 200) in restoration gates without behavioral correctness validation against a golden reference dataset, missing subtle output degradation that passes availability checks", "Permitting a single engineer to both invoke the bypass procedure and restore traffic without a second independent approver, allowing individual judgment under pressure to override the gate system without accountability", "Omitting the security posture gate from post-recovery validation, failing to detect configuration drift or exposed credentials introduced as side-effects of the recovery process itself", "Excluding compliance status from restoration gates, restoring traffic to systems where compliance-relevant configuration \u2014 data residency settings, audit logging, model card currency \u2014 was altered during the incident response without detection" ], "update_status": "current", "layer_code": "RO" }, { "id": "RO-06", "layer": "RO", "plane": "control", "name": "Recovery Operations Audit Trail", "plain": "All actions taken during AI system recovery operations must be recorded in a complete, timestamped, and tamper-evident audit trail that captures who acted, what was done, when, against which system, and with what outcome, enabling post-incident review, regulatory reporting, and accountability.", "threat": { "tags": [ "audit-tampering", "evidence-loss", "unaccountable-recovery", "forensic-gap" ], "desc": "Without a tamper-evident audit trail, post-incident investigations cannot reliably determine whether recovery actions were executed correctly, whether unauthorized changes were made under cover of the incident, or whether the incident was caused or extended by responder error. An adversary with access to the recovery environment can make unauthorized changes and then alter or delete logs to conceal their actions. In regulated environments, absence of a complete audit trail for recovery operations creates independent regulatory exposure beyond the underlying incident itself." }, "standard": [ { "id": "iso_22301", "section": "\u00a79.1", "title": "Performance evaluation \u2014 monitoring and measurement of business continuity" }, { "id": "nist_sp800_34", "section": "\u00a73.5", "title": "Contingency plan \u2014 plan testing, training, and exercises; audit trail" }, { "id": "dora", "section": "RTS (EU) 2024/1774 Art. 12", "title": "Logging \u2014 ICT risk management technical standards under DORA" }, { "id": "cobit_dss04", "section": "DSS04.08", "title": "Conduct post-resumption review" } ], "sources": [ { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RO-06 Recovery Operations Audit Trail control.", "reviewed_on": "2026-07-02" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RO-06 Recovery Operations Audit Trail control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RO-06 Recovery Operations Audit Trail control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RO-06 Recovery Operations Audit Trail control.", "reviewed_on": "2026-07-01" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RO-06 Recovery Operations Audit Trail control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Route all recovery actions through the incident management platform, which appends each action to an append-only, cryptographically chained audit log. Integrate infrastructure automation tools (runbook executors, deployment pipelines) to emit structured action records. Store the audit log in write-once storage separate from the systems being recovered.", "steps": [ "Define the required fields for each audit log entry: timestamp (ISO 8601 with millisecond precision), actor identity (authenticated user or service account), action type (enum), target system, action parameters, outcome (success/failure/partial), and source system.", "Configure all recovery automation tools (runbook executors, deployment pipelines, data restore scripts) to emit structured audit events to the centralized log, and require manual actions to be recorded through the incident management platform UI.", "Store the audit log in a write-once, append-only storage system logically and physically separate from the systems being recovered; implement cryptographic chaining (hash of previous entry included in each new entry) to detect tampering.", "Define a log completeness check that runs at incident close: identify any recovery actions recorded in system changelogs that do not have a corresponding entry in the audit log, and flag gaps for investigation.", "Retain recovery audit logs for a minimum of seven years or the applicable regulatory retention period, whichever is longer; implement access controls that allow read access for authorized auditors but deny write and delete access to all parties including administrators." ], "site_reliability": { "summary": "Audit trail completeness depends on instrumentation of recovery automation. SRE teams must ensure every tool in the recovery path emits structured audit events, not just human-readable logs.", "actions": [ "Audit the recovery toolchain to identify all tools that do not yet emit structured audit events and create a backlog to instrument them.", "Implement the log completeness check as an automated post-incident job that runs before the incident is closed.", "Store audit logs in a separate storage account with no write or delete access from the operational accounts used during recovery." ], "failure_signals": [ "Recovery automation tools emit unstructured log lines that cannot be reliably parsed into audit records.", "Audit log storage is in the same account or bucket as the systems being recovered.", "Log completeness check is not automated and relies on manual review at incident close." ] }, "security_architect": { "summary": "The audit trail is both a forensic tool and an attack target. Tamper protection must be implemented at the storage layer, not just the application layer.", "actions": [ "Implement cryptographic chaining in the audit log so any deletion or modification of historical entries is detectable.", "Enforce a separation-of-duties control so the team executing recovery cannot delete or modify their own audit records.", "Regularly test tamper detection by attempting to modify a historical entry in a non-production copy and verifying the detection mechanism fires." ], "failure_signals": [ "Audit log has no cryptographic integrity protection against modification of historical entries.", "Recovery team members have delete or write access to the audit log storage.", "Tamper detection has never been tested." ] }, "grc_auditor": { "summary": "The recovery audit trail is the primary evidence artifact for demonstrating that recovery operations were authorized, executed correctly, and complete. Auditors must verify completeness, integrity, and retention.", "actions": [ "Request the audit log for a sample of recovery events and verify all required fields are present in each entry.", "Test cryptographic chain integrity by running the chain verification tool against a sample of log segments.", "Confirm retention period meets the longer of the internal policy and any applicable regulatory requirement." ], "metrics": [ "Audit log completeness rate: 100% of recovery actions have a corresponding structured log entry.", "Integrity verification: cryptographic chain passes verification for 100% of log segments sampled.", "Retention compliance: 100% of audit logs retained for the required period." ], "failure_signals": [ "Log completeness check identifies gaps between system changelogs and audit log entries.", "Cryptographic chain verification fails for any segment.", "Retention period is shorter than applicable regulatory requirements." ] }, "business_continuity": { "summary": "Recovery audit trails support post-incident learning and regulatory obligations. Business continuity managers must ensure the trail supports both internal review and external reporting requirements.", "actions": [ "Include audit trail review as a mandatory step in the post-incident review process to identify action sequences that could be improved.", "Confirm that the audit trail format and retention satisfies requirements for all applicable regulatory bodies in each jurisdiction.", "Use audit trail data to calculate actual recovery timelines for comparison against RTO targets in the continuous improvement process." ], "failure_signals": [ "Post-incident reviews are conducted without reference to the audit trail, relying instead on participants' recollections.", "Audit trail format has not been reviewed against regulatory reporting requirements for applicable jurisdictions.", "Actual recovery timelines cannot be calculated from the audit trail due to missing timestamps." ] }, "it_operations": { "summary": "The audit trail is built by the people doing the recovery: every action, actor, timestamp, and outcome must land in the log as work happens \u2014 tooling should make that automatic.", "actions": [ "Route recovery actions through tooling that logs actor, action, target, and outcome automatically.", "Reconcile the action log against the incident timeline before closing the recovery.", "Protect recovery logs with append-only storage and restricted access." ], "failure_signals": [ "Recovery actions are reconstructed from memory after the fact.", "Console changes during recovery bypass logged tooling.", "Log entries can be edited or deleted by the same operators who act." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most organizations have logs from individual recovery tools but lack a unified, tamper-evident audit trail that spans the entire recovery operation. Completeness checks and cryptographic integrity protection are rarely implemented." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise", "multi-tenant" ], "implementers": [ "Incident Management", "Security Architecture", "IT Operations" ], "frameworks": [ { "framework": "dora", "requirement_id": "RTS (EU) 2024/1774 Art. 12", "fit": "direct", "rationale": "Logging obligations under DORA sit in the ICT risk-management technical standards: Commission Delegated Regulation (EU) 2024/1774 Article 12 requires financial entities to log relevant ICT events and protect those logs against tampering. Tamper-evident recovery audit trails implement that requirement for AI recovery actions (DORA Article 9 itself addresses protection and prevention, not logging).", "normative_force": "binding-law", "source_version": "2024/1774", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "iso_22301", "requirement_id": "\u00a79.1", "fit": "direct", "rationale": "ISO 22301:2019 \u00a79.1 requires organizations to monitor, measure, analyze, and evaluate their business continuity management system. Recovery audit trails provide the measurement data for this requirement, enabling organizations to assess whether recovery operations met their objectives and identify areas for improvement.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.5", "fit": "direct", "rationale": "NIST SP 800-34 \u00a73.5 addresses contingency plan testing, training, and exercises, requiring that outcomes be documented and used to improve the plan. For actual recovery events, the audit trail serves this documentation function, capturing the real-world execution record that drives improvement more effectively than exercise outcomes alone.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.08", "fit": "partial", "rationale": "COBIT 2019 DSS04.08 requires a post-resumption review to assess whether continuity actions were effective once operations resume. The recovery audit trail is the evidence base that makes an objective post-resumption review possible (DSS04.07 covers backup arrangements, not reviews).", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "cis_controls_v8", "requirement_id": "Control 8.2", "fit": "partial", "rationale": "CIS Controls v8 Control 8.2 requires that audit logs support incident response and forensic investigations. Recovery operations audit trails extend this requirement specifically to the recovery phase, ensuring that forensic capability is maintained throughout the incident lifecycle, not just during initial detection and containment.", "normative_force": "best-practice", "source_version": "v8", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "google_sre", "requirement_id": "Postmortem practices \u2014 Timestamped action log and incident timeline reconstruction", "fit": "direct", "rationale": "The Recovery Operations Audit Trail control requires a complete, timestamped, and tamper-evident record capturing who acted, what was done, when, against which system, and with what outcome during AI system recovery. Google SRE's postmortem practices explicitly require the construction of a detailed incident timeline documenting each action taken during the incident and recovery, who took it, and what effect it had \u2014 making the SRE postmortem methodology the canonical reference for this control's audit trail content requirements. The SRE Workbook's incident management chapter prescribes real-time action logging during incidents as the foundation for the post-incident review, directly supporting this control's tamper-evident sequential logging requirements.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "aws_reliability", "requirement_id": "REL 6 \u2014 Monitor workload resources and capture tamper-resistant change records via CloudTrail", "fit": "partial", "rationale": "The Recovery Operations Audit Trail control requires that all infrastructure changes made during AI system recovery are captured in a structured, timestamped, and tamper-evident log. The AWS Reliability Pillar's REL 06 guidance on workload monitoring establishes logging and instrumentation patterns that provide the infrastructure for recovery audit trails on AWS-hosted AI systems, and the Failure management section specifically describes using AWS CloudTrail and AWS Config to generate tamper-resistant records of all infrastructure actions taken during incident response and recovery. These AWS-native controls fulfill the immutability and completeness requirements this control places on infrastructure-layer recovery log entries.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "BCDR design guidance \u2014 Recovery documentation and activity log requirements", "fit": "partial", "rationale": "The Recovery Operations Audit Trail control requires complete documentation of recovery actions with tamper-evidence to support post-incident review and regulatory compliance for AI system incidents. Microsoft Azure's BCDR design guidance includes documentation requirements specifying that all actions taken during failover and recovery must be recorded to satisfy audit and compliance obligations, and Azure Activity Log provides the infrastructure-layer component of this audit trail for AI systems hosted on Azure. The Azure Monitor integration with BCDR procedures enables the correlated, timestamped audit record that this control requires across infrastructure, application, and recovery-action event streams.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RO-06", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "The recovery operations audit log must contain a structured entry for every action taken during recovery, with all required fields populated (actor identity, action type, target system, outcome, ISO 8601 timestamp), stored in a write-once tamper-evident store with an intact cryptographic hash chain. No recovery event may close without a passing log completeness check that cross-references all source system changelogs against the audit log.", "evidence_required": [ "recovery_audit_log with structured entries for every recovery action containing actor_identity, action_type, target_system, outcome, and ISO 8601 millisecond-precision timestamp", "cryptographic_chain_verification_report confirming hash chain integrity across all log segments sampled from the audit period with zero chain_invalid results", "log_completeness_check_results cross-referencing system changelogs against audit entries and confirming zero unmatched action gaps at incident close", "write_once_storage_configuration_record confirming audit log resides in a separate storage account with no delete or write access for operational accounts", "retention_compliance_certificate confirming log retention meets the longer of seven years or the applicable regulatory period for all jurisdictions in scope" ], "machine_tests": [ "Inject a synthetic recovery action via the recovery automation API \u2192 assert a structured audit entry is created within 5 seconds containing actor_identity, action_type, target_system, outcome, and timestamp fields", "Attempt to modify a historical audit log entry in a non-production copy \u2192 assert the cryptographic chain verification tool detects the tamper and returns chain_invalid=true for the affected segment", "Simulate incident close without completing all recovery tool actions \u2192 assert the log completeness check identifies the gap and emits an unmatched_action_detected alert before close is permitted", "Attempt to delete an audit log entry using operational account credentials \u2192 assert 403 Forbidden response and a deletion_attempt_blocked event in the security information event log" ], "human_review": [ "Review the cryptographic chain implementation to confirm each entry includes the hash of the preceding entry and that storage enforcement prevents administrative delete access for all parties including security administrators", "Assess the log completeness check procedure to verify it cross-references all recovery toolchain outputs automatically and is required before incident close rather than run on request", "Verify retention policy configuration satisfies the longer of seven years or the applicable regulatory period for every jurisdiction where AI systems operate" ], "blocking_effect": "advisory", "normative_status": "binding-law", "anti_patterns": [ "Routing recovery actions through tools that emit unstructured log lines instead of structured audit events, preventing automated completeness verification", "Storing the recovery audit log in the same storage account as the systems being recovered, allowing responders to alter or delete their own audit records", "Relying on manual collection of audit records at incident close rather than automated append-only logging during recovery execution", "Implementing cryptographic chaining at the application layer without write-once storage enforcement at the storage layer, enabling hash manipulation bypasses", "Setting log retention to the internal policy minimum without checking whether it satisfies applicable regulatory retention periods for each jurisdiction in scope" ], "update_status": "current", "layer_code": "RO" }, { "id": "RO-07", "layer": "RO", "plane": "control", "name": "Recovery Time Compliance Verification", "plain": "After completing AI system recovery operations, the enterprise must verify that actual recovery times and recovery point objectives were met against the commitments defined in recovery plans, document any deviations, assess their business and regulatory impact, and initiate corrective action when targets are missed.", "threat": { "tags": [ "rto-breach", "rpo-violation", "unverified-compliance", "undetected-sla-failure" ], "desc": "Organizations frequently believe their recovery capabilities meet defined RTO/RPO targets without actually measuring whether they do. Unverified compliance creates false confidence in resilience posture, causes organizations to miss SLA breach notifications owed to customers and regulators, and prevents corrective investment from reaching the controls where it is most needed. In federated or multi-tenant architectures, a single component's RTO breach can cascade into missed targets for dependent services, amplifying the business and regulatory impact of what appeared to be a minor failure." }, "standard": [ { "id": "iso_22301", "section": "\u00a79.1", "title": "Monitoring, measurement, analysis and evaluation" }, { "id": "nist_sp800_34", "section": "\u00a73.5", "title": "Plan testing, training, and exercises \u2014 validating recovery against objectives" }, { "id": "dora", "section": "Art. 12(2)", "title": "Backup systems and periodically tested restoration and recovery" }, { "id": "cobit_dss04", "section": "DSS04.08", "title": "Conduct post-resumption review" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RO-07 Recovery Time Compliance Verification control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RO-07 Recovery Time Compliance Verification control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RO-07 Recovery Time Compliance Verification control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "At incident close, automatically extract recovery timeline data from the audit trail and compare against the documented RTO/RPO for each affected system. Generate a compliance report with pass/fail status for each target, flag breaches for management review, and feed results into the continuity plan improvement cycle.", "steps": [ "Maintain a recovery target register mapping each AI system component to its documented RTO (time to restore service) and RPO (maximum acceptable data loss window).", "At incident close, extract recovery timeline data from the audit trail: incident start time, failover or recovery completion time, and data recovery point from backup metadata.", "Calculate actual RTO and RPO for each affected component and compare against the target register; generate a compliance report with pass/fail status for each component.", "Route RTO/RPO breach reports automatically to the business continuity manager and the owning engineering team for acknowledgment and corrective action within five business days.", "Aggregate compliance data quarterly to identify systemic patterns (e.g., a specific component type that consistently misses RTO) and feed findings into the business continuity plan improvement cycle." ], "site_reliability": { "summary": "RTO/RPO compliance measurement should be automated as a post-incident job using audit trail data. Manual measurement is too slow and too inconsistent to support continuous improvement.", "actions": [ "Build an automated post-incident job that extracts timeline data from the audit trail and computes RTO/RPO actuals within 30 minutes of incident close.", "Create an SLO dashboard that tracks rolling RTO/RPO compliance by system component over a 90-day window.", "Automate escalation alerts when a breach is detected so engineering teams receive notification before the post-incident review." ], "failure_signals": [ "RTO/RPO actuals are calculated manually from memory or chat logs rather than from structured audit trail data.", "No dashboard exists to track compliance trends over time.", "Engineering teams learn of RTO breaches for the first time at the post-incident review meeting." ] }, "grc_auditor": { "summary": "RTO/RPO compliance verification is a direct regulatory obligation in several frameworks. Auditors must verify that targets are documented, measured, and that breaches are reported and remediated.", "actions": [ "Request the recovery target register and confirm all Tier-1 AI systems have documented RTO and RPO values approved by business ownership.", "Review compliance reports for the past four quarters and verify breach rates, documentation of root cause, and corrective action completion.", "Confirm that RTO/RPO breach notifications reach the business continuity manager within the defined window and are escalated to regulators where applicable." ], "metrics": [ "RTO compliance rate: target greater than 98% of recovery events meet the documented RTO for each system.", "RPO compliance rate: target greater than 99% of recovery events meet the documented RPO for each system.", "Breach remediation completion: 100% of documented RTO/RPO breaches have a corrective action record within 30 days." ], "failure_signals": [ "Recovery target register does not exist or has not been updated in the past year.", "RTO/RPO compliance has never been calculated for any real incident.", "Breach reports have been generated but corrective actions have not been initiated." ] }, "business_continuity": { "summary": "Recovery time compliance is the primary metric for measuring business continuity effectiveness. Business continuity managers must own the measurement process and drive improvement when targets are missed.", "actions": [ "Review RTO/RPO compliance reports at each monthly business continuity management review.", "Ensure that RTO/RPO targets in the recovery target register are aligned with contractual SLA commitments and regulatory obligations.", "Use compliance trend data to prioritize investment in the components with the worst compliance history." ], "failure_signals": [ "RTO/RPO targets have not been validated against contractual SLAs in the past year.", "Management reviews do not reference compliance measurement data.", "Investment in recovery capability improvements is not linked to compliance trend data." ] }, "it_operations": { "summary": "IT operations must ensure that all recovery actions are captured in the audit trail with sufficient timestamp precision to support accurate RTO/RPO calculation.", "actions": [ "Verify that all recovery automation tools record timestamps with at least one-second precision in the audit trail.", "Confirm that the incident management platform records the formal incident declaration time, as this is the RTO clock start.", "Participate in the post-incident RTO/RPO calculation review to validate that calculated actuals match the operational experience of the recovery team." ], "failure_signals": [ "Audit trail timestamps lack sufficient precision to calculate RTO within a one-minute margin.", "Incident declaration time is not consistently recorded in the incident management platform.", "Operations team disputes the calculated RTO actuals after the fact due to data quality issues." ] }, "security_architect": { "summary": "Recovery-time compliance has a security dimension: missed RTOs often trace to security steps (credential restoration, verification gates) that were never engineered for speed. Make security-path latency measurable and improvable.", "actions": [ "Instrument the security steps of recovery (secret rehydration, access re-provisioning, validation gates) as measured segments of RTO.", "Engineer pre-approved fast paths for security steps that dominate recovery time.", "Review RTO misses for security-step causes and feed fixes into the security roadmap." ], "failure_signals": [ "Security steps are the recurring long pole in recovery timelines but are never itemized.", "Teams bypass security validation to hit RTO targets.", "No data exists on how long credential restoration actually takes." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "The majority of organizations document RTO/RPO targets but do not systematically measure compliance after actual recovery events. The gap between stated and actual recovery capability is typically unknown." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "cloud-native" ], "implementers": [ "Business Continuity", "Site Reliability Engineering", "IT Operations" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a79.1", "fit": "direct", "rationale": "ISO 22301:2019 \u00a79.1 requires the organization to determine what needs to be monitored and measured and to evaluate BCMS performance against objectives \u2014 including whether recovery time objectives are actually met (the 2019 edition has no third-level subclauses here). Systematic RTO/RPO compliance measurement is that evaluation for AI systems.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 12(2)", "fit": "partial", "rationale": "EU DORA Article 12(2) requires backup systems and restoration and recovery procedures and methods that are tested periodically. Verifying that actual recovery performance met documented RTO/RPO commitments after each activation supplies the evidence that this testing obligation is met in substance.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.5", "fit": "direct", "rationale": "NIST SP 800-34 Rev 1 \u00a73.5 requires contingency plans to be validated through tests and exercises that confirm recovery can be achieved within stated objectives. Measuring actual recovery times against documented RTO/RPO \u2014 in exercises and in real events \u2014 extends that validation discipline into operations.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.08", "fit": "direct", "rationale": "COBIT 2019 DSS04.08 (Conduct post-resumption review) requires assessing whether recovery met its objectives once business operations resume. Measuring actual RTO/RPO against documented commitments and initiating corrective action is the substance of that review.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "ID.IM-03", "fit": "partial", "rationale": "NIST CSF 2.0 ID.IM-03 requires improvements to be identified from the execution of operational processes, procedures, and activities. RTO/RPO compliance measurement after each recovery event generates precisely those improvement inputs.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "REL 13 \u2014 Test resiliency and verify that recovery objectives can be met", "fit": "direct", "rationale": "The Recovery Time Compliance Verification control requires that actual recovery times and recovery point objectives are measured against documented commitments after each recovery event, with deviations documented and corrective action initiated when targets are missed. The AWS Well-Architected Reliability Pillar's REL 13 guidance on testing resiliency explicitly addresses verifying that RTO and RPO commitments can be met, and the Testing reliability and RTO/RPO alignment sections provide the measurement methodology for comparing actual versus target recovery metrics across real incidents and exercises alike. The corrective-action loop this control requires directly aligns with the REL 13 practice of using resiliency test outcomes to drive workload improvement.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "SLO management \u2014 Error budget consumption and SLA breach detection", "fit": "direct", "rationale": "The Recovery Time Compliance Verification control measures actual recovery durations and recovery point gaps against contractual and regulatory RTO/RPO commitments, initiating corrective action when targets are missed. Google SRE's SLO/SLA management and error budget practices provide the quantitative framework for measuring whether AI system availability commitments were honored during recovery events, directly informing this control's compliance dashboard and automated measurement requirements. The SRE approach to error budget consumption tracking \u2014 treating availability shortfalls as budget draws that require management attention \u2014 translates directly to the RTO over-run detection and escalation that this control mandates.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "Recovery objectives \u2014 RTO/RPO measurement and BCDR compliance verification", "fit": "direct", "rationale": "The Recovery Time Compliance Verification control requires automated measurement of actual versus target recovery metrics, regulatory impact assessment for misses, and improvement plans when targets are not achieved. Microsoft Azure's BCDR design guidance includes specific patterns for measuring recovery objective compliance using Azure Monitor and Azure Site Recovery metrics, with dashboards tracking actual RTO/RPO performance against business continuity plan commitments. The recovery objectives section provides the measurement framework and deviation analysis approach that this control requires enterprises to operationalize for each AI system recovery event.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RO-07", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "For every AI system recovery event, actual RTO and RPO values must be automatically calculated from structured audit trail timestamps and compared against the documented targets in the recovery target register, with a compliance report generated at incident close and any breach routed to the business continuity manager and owning engineering team within five business days for corrective action.", "evidence_required": [ "recovery_target_register mapping each AI system component to business-owner-approved RTO and RPO values validated against contractual SLA commitments", "rto_rpo_compliance_report with actual_rto, target_rto, actual_rpo, target_rpo, and compliance_status fields per affected component generated automatically from audit trail data for each recovery event", "breach_remediation_records showing corrective action acknowledgment, root cause documentation, and completion status for all documented RTO or RPO breaches within 30 days", "rolling_compliance_dashboard export showing 90-day RTO and RPO compliance rates by system component used in quarterly management reviews" ], "machine_tests": [ "Execute the post-incident RTO/RPO calculation job against audit trail data from a past recovery event \u2192 assert the output report contains actual_rto, target_rto, actual_rpo, target_rpo, and compliance_status for every affected component with timestamp precision within one minute", "Inject a synthetic recovery event where actual_rto exceeds target_rto by 10% \u2192 assert a breach alert is routed to the business continuity manager within 30 minutes and a corrective action record is created with breach_severity and affected_component fields", "Submit RTO/RPO calculation request for a system absent from the recovery target register \u2192 assert the job returns error_code=system_not_registered rather than producing an unvalidated compliance result" ], "human_review": [ "Review the recovery target register to confirm all Tier-1 AI systems have documented RTO and RPO values approved by business owners and validated against contractual SLA commitments and applicable regulatory obligations", "Assess the breach remediation workflow to verify corrective actions are completed within the defined window, tracked to closure with documented root cause, and fed back into the business continuity plan improvement cycle", "Examine quarterly trend reports to confirm compliance data is actively used to prioritize investment in the system components with the worst compliance history" ], "blocking_effect": "advisory", "normative_status": "binding-law", "anti_patterns": [ "Calculating RTO/RPO actuals from participant memory or incident chat logs at the post-incident review rather than from structured audit trail timestamp data", "Documenting RTO/RPO targets in the recovery plan narrative without creating a machine-queryable register that the compliance calculation job can reference by system component identifier", "Generating breach reports but failing to initiate corrective action within the defined window, allowing breaches to accumulate without remediation or management escalation", "Treating RTO/RPO compliance measurement as optional for incidents below Sev-1, missing systemic patterns in lower-severity events that predict future high-severity breaches", "Setting RTO/RPO targets based on assumed recovery capability rather than validating them against actual performance from exercises or past incidents" ], "update_status": "current", "layer_code": "RO" }, { "id": "RO-08", "layer": "RO", "plane": "both", "name": "Recovery Operations Evidence Package", "plain": "At the close of any AI system recovery event, the enterprise must compile a structured evidence package that aggregates artifacts from RO-01 through RO-07, demonstrating that failover procedures were followed, model state was restored correctly, data integrity was verified, communications were timely, restoration validation passed, actions were fully audited, and recovery time targets were met.", "threat": { "tags": [ "evidence-gap", "audit-failure", "unverifiable-recovery", "regulatory-exposure" ], "desc": "Without a compiled and structured evidence package, recovery operations cannot be verified by auditors, regulators, or post-incident reviewers. Individual artifacts scattered across disparate systems create an evidence reconstruction burden that degrades over time as systems rotate logs and personnel change roles. In regulated sectors, the inability to produce a coherent recovery evidence package in response to a regulatory inquiry creates independent liability beyond the underlying incident. Fragmented evidence also prevents the organization from identifying systematic weaknesses across the full recovery lifecycle." }, "standard": [ { "id": "iso_22301", "section": "\u00a79.2", "title": "Internal audit \u2014 evidence of business continuity management performance" }, { "id": "dora", "section": "Art. 13(2)", "title": "Learning and evolving \u2014 post-incident reviews" }, { "id": "nist_csf", "section": "ID.IM-04", "title": "Cybersecurity plans that affect operations are communicated, maintained, and improved" }, { "id": "cobit_dss04", "section": "DSS04.08", "title": "Conduct post-resumption review" } ], "sources": [ { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RO-08 Recovery Operations Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RO-08 Recovery Operations Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RO-08 Recovery Operations Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RO-08 Recovery Operations Evidence Package control.", "reviewed_on": "2026-07-01" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RO-08 Recovery Operations Evidence Package control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Define a recovery evidence package schema that specifies required artifacts from each RO control. At incident close, execute an evidence compilation job that retrieves the required artifacts from their source systems, validates completeness, generates a summary report, cryptographically signs the package, and archives it to long-term retention storage.", "steps": [ "Define the evidence package schema with eight required artifact categories: (1) failover execution log (RO-01), (2) model version rollback record with validation results (RO-02), (3) data recovery log with integrity verification results (RO-03), (4) communication log with stakeholder notification timestamps (RO-04), (5) pre-restoration gate orchestrator results (RO-05), (6) complete audit trail extract (RO-06), (7) RTO/RPO compliance report (RO-07), and (8) package metadata with incident summary and overall recovery verdict.", "Implement an evidence compilation job triggered at incident close that retrieves each required artifact from its source system via API, validates that each artifact is present and complete, and flags any missing artifacts as package exceptions.", "Generate a human-readable summary report from the compiled artifacts that includes: incident timeline, recovery actions taken, deviations from planned procedures, RTO/RPO compliance status, and open corrective actions.", "Cryptographically sign the complete evidence package with the enterprise PKI and store it in write-once, long-term retention storage with metadata enabling retrieval by incident ID, date range, and system name.", "Implement an evidence completeness SLA: the full package must be compiled, signed, and archived within 24 hours of incident close for Sev-1 incidents and 72 hours for Sev-2 and below." ], "site_reliability": { "summary": "Evidence package compilation should be automated to the maximum extent possible. Manual compilation is slow, error-prone, and dependent on responders who are often already exhausted at incident close.", "actions": [ "Automate artifact retrieval from all source systems (incident management platform, deployment pipeline, audit log store) as a post-incident pipeline job.", "Build completeness validation into the compilation job so missing artifacts trigger an alert immediately rather than being discovered weeks later during an audit.", "Track evidence package compilation time as an operational metric and set an SLO to ensure the 24-hour target is consistently met." ], "failure_signals": [ "Evidence package compilation relies on manual steps that are frequently incomplete at the 24-hour mark.", "Completeness validation has not been implemented; missing artifacts are only discovered during audits.", "Evidence packages from more than 10% of incidents are incomplete or missing at the 30-day mark." ] }, "security_architect": { "summary": "The evidence package must be cryptographically integrity-protected to prevent post-incident tampering. The signing key and the package archive must be in separate custody from the recovery teams whose actions the package documents.", "actions": [ "Implement package signing using a PKI certificate held by the security team, separate from the engineering teams whose recovery actions are documented.", "Store the package archive in a storage system with separate access credentials from the systems used during recovery, preventing responders from modifying their own evidence.", "Include the signing certificate chain in the archived package to ensure verifiability without dependency on live PKI services." ], "failure_signals": [ "Evidence packages are not cryptographically signed.", "Package archive storage is accessible to the same teams whose actions are documented in the package.", "Package signatures cannot be verified without querying a live PKI service, creating a dependency that could fail during future audits." ] }, "grc_auditor": { "summary": "The evidence package is the primary artifact for demonstrating recovery operations compliance to regulators, auditors, and certification bodies. Its completeness, integrity, and availability are the auditor's first concern.", "actions": [ "Request evidence packages for a sample of recovery events spanning the audit period and verify all eight required artifact categories are present.", "Verify cryptographic signatures on sampled packages using the enterprise PKI certificate.", "Confirm that packages are available within the defined compilation SLA and that the archive is retained for the required period." ], "metrics": [ "Package completeness rate: 100% of Sev-1 incidents have a complete package with all eight artifact categories.", "Package compilation SLA compliance: 100% of packages compiled within 24 hours (Sev-1) or 72 hours (Sev-2).", "Signature verification rate: 100% of sampled packages have a valid cryptographic signature.", "Retention compliance: 100% of packages retained for the required period." ], "failure_signals": [ "Any Sev-1 incident package is missing one or more required artifact categories.", "Package compilation SLA was missed for more than 5% of incidents in the review period.", "Any package fails cryptographic signature verification.", "Packages older than 12 months are unavailable due to retention policy misconfiguration." ] }, "business_continuity": { "summary": "The evidence package is the foundation for post-incident review and continuous improvement. Business continuity managers must ensure the package structure captures the information needed to drive meaningful improvement, not just regulatory compliance.", "actions": [ "Review the evidence package schema annually to ensure it captures the information most useful for identifying systemic recovery weaknesses.", "Use evidence packages as the primary input to the post-incident review process, ensuring reviews are grounded in documented facts rather than participant recollections.", "Maintain a trend analysis of evidence package findings across incidents to identify patterns that indicate systemic capability gaps requiring investment." ], "failure_signals": [ "Post-incident reviews are conducted from recollection rather than evidence package data.", "Evidence package schema has not been reviewed or updated in the past year.", "No trend analysis of package findings exists to identify systemic weaknesses." ] }, "it_operations": { "summary": "Operations teams must ensure that source systems for all required artifacts are accessible and API-queryable at incident close, and that artifact retention in source systems covers the compilation SLA window.", "actions": [ "Verify that all artifact source systems retain data for at least 48 hours post-incident, ensuring availability for the compilation job within the SLA window.", "Test the evidence compilation job quarterly by running it against a past incident and verifying completeness.", "Ensure that the evidence archive is included in the organization's backup and DR coverage." ], "failure_signals": [ "Any artifact source system purges data before the compilation SLA window expires.", "Evidence compilation job has not been tested in the past quarter.", "Evidence archive is not included in backup and DR coverage, creating a risk of evidence loss." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Evidence packaging for recovery operations is almost universally ad hoc. Organizations rarely automate compilation, apply cryptographic integrity protection, or enforce retention against a structured schema. This gap is most exposed during regulatory inquiries following major incidents." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise", "multi-tenant" ], "implementers": [ "Business Continuity", "GRC / Compliance", "Site Reliability Engineering" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a79.2", "fit": "direct", "rationale": "ISO 22301:2019 \u00a79.2 requires internal audits to verify conformity of the BCMS to the organization's own requirements. The recovery evidence package is the artifact that makes such audits possible by providing a structured, complete record of each recovery event. Without a compiled package, internal audits must reconstruct evidence from disparate sources, introducing gaps and inaccuracies.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 13(2)", "fit": "partial", "rationale": "EU DORA Article 13(2) (Learning and evolving) requires post-incident reviews after significant ICT disruptions, analysing causes and identifying improvements to ICT operations or the continuity policy. The recovery evidence package is the source record for those reviews; reporting of major incidents to authorities is governed separately by Articles 17-23.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_csf", "requirement_id": "ID.IM-04", "fit": "partial", "rationale": "NIST CSF 2.0 ID.IM-04 requires that incident response and other operational plans are communicated, maintained, and improved. The evidence package's summary report is the vehicle for communicating recovery lessons to stakeholders and driving those plan improvements.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.08", "fit": "direct", "rationale": "COBIT 2019 DSS04.08 (Conduct post-resumption review) requires assessing whether the continuity response was effective, with evidence that plans were followed during actual events (DSS04 has no .09 practice). The recovery evidence package is the primary artifact demonstrating that compliance.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.5", "fit": "partial", "rationale": "NIST SP 800-34 \u00a73.5 requires that contingency plan test results be documented and used to improve the plan. The evidence package extends this documentation requirement from planned exercises to actual recovery events, providing a richer and more operationally relevant basis for plan improvement than exercise results alone.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_27031", "requirement_id": "Cl. 11", "fit": "partial", "rationale": "ISO/IEC 27031:2025 Clause 11 covers testing, exercising and audit of ICT readiness, including review of performance after actual disruptions. The recovery evidence package supplies the documented record that such reviews and audits consume.", "normative_force": "voluntary-standard", "source_version": "2025", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "google_sre", "requirement_id": "Postmortem practices \u2014 Structured incident report with timeline, root cause, and remediation tracking", "fit": "direct", "rationale": "The Recovery Operations Evidence Package control requires compiling a structured artifact at the close of each AI system recovery event, aggregating outputs from RO-01 through RO-07 to demonstrate that all recovery obligations were met. Google SRE's postmortem practices define the canonical structure for post-incident documentation \u2014 including timeline reconstruction, contributing cause analysis, remediation action tracking, and stakeholder distribution \u2014 which maps directly to the eight artifact categories required in this control's evidence package. The SRE postmortem methodology's emphasis on blameless, systematic documentation and closed-loop remediation tracking informs both the content schema and the improvement-cycle requirements this control places on the evidence package output.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "aws_reliability", "requirement_id": "REL 13 \u2014 Document resiliency events and use outcomes to improve recovery capabilities", "fit": "partial", "rationale": "The Recovery Operations Evidence Package control aggregates recovery artifacts into a structured package that supports post-incident improvement, audit, and regulatory reporting for AI system recovery events. The AWS Well-Architected Reliability Pillar's REL 13 guidance specifies that recovery events should be documented and reviewed to drive future resiliency improvements, and the Testing reliability section provides documentation structure guidance that informs the evidence package schema. AWS recommends using CloudWatch, CloudTrail, and X-Ray artifacts as evidence components in post-incident documentation, supporting this control's integrated artifact model that spans infrastructure, application, and behavioral event sources.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "BCDR design guidance \u2014 Post-recovery documentation and compliance reporting requirements", "fit": "partial", "rationale": "The Recovery Operations Evidence Package control creates a structured compliance record for each AI system recovery event by aggregating artifacts from failover execution through RTO compliance verification. Microsoft Azure's BCDR design guidance requires compiling post-recovery documentation that demonstrates recovery procedures were followed and recovery objectives were met, supporting both internal governance and external regulatory reporting obligations. The Azure reliability documentation's guidance on post-incident reporting aligns with this control's structured artifact schema, and Azure Monitor, Activity Log, and Site Recovery dashboards provide the primary evidence sources for the infrastructure and recovery timeline artifact categories.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RO-08", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Within 24 hours of Sev-1 incident close and 72 hours of Sev-2 incident close, a cryptographically signed recovery evidence package containing all eight required artifact categories must be compiled, completeness-validated with zero missing categories, and archived in write-once long-term retention storage with a verifiable enterprise PKI signature chain.", "evidence_required": [ "compiled_evidence_package containing all eight artifact categories (failover execution log, model rollback record with validation results, data recovery log with integrity verification, communication log with notification timestamps, pre-restoration gate orchestrator results, complete audit trail extract, RTO/RPO compliance report, package metadata with incident summary and overall recovery verdict)", "package_completeness_validation_report confirming zero missing artifact categories and no package exceptions at the time of archival", "cryptographic_signature_record including the signing certificate chain and signature timestamp for the complete package", "retention_archive_metadata confirming write-once storage location, access control configuration denying write and delete to all parties including administrators, and retention period compliance" ], "machine_tests": [ "Trigger the evidence compilation job after a synthetic recovery event \u2192 assert all eight artifact categories are retrieved, the completeness check returns zero exceptions, and a signed package is archived within 24 hours of simulated incident close", "Verify the package signature using the enterprise PKI certificate against a sample of archived packages \u2192 assert signature_valid=true for 100% of sampled packages in the audit period", "Attempt to modify an archived evidence package using operational account credentials \u2192 assert 403 Forbidden response and a modification_attempt_blocked event in the security log", "Run the compilation job with one artifact source system unavailable \u2192 assert the completeness check immediately emits a missing_artifact_category alert rather than archiving an incomplete package" ], "human_review": [ "Review the evidence package schema to confirm all eight artifact categories are present, each maps to a corresponding RO control, and the mapping provides complete auditable traceability of the recovery lifecycle", "Assess custody separation to verify the evidence signing key is held by the security team rather than the engineering teams whose recovery actions are documented, preventing self-signing of evidence", "Verify the package archive is included in the organization's backup and DR coverage and that retention configuration satisfies applicable regulatory periods" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Compiling evidence packages manually from participant recollections and scattered tool outputs rather than via automated artifact retrieval pipelines from source systems", "Storing evidence packages in the same storage account with delete access available to the recovery teams whose actions are documented in the package", "Archiving evidence packages without cryptographic signing, leaving the organization unable to prove package integrity was maintained over the retention period during a regulatory inquiry", "Defining the package schema to accept narrative summaries rather than structured machine-readable artifacts, preventing automated completeness validation", "Treating evidence package compilation as optional for incidents below Sev-1, creating gaps in the regulatory compliance record for the full audit period" ], "update_status": "current", "layer_code": "RO" }, { "id": "FO-01", "layer": "FO", "plane": "control", "name": "Graceful Degradation Design Patterns", "plain": "AI systems must be architecturally designed to deliver reduced but functional service when one or more components fail, preserving core user value while clearly communicating degraded state rather than producing silent failures or complete outages.", "threat": { "tags": [ "service-degradation", "dependency-failure", "availability-loss", "silent-failure" ], "desc": "AI systems with hard dependencies on all components fail completely when any single dependency is unavailable. Without designed degradation paths, a partial infrastructure failure cascades into total service loss. Users receive no output or incorrect error states, and operators have no signal that degradation has occurred, masking the true blast radius of individual component failures." }, "standard": [ { "id": "google_sre", "section": "Ch. 22", "title": "Addressing Cascading Failures \u2014 Load Shedding and Graceful Degradation" }, { "id": "aws_reliability", "section": "REL 11", "title": "Design your workload to withstand component failures" }, { "id": "nist_800_160_v2", "section": "\u00a72.1.1", "title": "Cyber resiliency goals \u2014 Anticipate, Withstand, Recover, Adapt" }, { "id": "nist_csf", "section": "RC.RP", "title": "Recovery planning and execution" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/FO-01 Graceful Degradation Design Patterns control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/FO-01 Graceful Degradation Design Patterns control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Decompose AI service into core and enhanced capability tiers. Design each tier with an explicit degraded-mode contract that specifies reduced SLAs, disabled features, and user-visible status signals. Implement health-check-driven feature flags that automatically disable enhanced tiers when upstream health checks fail.", "steps": [ "Inventory all AI system capabilities and classify each as core (must function at all times) vs. enhanced (can be disabled under failure).", "Define explicit degraded-mode contracts for each capability tier, including reduced SLA targets, acceptable fallback outputs, and user notification requirements.", "Implement health-check-driven feature toggles that automatically gate enhanced capabilities behind upstream dependency health, tested in staging with fault injection.", "Establish observability instrumentation that surfaces degraded-mode activation events as distinct signals in monitoring dashboards and on-call alerting." ], "site_reliability": { "summary": "Graceful degradation is a reliability primitive. Every AI service dependency must have a documented degradation path that SREs can validate in production-like conditions.", "actions": [ "Map all upstream dependencies and define the degraded behavior the service adopts for each failure scenario.", "Implement synthetic canaries that verify degraded-mode behavior under simulated dependency failures during off-peak windows.", "Add degraded-mode activation counters to SLO burn-rate calculations." ], "failure_signals": [ "AI service returns 500 errors instead of degraded-mode responses when upstream latency exceeds threshold.", "Degraded mode is never observed in production despite known upstream instability.", "Feature flags are missing for non-core AI capabilities." ] }, "security_architect": { "summary": "Degraded modes must not introduce security regressions. Fallback paths that bypass authentication, authorization, or audit logging are unacceptable degraded states.", "actions": [ "Review all degraded-mode paths to confirm security controls remain active at reduced service levels.", "Prohibit degraded modes that disable rate limiting, authentication checks, or output filtering.", "Ensure degraded-mode outputs are clearly marked to prevent downstream consumers from treating stale or partial outputs as authoritative." ], "failure_signals": [ "Degraded mode bypasses input validation or output filtering.", "Cached degraded-mode responses lack staleness metadata.", "Authorization checks are omitted in fallback code paths." ] }, "grc_auditor": { "summary": "Degradation design must be documented, tested, and traceable to availability commitments in contracts and regulatory filings.", "actions": [ "Confirm documented degradation tiers exist for each AI service and are reviewed at least annually.", "Verify that degradation activation events are logged with timestamps and duration for post-incident review.", "Cross-reference degraded-mode SLAs against contractual availability commitments to confirm alignment." ], "metrics": [ "Percentage of AI services with documented degradation tiers: target 100%.", "Degradation event mean time to detect: target under 2 minutes.", "Services tested in degraded mode within past 90 days: target 100%." ], "failure_signals": [ "No degraded-mode test records for a service in the past quarter.", "Degradation events not logged or duration not captured.", "Degraded-mode SLA exceeds contractual availability floor." ] }, "business_continuity": { "summary": "Graceful degradation extends BCP recovery time objectives by enabling partial service continuity during failures rather than requiring full recovery before resuming operations.", "actions": [ "Map degradation tiers to BCP continuity objectives to confirm partial service satisfies minimum viable business function.", "Include degraded-mode operation in tabletop exercises so business stakeholders understand which capabilities are unavailable under failure scenarios.", "Document degraded-mode communication templates for notifying customers and partners during extended degradation events." ], "failure_signals": [ "BCP documentation treats AI service degradation as equivalent to full outage.", "No stakeholder communication templates exist for degraded-mode conditions.", "Business stakeholders are unaware of which features are unavailable in degraded mode." ] }, "it_operations": { "summary": "Operations keeps degradation tiers real: monitoring must show which tier each AI service is in, and runbooks must cover entering, operating in, and exiting degraded states.", "actions": [ "Expose current degradation tier as a monitored, alertable service state.", "Maintain runbooks for tier transitions, including manual overrides and exit criteria.", "Verify capacity and dependencies of each degraded tier during regular operational tests." ], "failure_signals": [ "No dashboard shows which degradation tier a service is operating in.", "Services enter degraded mode silently and never formally exit.", "Degraded tiers fail under load because their capacity was never validated." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most enterprises deploy AI components without documented degraded-mode contracts, relying on error handling rather than intentional degradation architecture." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "eu-high-risk-ai" ], "implementers": [ "Platform Engineering", "Site Reliability Engineering", "AI/ML Engineering" ], "frameworks": [ { "framework": "google_sre", "requirement_id": "Ch. 22", "fit": "direct", "rationale": "Google SRE (SRE Book, 2016) Chapter 22, Addressing Cascading Failures, defines load shedding and graceful degradation as core mechanisms for keeping services partially useful under failure and overload. FO-01's degradation tiers implement this pattern for AI capability stacks (Chapter 20 covers datacenter load balancing).", "normative_force": "best-practice", "reviewed_on": "2026-07-02", "source_version": "2016", "basis": "anchored", "relation": "informs" }, { "framework": "aws_reliability", "requirement_id": "REL 11", "fit": "direct", "rationale": "AWS Well-Architected Reliability Pillar REL 11 (Design your workload to withstand component failures) requires workloads to continue operating \u2014 possibly degraded \u2014 when components fail, including through graceful degradation. AI inference and orchestration services with designed degradation tiers implement this requirement.", "normative_force": "best-practice", "reviewed_on": "2026-07-02", "source_version": "2024", "basis": "anchored", "relation": "informs" }, { "framework": "nist_800_160_v2", "requirement_id": "\u00a72.1.1", "fit": "direct", "rationale": "NIST SP 800-160 Vol 2 Rev 1 \u00a72.1.1 codifies anticipate, withstand, recover, and adapt as the cyber resiliency goals for engineered systems. Graceful degradation is a primary mechanism by which AI systems satisfy the withstand goal, preserving essential function under adverse conditions.", "normative_force": "voluntary-standard", "reviewed_on": "2026-07-02", "source_version": "Rev 1", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "RC.RP-01 \u2014 Recovery Plan Execution", "fit": "partial", "rationale": "NIST CSF 2.0 Recover function requires organizations to restore systems and services after incidents. Graceful degradation reduces recovery scope by preserving partial functionality, directly shortening recovery timelines and reducing the business impact of AI service incidents. The partial fit reflects that CSF focuses on post-incident recovery while FO-01 addresses pre-incident design.", "normative_force": "voluntary-standard", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 11 \u2014 ICT Business Continuity Policy", "fit": "partial", "rationale": "EU DORA Article 11 requires financial entities to implement ICT business continuity policies covering response and recovery from ICT disruptions. Graceful degradation directly supports DORA continuity obligations by ensuring AI-dependent financial services can operate at reduced capacity during ICT disruptions rather than going fully offline. The partial fit reflects DORA's financial sector scope, while FO-01 applies universally.", "normative_force": "binding-law", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" } ], "canonical_id": "apeiris://resilience/controls/FO-01", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every AI system must activate a documented degraded-mode behavior profile within 30 seconds of detecting a core dependency failure, preserve core capability delivery at the defined reduced SLA, and emit a distinct degradation_activation operational event. No enhanced capability may serve uncommunicated stale or partial outputs as authoritative responses during an active degradation period.", "evidence_required": [ "capability_tier_inventory classifying each AI system capability as core or enhanced with documented degraded-mode SLA targets and fallback output type for each", "health_check_driven_feature_toggle_test_results confirming automatic degradation activation and correct output behavior under simulated upstream dependency failures in staging", "degradation_event_log with degradation_type, affected_capability, activation_timestamp, and duration fields for all degradation events observed in the audit period", "degraded_mode_security_review confirming authentication, authorization, rate limiting, and output filtering controls remain active on every fallback code path" ], "machine_tests": [ "Inject a synthetic failure for an upstream dependency in staging \u2192 assert the AI system activates the defined degraded mode within 30 seconds and a degradation_activation event appears in the operational metrics feed with affected_capability and trigger fields", "Submit a request to an enhanced capability during simulated dependency failure \u2192 assert the response communicates degraded state explicitly rather than returning a 500 error or a silent partial result", "Restore the upstream dependency in staging after a simulated failure \u2192 assert the AI system returns to full-capability mode within the documented recovery window and a degradation_end event is emitted" ], "human_review": [ "Review each degraded-mode contract to confirm security controls including authentication, authorization, and output filtering remain active on all fallback paths and no bypass is permitted under any degradation scenario", "Assess the capability tier inventory to verify the classification of core versus enhanced capabilities reflects current minimum viable business function requirements and has been reviewed by business owners", "Verify degraded-mode SLAs are aligned with contractual availability commitments and that communication templates for customers and partners covering degradation scenarios are current and approved" ], "blocking_effect": "advisory", "normative_status": "best-practice", "anti_patterns": [ "Treating any upstream dependency failure as equivalent to a full service outage rather than designing explicit degraded-mode behaviors for each identified dependency failure scenario", "Activating degraded modes silently without emitting a distinct operational event, masking degradation duration from SLO calculations and post-incident review", "Implementing degraded modes that bypass input validation, output filtering, or rate limiting on the grounds that the primary path controls are unavailable", "Serving cached responses in degraded mode without staleness metadata, allowing downstream consumers to treat stale AI outputs as current authoritative results", "Defining degraded-mode SLAs that exceed contractual availability floors, converting any degradation event into a contractual breach regardless of technical cause" ], "update_status": "current", "layer_code": "FO" }, { "id": "FO-02", "layer": "FO", "plane": "control", "name": "Circuit Breaker Implementation for AI Integrations", "plain": "Every AI system integration point with an external or downstream AI dependency must implement a circuit breaker that detects sustained failure or latency degradation and temporarily halts calls to the failing dependency, preventing resource exhaustion and cascade failure propagation.", "threat": { "tags": [ "cascade-failure", "resource-exhaustion", "timeout-storm", "dependency-unavailability" ], "desc": "Without circuit breakers, AI systems continue dispatching requests to a failing downstream service, accumulating in-flight requests that exhaust thread pools, connection pools, and memory. This resource exhaustion propagates upstream, causing the calling service to fail even though the fault originated downstream. Timeout storms compound the problem by locking resources for the full timeout duration before releasing them, amplifying the blast radius of a single dependency failure across the entire service mesh." }, "standard": [ { "id": "google_sre", "section": "Ch. 22", "title": "Addressing Cascading Failures" }, { "id": "aws_reliability", "section": "REL 5", "title": "Design interactions in a distributed system to mitigate or withstand failures" }, { "id": "dora", "section": "Art. 12", "title": "Backup policies, restoration and recovery procedures and methods" }, { "id": "microsoft_azure_resil", "section": "Circuit Breaker Pattern", "title": "Azure Architecture Center resilience patterns" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/FO-02 Circuit Breaker Implementation for AI Integrations control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/FO-02 Circuit Breaker Implementation for AI Integrations control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/FO-02 Circuit Breaker Implementation for AI Integrations control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Implement the circuit breaker pattern at every AI integration boundary using a three-state automaton (closed/open/half-open). Configure failure rate and latency thresholds that trigger the open state, a configurable sleep window before transitioning to half-open, and a probe request count to verify recovery before closing. Expose circuit state as a health signal integrated into operational dashboards.", "steps": [ "Identify all outbound AI integration points (LLM APIs, vector stores, embedding services, agent orchestrators) and wrap each with an independently tuned circuit breaker instance.", "Configure circuit breaker parameters per integration: failure rate threshold (e.g., >50% failures in a 10-second window), open-state sleep window (e.g., 30 seconds), and half-open probe request count (e.g., 3 successful probes required to close).", "Expose circuit state (closed/open/half-open) as a metric and health check endpoint, and integrate circuit-open events into on-call alerting pipelines with integration identity and first-failure timestamp.", "Implement circuit-open fallback behavior for each integration \u2014 cached response, degraded mode, or explicit user notification \u2014 never silent failure on circuit open." ], "site_reliability": { "summary": "Circuit breakers are the primary blast-radius containment mechanism for AI dependency failures. Each breaker must be independently observable and its open/close events traceable to a specific integration and failure reason.", "actions": [ "Instrument all circuit breakers with open/close transition counters, failure rate gauges, and half-open probe success/failure metrics per integration identity.", "Define per-integration SLO impact thresholds that correlate circuit-open events with error budget burn rate calculations.", "Conduct regular chaos experiments that induce circuit-open state and verify that fallback behavior activates correctly and non-affected integrations remain healthy." ], "failure_signals": [ "Circuit breakers exist in code but circuit state is not observable in monitoring.", "Circuit opens trigger no alerting and are discovered only during post-incident review.", "Fallback behavior on circuit open returns a 500 error instead of a graceful degraded response." ] }, "security_architect": { "summary": "Circuit breakers prevent resource exhaustion attacks where adversarial inputs induce repeated slow calls to a downstream AI service. Breaker configuration must be reviewed to prevent abuse as a denial-of-service amplifier or an oracle for internal service health.", "actions": [ "Review circuit breaker thresholds to ensure they cannot be trivially triggered by a small number of adversarial requests.", "Confirm that circuit state health endpoints are not accessible to unauthenticated callers.", "Ensure circuit-open events are logged with sufficient context (integration endpoint, failure reason, first-failure timestamp) to support incident investigation and forensic analysis." ], "failure_signals": [ "A single adversarial request can force a circuit to open state.", "Circuit state health endpoints expose internal topology to unauthenticated callers.", "Circuit transition events are absent from the security information event log." ] }, "it_operations": { "summary": "Operations teams must have the visibility and runbooks to interpret circuit state, manually reset a stuck breaker, and escalate persistent open states without requiring developer involvement.", "actions": [ "Create runbooks for each AI integration circuit breaker covering how to read circuit state, when to manually reset, and the escalation path for persistent failures.", "Add circuit breaker state to operational dashboards alongside dependency health indicators and error rate trends.", "Test manual circuit reset procedures in non-production environments at least quarterly." ], "failure_signals": [ "No operational runbook exists for circuit breaker incidents.", "Operations staff cannot identify which circuit breakers are open without developer assistance.", "Manual reset procedures have never been tested in a controlled environment." ] }, "grc_auditor": { "summary": "Circuit breaker implementation demonstrates that the organization has engineered controls to prevent AI dependency failures from propagating into broader service outages, satisfying continuity and operational resilience requirements.", "actions": [ "Verify that all external AI integration points are covered by circuit breakers and that the inventory of integration points is current.", "Request evidence that circuit breaker configurations have been reviewed and approved by engineering leads within the past year.", "Sample circuit breaker event logs to confirm open/close transitions are recorded with required metadata and are retained for the audit period." ], "metrics": [ "Percentage of AI integration points covered by circuit breakers: target 100%.", "Circuit breaker configuration review frequency: at minimum annually.", "Mean time from circuit open to alert acknowledgment: target under 5 minutes." ], "failure_signals": [ "AI integration points in production without circuit breaker coverage.", "Circuit breaker configurations not reviewed in over 12 months.", "No circuit breaker event logs retained for post-incident analysis." ] }, "business_continuity": { "summary": "Circuit breakers determine what the business experiences when an AI dependency fails: a controlled fallback or a hard stop. Ensure breaker behavior maps to documented business continuity expectations.", "actions": [ "Map each breaker's open-state behavior to a documented business fallback (queue, manual process, cached response).", "Include breaker-open scenarios in continuity exercises for AI-dependent processes.", "Set breaker thresholds with business tolerance for latency and error rates, not just technical defaults." ], "failure_signals": [ "A breaker opens and the dependent business process simply halts with no fallback.", "Business owners are unaware which processes sit behind which breakers.", "Breaker thresholds were never reviewed against business impact tolerances." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Most AI integration layers rely on retry logic alone; circuit breakers are rarely implemented at the AI-specific integration layer even when used at the general HTTP client level." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "federated-enterprise", "eu-high-risk-ai" ], "implementers": [ "Platform Engineering", "Site Reliability Engineering", "AI/ML Engineering" ], "frameworks": [ { "framework": "google_sre", "requirement_id": "Ch. 22", "fit": "direct", "rationale": "Google SRE (SRE Book, 2016) Chapter 22, Addressing Cascading Failures, covers preventing overload-driven cascades with techniques such as load shedding, retry budgets, and deadline propagation (per-service overload handling is Chapter 21). Circuit breakers for AI dependencies implement the same cascade-prevention discipline.", "normative_force": "best-practice", "reviewed_on": "2026-07-02", "source_version": "2016", "basis": "anchored", "relation": "informs" }, { "framework": "aws_reliability", "requirement_id": "REL 5", "fit": "direct", "rationale": "AWS Well-Architected Reliability Pillar REL 5 (Design interactions in a distributed system to mitigate or withstand failures) prescribes graceful degradation, throttling, retry limits with backoff, and fail-fast patterns. Per-dependency circuit breakers are a canonical implementation for AI integrations.", "normative_force": "best-practice", "reviewed_on": "2026-07-02", "source_version": "2024", "basis": "anchored", "relation": "informs" }, { "framework": "dora", "requirement_id": "Art. 12", "fit": "partial", "rationale": "EU DORA Article 12 requires backup policies and restoration and recovery procedures and methods that limit downtime and disruption. Circuit breakers contain a failing AI dependency so restoration operates on an isolated fault rather than a propagated outage, supporting the recovery objectives Article 12 protects (business continuity policy itself is Article 11).", "normative_force": "binding-law", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "microsoft_azure_resil", "requirement_id": "Circuit Breaker Pattern \u2014 Azure Architecture Center", "fit": "direct", "rationale": "Microsoft Azure Architecture Center's Circuit Breaker pattern documentation provides canonical implementation guidance for the three-state automaton used in FO-02, covering threshold configuration, state transition logic, and integration with Azure API Management and Service Bus. The pattern is directly applicable to AI service integrations on any cloud platform and provides implementation blueprints for the control's required behavior.", "normative_force": "best-practice", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "nist_800_160_v2", "requirement_id": "\u00a72.1.3", "fit": "partial", "rationale": "NIST SP 800-160 Vol 2 Rev 1 \u00a72.1.3 includes Segmentation among its cyber resiliency techniques, and Appendix D's structural design principle 'Contain and exclude behaviors' directs systems to limit the spread of failure effects. Circuit breakers are a concrete containment mechanism that keeps a failing AI dependency from propagating.", "normative_force": "voluntary-standard", "reviewed_on": "2026-07-02", "source_version": "Rev 1", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://resilience/controls/FO-02", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every external and downstream AI dependency integration point must be wrapped with an independently configured circuit breaker that transitions to open state within 30 seconds of sustained failure at or above the configured failure rate threshold, emits a circuit_open event to the operational monitoring system with integration_id and failure_reason, and activates the documented fallback behavior without returning a 500 error to the calling service.", "evidence_required": [ "ai_integration_inventory listing all outbound AI integration points with assigned circuit breaker instance identifier, per-integration failure rate threshold, sleep window, and half-open probe count configuration", "circuit_state_metric_timeseries showing open, closed, and half-open state per integration over the audit period, retained in the operational monitoring system", "circuit_transition_event_log with integration_id, failure_reason, first_failure_timestamp, state_transition, and fallback_activated fields for all open and close events in the audit period", "chaos_experiment_report confirming circuit-open activation, correct fallback behavior, and non-500 caller response under injected dependency failures for each integration within the past 90 days" ], "machine_tests": [ "Force the failure rate of a downstream AI integration above the configured threshold in staging \u2192 assert the circuit transitions to open state within 30 seconds and a circuit_open metric event is emitted with integration_id and failure_reason fields", "Submit a request to a circuit-open integration \u2192 assert the fallback behavior activates and the caller receives a non-500 response with a structured error containing error_code=integration_unavailable or a cached degraded output", "Wait for the circuit sleep window to expire in staging \u2192 assert the circuit transitions to half-open, emits probe requests, and closes after the required number of successful probes with a circuit_closed event", "Query the circuit state health endpoint without authentication credentials \u2192 assert 401 Unauthorized response with no internal topology information in the response body" ], "human_review": [ "Review the AI integration inventory to confirm all outbound integration points have circuit breaker coverage and that threshold configurations have been reviewed and approved by engineering leads within the past year", "Assess operational runbooks for circuit breaker incidents to verify operations staff can identify circuit state, perform manual reset, and escalate persistent failures without requiring developer involvement", "Examine chaos experiment records to confirm fallback behavior for each integration has been validated in production-like conditions and that no integration has gone untested for more than 90 days" ], "blocking_effect": "advisory", "normative_status": "binding-law", "anti_patterns": [ "Using retry-with-exponential-backoff as a substitute for circuit breakers, which accumulates in-flight requests against a failing dependency and accelerates resource exhaustion during sustained failures", "Configuring a single global circuit breaker across all AI integrations rather than per-integration instances with independently tuned failure thresholds appropriate to each dependency's SLA", "Exposing circuit state health endpoints to unauthenticated callers, revealing internal service topology and real-time failure rates to external parties", "Implementing circuit-open fallback that returns a 500 error to the calling service rather than a graceful degraded response, propagating the cascade failure upstream", "Deploying circuit breakers without observable state metrics or event logging, preventing operations staff from identifying open circuits without developer-level system access" ], "update_status": "current", "layer_code": "FO" }, { "id": "FO-03", "layer": "FO", "plane": "lifecycle", "name": "Fallback Behavior Governance", "plain": "Every AI system capability must have explicitly defined, formally approved, and regularly tested fallback behaviors that activate when the primary AI capability is degraded or unavailable. Fallback behaviors must be governed through a formal review cycle to ensure they remain appropriate, safe, and compliant as the system and its regulatory environment evolve.", "threat": { "tags": [ "ungoverned-fallback", "silent-degradation", "compliance-gap", "stale-fallback" ], "desc": "Without formal governance, fallback behaviors are coded ad hoc and never reviewed for correctness, safety, or compliance. A fallback that was acceptable at initial deployment may become a compliance violation when regulatory requirements change. Silent degradation \u2014 where a fallback activates without any user or operator notification \u2014 masks service quality erosion and prevents timely remediation, creating sustained compliance exposure that is discoverable only after an audit or incident." }, "standard": [ { "id": "dora", "section": "Art. 11", "title": "ICT Business Continuity Policy \u2014 fallback procedure requirements" }, { "id": "iso_22301", "section": "\u00a78.4.4", "title": "Business continuity plans and procedures" }, { "id": "nist_sp800_34", "section": "\u00a73.5", "title": "Plan testing, training, and exercises" }, { "id": "google_sre", "section": "Ch. 22", "title": "Addressing Cascading Failures \u2014 graceful degradation" } ], "sources": [ { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/FO-03 Fallback Behavior Governance control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "For each AI capability, define a fallback behavior contract that specifies: trigger conditions, fallback output type (cached/static/rule-based/human-escalation), user notification requirement, maximum fallback duration before escalation, and governance approval status. Maintain fallback contracts in version control and review annually or after any material system change.", "steps": [ "Enumerate all AI capabilities and for each define a fallback behavior contract covering: trigger condition, output type, user and operator notification, maximum acceptable fallback duration, and compliance constraints under applicable regulations.", "Obtain formal approval of each fallback contract from the system owner, AI risk lead, and (for regulated entities) compliance officer before the capability is promoted to production.", "Implement fallback activation as an observable, logged event with a distinct event type that feeds into operational dashboards and SLO monitoring, emitting at minimum: capability ID, trigger condition, activation timestamp, and fallback type.", "Schedule annual fallback contract reviews and require re-approval after any change to the primary AI capability, applicable regulatory requirements, or business process the AI capability supports." ], "site_reliability": { "summary": "Fallback contracts define the reliability floor for each AI capability. SREs must be able to trigger, observe, and measure fallback activation in production-like environments to validate that the contract is implemented correctly.", "actions": [ "Build test harnesses that inject the trigger conditions defined in each fallback contract and assert that the correct fallback output is produced within the contract's notification and timing requirements.", "Add fallback activation rate to SLO calculations as a leading indicator of primary AI capability health degradation.", "Document fallback duration limits in runbooks and automate escalation when fallback duration exceeds the contractual maximum without primary capability recovery." ], "failure_signals": [ "Fallback behavior activates but no log entry or operational metric is emitted.", "Fallback has been active longer than the contract maximum without triggering escalation.", "Fallback trigger conditions cannot be injected in a non-production test environment." ] }, "security_architect": { "summary": "Fallback behaviors must not create security regressions. Any fallback path that bypasses input validation, output filtering, or audit logging introduces a security vulnerability that may be exploitable by deliberately inducing fallback activation.", "actions": [ "Review all fallback contracts to confirm security controls are maintained at the same level as primary-path controls, with no bypass of input validation, output filtering, or access controls.", "Specifically prohibit fallback paths that serve stale outputs without staleness metadata or that omit audit log entries.", "Include fallback trigger conditions in the AI system's threat model as a potential attack surface where adversaries may attempt to induce fallback to bypass primary-path controls." ], "failure_signals": [ "Fallback output bypasses toxicity or content filtering controls active on the primary path.", "Fallback code paths do not emit audit log entries.", "Threat model has not analyzed fallback trigger conditions as an adversarial attack surface." ] }, "grc_auditor": { "summary": "Fallback behavior governance provides the audit trail demonstrating that degraded-mode operations are controlled, reviewed, and compliant with regulatory continuity requirements across applicable frameworks.", "actions": [ "Request the fallback contract register and verify each AI capability has a current, formally approved fallback contract with documented governance approval chain.", "Sample fallback activation logs from the past audit period to confirm activations were logged with required metadata and that activation durations were within contractual limits.", "Verify fallback contracts were reviewed after any regulatory changes affecting the AI capability's compliance obligations." ], "metrics": [ "Percentage of AI capabilities with current approved fallback contracts: target 100%.", "Fallback contract review cycle compliance: target 100% reviewed within 12 months.", "Fallback activations without log entries in the audit period: target 0." ], "failure_signals": [ "AI capabilities in production without approved fallback contracts.", "Fallback contracts not reviewed following a material regulatory change.", "Fallback activations are discoverable only via application error logs rather than structured operational events." ] }, "business_continuity": { "summary": "Fallback contracts are the operational continuity specification for AI capabilities. BCM teams must incorporate fallback behavior into continuity plans and communicate fallback-state service levels to business stakeholders so they can make informed decisions during AI service degradation events.", "actions": [ "Integrate AI fallback contracts into the organization's BCP to define minimum viable service levels for each AI-dependent business process during capability degradation.", "Develop business-facing communication templates for each fallback scenario, explaining which AI-dependent processes are affected and at what reduced capacity.", "Include fallback scenario exercises in annual BCP tests to validate that business units can operate within fallback-state service levels and that stakeholder communication procedures work as documented." ], "failure_signals": [ "BCP documentation does not account for AI capability fallback states as distinct from full outage.", "Business stakeholders are unaware that service levels change when AI systems enter fallback mode.", "No stakeholder communication templates exist for AI fallback scenarios." ] }, "it_operations": { "summary": "Operations enforces fallback governance day to day: activations must be visible, logged, and reversed when the primary path recovers \u2014 silent permanent fallback is an outage in disguise.", "actions": [ "Alert on every fallback activation and track time-in-fallback as an operational metric.", "Follow runbooks for fallback exit and primary-path restoration with validation steps.", "Test fallback paths on schedule so contracts reflect behavior that actually works." ], "failure_signals": [ "Services run in fallback for days before anyone notices.", "Fallback exit is ad hoc, with no validation that primary is healthy.", "Documented fallback behavior diverges from what the code actually does." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Fallback behavior is commonly implemented informally in error handlers without formal governance; regulatory pressure from EU DORA and sector-specific AI rules is driving formalization of fallback contracts." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "AI/ML Engineering", "Risk & Compliance", "Business Continuity Management" ], "frameworks": [ { "framework": "dora", "requirement_id": "Art. 11 \u2014 ICT Business Continuity Policy", "fit": "direct", "rationale": "EU DORA Article 11 requires financial entities to implement ICT business continuity policies that explicitly include fallback procedures for ICT failures. FO-03 directly implements the fallback procedure governance requirements of DORA by mandating documented, approved, and tested fallback contracts for each AI capability. For financial entities, this control is binding and subject to supervisory audit.", "normative_force": "binding-law", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "iso_22301", "requirement_id": "\u00a78.4.4", "fit": "direct", "rationale": "ISO 22301:2019 \u00a78.4.4 requires documented business continuity plans and procedures for responding to disruptions, including alternative ways of delivering outputs. Governed fallback contracts are those documented alternative arrangements for AI-driven functions (\u00a78.4.3 covers warning and communication).", "normative_force": "certification-standard", "reviewed_on": "2026-07-02", "source_version": "2019", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.5", "fit": "partial", "rationale": "NIST SP 800-34 Rev 1 \u00a73.5 requires contingency plans to be exercised so that gaps are identified and recovery behavior is confirmed. Fallback behavior testing is the AI-system-specific application of that testing requirement (\u00a73.4 covers recovery strategies, not testing).", "normative_force": "voluntary-standard", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "google_sre", "requirement_id": "Ch. 22", "fit": "direct", "rationale": "Google SRE (SRE Book, 2016) Chapter 22 treats graceful degradation \u2014 serving reduced results under failure \u2014 as an engineered, explicitly specified behavior rather than an accident. FO-03's fallback contracts formalize exactly that specification (trigger conditions, output guarantees, notification) for AI services.", "normative_force": "best-practice", "reviewed_on": "2026-07-02", "source_version": "2016", "basis": "anchored", "relation": "informs" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.05 \u2014 Review, Maintain, and Improve Continuity Plans", "fit": "partial", "rationale": "COBIT 2019 DSS04.05 requires organizations to review and improve continuity plans regularly to reflect changes in the operating environment. The annual fallback contract review cycle mandated by FO-03 directly implements DSS04.05's continuous improvement requirement for continuity procedures applied to AI capabilities. The partial fit reflects that COBIT DSS04 focuses on organizational continuity plans broadly, while FO-03 specifically governs AI capability fallback behaviors.", "normative_force": "industry-framework", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" } ], "canonical_id": "apeiris://resilience/controls/FO-03", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every AI capability in production must have a formally approved, version-controlled fallback contract specifying trigger conditions, output type, notification requirements, and maximum fallback duration before escalation. Every fallback activation must produce a structured operational event with all required fields, and contracts must be reviewed and re-approved within 12 months or after any material change to the capability or its regulatory environment.", "evidence_required": [ "fallback_contract_register with version history, approval chain documenting system owner, AI risk lead, and compliance officer approvals, and current approval date for each production AI capability", "fallback_activation_event_log showing structured events with capability_id, trigger_condition, activation_timestamp, fallback_type, and duration fields for all activations in the audit period with zero activations lacking a corresponding log entry", "annual_review_completion_records confirming all fallback contracts were reviewed within the 12-month cycle with documented reviewer and outcome for each contract", "regulatory_change_review_records documenting assessment of regulatory changes against each affected fallback contract and confirming re-approval was obtained following material changes" ], "machine_tests": [ "Inject each trigger condition defined in each fallback contract in a non-production test environment \u2192 assert the correct fallback output type is produced and a structured activation event is emitted with capability_id, trigger_condition, activation_timestamp, and fallback_type fields within 5 seconds", "Simulate fallback duration exceeding the contractual maximum for a capability without primary capability recovery \u2192 assert an automated escalation alert is triggered and routed to the system owner within the defined escalation window", "Submit a request through a primary AI capability path that activates fallback \u2192 assert the caller receives a user notification consistent with the notification requirement documented in the fallback contract" ], "human_review": [ "Review the fallback contract register to confirm every production AI capability has a current approved contract with documented approval chain including compliance officer sign-off and that no capability is in production without a valid contract", "Assess a sample of fallback contracts to verify trigger conditions are specific and testable, output types comply with applicable content and safety policies, and maximum durations align with business continuity objectives", "Verify that fallback contract review records include documentation of regulatory changes considered during the review cycle and confirm re-approval was obtained following material changes to applicable requirements" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Implementing fallback behavior in error handlers without a corresponding formal fallback contract, leaving fallback logic ungoverned, untested, and invisible to compliance review", "Using a single generic fallback policy across all AI capabilities rather than per-capability contracts that reflect the specific compliance and business continuity requirements of each capability", "Treating fallback activation as an invisible implementation detail without a distinct operational event, preventing duration tracking, SLO calculation, and automatic escalation on breach", "Approving fallback contracts once at initial deployment and never re-reviewing them, allowing contracts to become stale as regulatory requirements or business processes evolve", "Omitting security control verification from fallback contract review, permitting deployment of fallback behaviors that bypass input validation or output filtering active on the primary capability path" ], "update_status": "current", "layer_code": "FO" }, { "id": "FO-04", "layer": "FO", "plane": "control", "name": "Input Validation and Malformed Request Handling", "plain": "AI systems must validate, sanitize, and gracefully handle all inputs \u2014 including malformed, oversized, adversarially crafted, and unexpected-type inputs \u2014 without crashing, leaking exception details, or producing unsafe outputs. Input validation must cover both the AI model interface and all upstream APIs and orchestration layers.", "threat": { "tags": [ "malformed-input", "adversarial-prompt-injection", "exception-leakage", "denial-of-service" ], "desc": "Malformed or adversarially crafted inputs can trigger unhandled exceptions that expose internal stack traces, crash AI inference workers, or cause memory exhaustion. Oversized context windows submitted to LLM endpoints exhaust compute budgets and block legitimate requests. Adversarial prompt injection through malformed inputs can manipulate AI system behavior in ways not captured by standard error handling, bypassing safety controls and producing policy-violating outputs without raising operational alerts." }, "standard": [ { "id": "nist_800_160_v2", "section": "\u00a72.1.3", "title": "Substantiated Integrity (cyber resiliency technique)" }, { "id": "cis_controls_v8", "section": "Control 16", "title": "Application Software Security \u2014 input validation requirements" } ], "sources": [ { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/FO-04 Input Validation and Malformed Request Handling control.", "reviewed_on": "2026-07-02" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/FO-04 Input Validation and Malformed Request Handling control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Implement a layered input validation pipeline at the AI system boundary: (1) structural validation (schema, type, length), (2) semantic validation (content policy, prompt injection detection), (3) resource budget enforcement (token limits, rate limits), (4) graceful rejection with non-leaking error responses. Log all validation failures with sufficient context for threat analysis without exposing sensitive internal implementation details.", "steps": [ "Define and enforce input schemas for all AI API endpoints, including maximum token/character limits, required fields, and permitted content types; return structured, non-leaking error responses for all schema violations.", "Implement prompt injection detection at the AI gateway layer using pattern matching and semantic analysis; quarantine suspicious inputs for review and return a safe rejection response without processing the input through the AI model.", "Enforce per-request resource budgets (token limits, compute timeouts) with graceful rejection at the API layer before dispatching to the AI inference engine, preventing single requests from exhausting shared inference capacity.", "Configure all exception handlers in the AI inference and orchestration layer to return sanitized error responses containing no stack traces, internal hostnames, model identifiers, or configuration details that could aid an attacker." ], "site_reliability": { "summary": "Input validation failures are both a reliability and security signal. High volumes of malformed inputs may indicate an attack or a misconfigured upstream client and should trigger automated investigation.", "actions": [ "Instrument the input validation pipeline with counters for each validation failure type (schema, length, injection detection, budget) and expose these as distinct operational metrics.", "Set alert thresholds on input validation failure rates; sudden spikes warrant on-call investigation as potential attack or upstream misconfiguration indicators.", "Regularly test oversized and structurally malformed inputs in load testing to confirm they are rejected gracefully without degrading throughput for concurrent legitimate requests." ], "failure_signals": [ "Malformed input causes AI inference worker crash requiring manual restart.", "Exception responses include stack traces visible to the calling client.", "Single oversized requests noticeably degrade throughput for concurrent legitimate requests." ] }, "security_architect": { "summary": "Input validation is the first line of defense against prompt injection, jailbreaking, and denial-of-service via resource exhaustion. The validation pipeline must be designed as a security boundary with defense-in-depth, not merely a data quality layer.", "actions": [ "Design the input validation pipeline with ordered defense layers: structural checks first, then semantic/content checks, then resource budget enforcement to ensure cheap checks precede expensive ones.", "Conduct adversarial testing of the prompt injection detection layer quarterly using current published injection techniques against the deployed model and gateway configuration.", "Ensure all validation rejection responses are uniform in format, latency, and content to prevent oracle attacks that enumerate internal validation rules through differential response analysis." ], "failure_signals": [ "Prompt injection detection has not been tested against current published attack patterns in the past 90 days.", "Error responses differ by failure type in ways that reveal internal validation logic to external callers.", "Resource budget enforcement can be bypassed by splitting oversized requests across multiple API calls within a session." ] }, "it_operations": { "summary": "Operations teams must be able to investigate and escalate input validation anomalies. Runbooks must cover how to identify attack patterns in validation failure logs and when to escalate to security operations.", "actions": [ "Create runbooks covering how to query input validation failure logs, identify attack patterns, and escalate to the security operations center when thresholds are exceeded.", "Establish validation failure rate thresholds that trigger automatic ticket creation in the IT operations queue for investigation.", "Test the ability to block specific source IPs or API keys at the AI gateway layer in response to active validation failure attack patterns without requiring code deployment." ], "failure_signals": [ "No operational runbook exists for responding to input validation failure rate spikes.", "Operations cannot block a specific source at the AI gateway without developer intervention.", "Validation failure logs are inaccessible to operations without developer-level access credentials." ] }, "grc_auditor": { "summary": "Input validation demonstrates that the organization has implemented controls against adversarial inputs and resource abuse, satisfying AI security and operational resilience requirements across multiple applicable frameworks.", "actions": [ "Verify that input validation schemas are documented for all AI API endpoints and have been reviewed within the past year.", "Request evidence that prompt injection detection coverage has been tested against current OWASP LLM Top 10 attack techniques within the past 90 days.", "Review validation failure log retention policies to confirm logs are retained for the audit period required by applicable regulations." ], "metrics": [ "Percentage of AI API endpoints with documented input validation schemas: target 100%.", "Prompt injection test coverage against current OWASP LLM Top 10: target 100%.", "Input validation failure log retention: target meets or exceeds applicable regulatory minimum." ], "failure_signals": [ "AI endpoints in production without documented input validation schemas.", "Prompt injection detection not tested against current attack techniques in the past 90 days.", "Validation failure logs not retained for the required audit period." ] }, "business_continuity": { "summary": "Malformed and malicious input handling protects business continuity: an AI system that crashes or misbehaves on bad input takes its business function down with it. Ensure input-failure modes have defined business responses.", "actions": [ "Document the business-visible behavior when inputs are rejected (error messages, queues, manual review paths).", "Include input-storm and malformed-batch scenarios in continuity considerations for AI-driven processes.", "Confirm rejected-input volumes are monitored so process owners see when a workflow is being starved." ], "failure_signals": [ "Legitimate work silently disappears because inputs are rejected with no fallback path.", "An input storm takes down an AI service and the business function has no manual alternative.", "Process owners have no visibility into rejection rates affecting their workflows." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "AI-specific input validation \u2014 particularly prompt injection detection \u2014 is a nascent practice; most organizations apply traditional API input validation without addressing LLM-specific attack surfaces like indirect prompt injection and context manipulation." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "eu-high-risk-ai", "multi-tenant" ], "implementers": [ "AI/ML Engineering", "Security Engineering", "Platform Engineering" ], "frameworks": [ { "framework": "nist_800_160_v2", "requirement_id": "\u00a72.1.3", "fit": "direct", "rationale": "NIST SP 800-160 Vol 2 Rev 1 \u00a72.1.3 includes Substantiated Integrity \u2014 ascertaining whether critical elements and inputs have been corrupted \u2014 among its cyber resiliency techniques. Validating and constraining inputs before they reach AI model contexts applies this technique at the system boundary.", "normative_force": "voluntary-standard", "reviewed_on": "2026-07-02", "source_version": "Rev 1", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cis_controls_v8", "requirement_id": "Control 16 \u2014 Application Software Security", "fit": "partial", "rationale": "CIS Controls v8 Control 16 requires organizations to validate and sanitize all application inputs to prevent injection and other input-based attacks. While written before widespread LLM deployment, the fundamental input validation requirements in CIS Control 16 apply directly to AI API endpoints and form the basis for the structural validation layer in FO-04's pipeline. The partial fit reflects that CIS Control 16 does not specifically address prompt injection as an AI-specific attack vector.", "normative_force": "certification-standard", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 9 \u2014 ICT Security Controls", "fit": "partial", "rationale": "EU DORA Article 9 requires financial entities to implement ICT security controls that prevent disruption to ICT services, encompassing input validation as a control against both denial-of-service via resource exhaustion and adversarial manipulation of AI-dependent financial services. The partial fit reflects that DORA does not specifically address AI input validation but the requirement applies by extension to AI systems handling financial operations.", "normative_force": "binding-law", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" } ], "canonical_id": "apeiris://resilience/controls/FO-04", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "The AI system must reject malformed, oversized, and prompt-injection-suspect inputs at the API gateway layer with a structured non-leaking error response before dispatching to the AI inference engine. Every exception handler in the inference and orchestration layer must return a sanitized response containing no stack traces, internal hostnames, model identifiers, or configuration details visible to the calling client.", "evidence_required": [ "input_validation_schema_documentation for all AI API endpoints specifying field types, maximum token and character limits, required fields, and content policy constraints, reviewed within the past year", "validation_failure_metric_report with per-validation-layer failure counts by type (schema, length, injection_detected, budget_exceeded) over the audit period showing rejection occurs at the gateway before inference dispatch", "prompt_injection_test_results confirming coverage of current OWASP LLM Top 10 injection techniques against the deployed model and gateway configuration, conducted within the past 90 days", "exception_response_audit_log or penetration test report confirming all exception responses returned to callers contain no stack traces, internal hostnames, model identifiers, or configuration details" ], "machine_tests": [ "Submit a request exceeding the defined token limit to the AI API gateway \u2192 assert 400 or 429 response with error_code=token_limit_exceeded is returned before inference dispatch and no internal implementation details appear in the response body", "Submit a request containing a known prompt injection pattern including indirect injection via simulated tool output format \u2192 assert the request is rejected or quarantined before reaching the AI inference engine and a validation_failure event with validation_type=injection_detected is emitted", "Trigger an unhandled exception in the AI inference layer using a structurally valid but semantically unexpected input \u2192 assert the error response contains a generic error message with no stack trace, hostname, model identifier, or configuration detail", "Submit concurrent oversized requests at the maximum defined concurrency limit \u2192 assert each is rejected within 100ms and throughput degradation for concurrent legitimate requests remains below 5%" ], "human_review": [ "Review the input validation pipeline architecture to confirm ordered defense layers (structural validation first, then semantic and injection detection, then resource budget enforcement) ensure cheap checks precede expensive ones, preventing denial-of-service via expensive validation", "Assess prompt injection detection coverage against current published attack techniques including indirect injection via retrieved context, tool outputs, and multi-turn conversation state manipulation", "Verify all exception handlers across the inference and orchestration layer have been audited to confirm sanitized responses within the past year or following any inference infrastructure change" ], "blocking_effect": "blocks-runtime-action", "normative_status": "voluntary-standard", "anti_patterns": [ "Applying only traditional API schema validation without addressing LLM-specific attack vectors such as indirect prompt injection through retrieved context, tool call outputs, or system prompt manipulation", "Configuring exception handlers to propagate raw stack traces or internal error messages to API callers, revealing implementation details that aid attacker reconnaissance and enumeration", "Enforcing token limits only at the AI inference layer after the request has been dispatched, allowing individual oversized requests to consume inference capacity before rejection", "Using differential error response latency or content for different validation failure types, creating a timing or content oracle that distinguishes validation failure reasons from the outside", "Implementing prompt injection detection as a one-time configuration without regular testing against current published attack techniques, accumulating detection coverage gaps as new injection methods are discovered" ], "update_status": "current", "layer_code": "FO" }, { "id": "FO-05", "layer": "FO", "plane": "control", "name": "Cascade Failure Prevention in Multi-Agent Pipelines", "plain": "Multi-agent AI orchestration pipelines must implement isolation boundaries, failure propagation limits, and independent fallback behaviors for each agent node so that a failure in one agent does not cause failures throughout the entire pipeline or in adjacent pipelines sharing orchestration infrastructure.", "threat": { "tags": [ "cascade-failure", "agent-chain-collapse", "blast-radius-amplification", "retry-storm" ], "desc": "In multi-agent pipelines, a single agent failure commonly propagates upstream as each orchestrator waits for a response that never arrives, blocking execution threads and eventually exhausting the orchestration layer's concurrency budget. Without isolation boundaries, a failing specialist agent can collapse the entire pipeline and starve unrelated pipelines competing for the same orchestration infrastructure. Retry storms from multiple orchestrators simultaneously retrying the same failed agent amplify the blast radius far beyond the scope of the original failure." }, "standard": [ { "id": "google_sre", "section": "Ch. 22", "title": "Addressing Cascading Failures" }, { "id": "aws_reliability", "section": "REL 5", "title": "Design interactions in a distributed system to mitigate or withstand failures" }, { "id": "nist_800_160_v2", "section": "\u00a72.1.3", "title": "Segmentation (cyber resiliency technique)" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/FO-05 Cascade Failure Prevention in Multi-Agent Pipelines control.", "reviewed_on": "2026-07-01" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/FO-05 Cascade Failure Prevention in Multi-Agent Pipelines control.", "reviewed_on": "2026-07-02" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/FO-05 Cascade Failure Prevention in Multi-Agent Pipelines control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Implement bulkhead isolation between agent nodes: each agent operates with independently bounded resource pools (thread pool, connection pool, timeout budget) that do not share capacity with other agents. Implement an agent-level circuit breaker at each pipeline step. Design orchestrators to produce partial results when downstream agents fail rather than propagating failure upstream. Impose per-pipeline and per-agent concurrency limits with exponential backoff and jitter on all retry logic.", "steps": [ "Inventory all multi-agent pipeline topologies and map the failure propagation path for each agent node failure, identifying which downstream agents are blocked and which pipeline paths remain unaffected by each failure scenario.", "Implement bulkhead isolation for each agent node: assign independently bounded thread pools, connection pools, and compute quotas so that one agent exhausting its quota does not starve adjacent agents or unrelated pipelines.", "Implement agent-level circuit breakers at each pipeline step with independent thresholds and fallback behaviors; configure orchestrators to record partial results and gracefully terminate rather than retrying indefinitely.", "Impose per-pipeline maximum retry budgets with exponential backoff and jitter to prevent retry storms when multiple pipeline instances simultaneously encounter the same failing agent node." ], "site_reliability": { "summary": "Multi-agent pipeline failures are among the most complex failure modes in AI systems. SREs must be able to identify the originating failure node, trace propagation paths, and contain blast radius during active incidents without having to examine individual agent logs serially.", "actions": [ "Instrument each agent node with independent health, latency, error rate, and queue depth metrics to enable precise identification of the originating failure node during live incidents.", "Define pipeline-level SLOs that distinguish partial completion (pipeline produces a degraded result) from complete failure (pipeline produces no result) to track containment effectiveness.", "Conduct quarterly chaos experiments that inject failures at each agent node position in each pipeline topology and verify that containment works as designed for that specific topology." ], "failure_signals": [ "A single agent node failure causes the entire orchestration layer to become unhealthy.", "Failure origination cannot be identified without manually inspecting individual agent log streams.", "Pipeline concurrency rises unboundedly under retry storms as orchestrators exhaust shared concurrency budget." ] }, "security_architect": { "summary": "Multi-agent pipeline failures can be deliberately induced to exhaust orchestration resources, disrupt unrelated pipelines sharing infrastructure, or force the orchestrator into a degraded state that accepts lower-trust inputs without full validation.", "actions": [ "Model multi-agent pipeline failure scenarios as deliberate attack vectors, not just accidental failures, and design isolation boundaries with adversarial triggering in mind.", "Ensure agent-level circuit breakers cannot be forced open by a single adversarially crafted input injected at the pipeline entry point.", "Verify that partial-result outputs from a pipeline with a failed agent node are clearly marked as incomplete and cannot be silently consumed downstream as complete authoritative outputs." ], "failure_signals": [ "A single adversarially crafted input can force a pipeline agent circuit breaker to open state.", "Partial pipeline results are not distinguished from complete results in downstream consumer interfaces.", "Orchestration layer has no per-pipeline concurrency quotas; one failing pipeline can monopolize total orchestration capacity." ] }, "grc_auditor": { "summary": "Cascade failure prevention demonstrates that agentic AI systems are designed with operational resilience appropriate to their risk level, satisfying operational resilience requirements under DORA, NIST CSF, and equivalent frameworks for complex automated systems.", "actions": [ "Request documentation of all multi-agent pipeline topologies and verify that each has a documented failure propagation analysis identifying blast radius per agent node failure.", "Verify that bulkhead isolation and agent-level circuit breakers are implemented for all pipelines classified as business-critical.", "Review chaos experiment records to confirm cascade failure prevention mechanisms are tested at least annually per pipeline topology." ], "metrics": [ "Percentage of business-critical multi-agent pipelines with documented failure propagation analysis: target 100%.", "Pipelines with bulkhead isolation implemented: target 100% of business-critical pipelines.", "Cascade failure containment tests per pipeline topology in past 12 months: target at least one." ], "failure_signals": [ "Multi-agent pipelines deployed to production without failure propagation analysis.", "Business-critical agent pipelines without bulkhead isolation between agent nodes.", "Cascade failure scenarios have never been tested in a controlled environment." ] }, "business_continuity": { "summary": "Multi-agent pipeline cascade failures can simultaneously impact multiple business processes that share the same orchestration infrastructure. BCP must identify which processes depend on each pipeline and document the continuity impact at each failure severity level.", "actions": [ "Map multi-agent pipelines to the business processes they support and document the continuity impact at each failure severity level \u2014 single agent failure, partial pipeline, and complete pipeline.", "Include multi-agent cascade failure scenarios in BCP tabletop exercises so business stakeholders understand the potential scope of a pipeline outage versus an isolated agent failure.", "Validate pipeline recovery time objectives against the technical capabilities of the isolation boundary design to confirm RTOs are achievable under the implemented architecture." ], "failure_signals": [ "BCP does not distinguish between single-agent failure and full pipeline failure for continuity planning purposes.", "Business stakeholders are unaware that a single agent failure could affect multiple business processes simultaneously.", "Pipeline RTOs have not been validated against the implemented isolation boundary design." ] }, "it_operations": { "summary": "Operations contains multi-agent cascades in real time: per-agent health, isolation controls, and kill-switches must be operable from runbooks, not rediscovered mid-incident.", "actions": [ "Monitor per-agent health, queue depths, and retry rates with alerts on cascade precursors.", "Maintain runbooks for isolating or halting individual agents without stopping the whole pipeline.", "Verify bulkhead and timeout configurations after every pipeline topology change." ], "failure_signals": [ "One agent's failure is first detected as whole-pipeline collapse.", "There is no operational procedure to isolate a single misbehaving agent.", "Retry storms between agents amplify incidents instead of containing them." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Multi-agent pipeline resilience is an emerging discipline; most deployments lack formal bulkhead isolation and failure propagation analysis is rarely performed before production deployment of complex agent topologies." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "federated-enterprise", "cloud-native", "high-risk-sector", "eu-high-risk-ai" ], "implementers": [ "AI/ML Engineering", "Platform Engineering", "Site Reliability Engineering" ], "frameworks": [ { "framework": "google_sre", "requirement_id": "Ch. 22", "fit": "direct", "rationale": "Google SRE (SRE Book, 2016) Chapter 22, Addressing Cascading Failures, provides the canonical guidance on preventing cascades in distributed systems \u2014 load shedding, retry budgets, and deadline propagation. Multi-agent AI pipelines are distributed systems to which these containment patterns apply directly (Chapter 19 covers frontend load balancing).", "normative_force": "best-practice", "reviewed_on": "2026-07-02", "source_version": "2016", "basis": "anchored", "relation": "informs" }, { "framework": "aws_reliability", "requirement_id": "REL 5", "fit": "direct", "rationale": "AWS Well-Architected Reliability Pillar REL 5 requires designing interactions in a distributed system to mitigate or withstand failures \u2014 throttling, limiting retries, failing fast, and shedding load. These are the mechanisms that keep one agent's failure from cascading through an orchestration pipeline.", "normative_force": "best-practice", "reviewed_on": "2026-07-02", "source_version": "2024", "basis": "anchored", "relation": "informs" }, { "framework": "nist_800_160_v2", "requirement_id": "\u00a72.1.3", "fit": "direct", "rationale": "NIST SP 800-160 Vol 2 Rev 1 \u00a72.1.3 includes Segmentation among its cyber resiliency techniques, realized through Appendix D's 'Contain and exclude behaviors' design principle. Bulkheads and isolation boundaries between agents implement that containment so a failing agent cannot take down the whole pipeline.", "normative_force": "voluntary-standard", "reviewed_on": "2026-07-02", "source_version": "Rev 1", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 11 \u2014 ICT Business Continuity", "fit": "partial", "rationale": "EU DORA's ICT business continuity requirements apply to AI-dependent financial services orchestration, where multi-agent pipeline cascade failures can disrupt critical financial operations across multiple services simultaneously. FO-05's cascade prevention controls reduce the continuity impact of individual agent failures in financial AI pipelines. The partial fit reflects that DORA does not specifically address multi-agent AI architectures but the requirement applies by extension.", "normative_force": "binding-law", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" } ], "canonical_id": "apeiris://resilience/controls/FO-05", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Each agent node in the pipeline must operate within independently bounded resource pools such that a failure in any single agent does not exhaust shared resources or propagate exceptions to upstream orchestrators beyond the configured blast-radius limit. Backpressure signals must be emitted and honored before unbounded retries are triggered.", "evidence_required": [ "bulkhead_configuration_record documenting per-agent thread pool, connection pool, and timeout budget allocations with no shared pool entries across node boundaries", "pipeline_failure_injection_report showing blast-radius containment for each agent node under simulated hard failure, including affected_request_count and propagation_depth_observed", "circuit_breaker_state_log with trip_threshold, cooldown_duration, and fallback_action_taken for each inter-agent call over a 30-day production window", "retry_policy_definition specifying max_retries, backoff_strategy, jitter_range, and burst_rate_limit per agent-to-agent dependency", "architectural_diagram annotating isolation boundaries between all agent nodes in each production pipeline with per-node resource limits labeled" ], "machine_tests": [ "Simulate hard failure of downstream agent node B in a 3-node pipeline \u2192 assert upstream agent A receives error response within timeout_budget_ms, does not retry beyond max_retries, and emits circuit-open signal with error_code=upstream_unavailable", "Inject connection pool exhaustion on agent node C \u2192 assert requests to C are rejected with 503 within circuit_breaker_threshold_ms and agent A's connection pool remains at pre-injection utilization level", "Trigger retry storm by failing agent B with transient 500s at 80% rate \u2192 assert exponential backoff with jitter fires and inter-agent request rate does not exceed burst_rate_limit", "Deploy pipeline artifact missing bulkhead_configuration_record \u2192 assert deployment validation rejects artifact with error=missing_bulkhead_config before reaching production" ], "human_review": [ "Review pipeline architecture diagram for completeness of blast-radius isolation, verifying each agent node has independently bounded resource pools with no shared capacity entries across node boundaries", "Assess fallback behaviors for each agent node to confirm they produce meaningful operational alternatives rather than silently swallowing errors or propagating partial state corruption", "Verify failure propagation limits are calibrated against measured production traffic patterns and not derived from framework defaults without load-based validation" ], "blocking_effect": "blocks-deployment", "normative_status": "voluntary-standard", "anti_patterns": [ "Sharing a single connection pool or thread pool across all agent nodes, allowing one failing agent to exhaust resources for the entire pipeline simultaneously", "Configuring retry policies without jitter or backoff caps on inter-agent calls, creating retry storms that amplify cascade failures rather than dampening them", "Treating orchestrator-level timeouts as the sole isolation mechanism instead of implementing per-node bulkhead resource limits with independent circuit breakers", "Defining fallback behaviors that silently continue execution and mask error state rather than emitting structured signals that upstream monitors can detect", "Relying on downstream agent health checks without circuit breakers, causing the orchestrator to continuously invoke a failing agent until the global request timeout fires" ], "update_status": "current", "layer_code": "FO" }, { "id": "FO-06", "layer": "FO", "plane": "control", "name": "Load Shedding and Backpressure Mechanisms", "plain": "AI systems must implement load shedding and backpressure mechanisms that gracefully reject or defer excess requests when approaching capacity limits, preserving service quality for accepted requests rather than degrading uniformly across all requests or allowing resource exhaustion to cause complete service failure.", "threat": { "tags": [ "capacity-exhaustion", "overload-failure", "resource-starvation", "latency-spike" ], "desc": "Without load shedding, AI systems under overload attempt to serve all requests simultaneously, causing response latency to increase until the service becomes effectively unavailable for every caller. Memory and compute exhaustion under overload can cause inference workers to crash, requiring manual restart and extending outage duration. Without backpressure signaling, upstream callers continue dispatching at full rate unaware of the downstream overload, amplifying the overload condition and preventing self-healing." }, "standard": [ { "id": "google_sre", "section": "Ch. 22", "title": "Load Shedding and Graceful Degradation (Addressing Cascading Failures)" }, { "id": "aws_reliability", "section": "REL 5", "title": "Design interactions to mitigate or withstand failures \u2014 throttle requests and shed load" }, { "id": "cloudflare_resilience", "section": "Rate limiting", "title": "Cloudflare rate limiting and DDoS protection (product capability)" }, { "id": "dora", "section": "RTS (EU) 2024/1774 Art. 9", "title": "ICT capacity and performance management" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/FO-06 Load Shedding and Backpressure Mechanisms control.", "reviewed_on": "2026-07-01" }, { "id": "cloudflare_resilience_2024", "title": "Cloudflare DDoS Protection", "authority": "Cloudflare, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.cloudflare.com/ddos/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "cloudflare_resilience_2024", "relationship": "informative_reference", "rationale": "Establishes Cloudflare DDoS Protection requirements informing the apeiris://resilience/controls/FO-06 Load Shedding and Backpressure Mechanisms control.", "reviewed_on": "2026-07-02" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/FO-06 Load Shedding and Backpressure Mechanisms control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Implement a multi-tier load shedding strategy: (1) token bucket rate limiting at the API gateway layer with client-specific quotas, (2) priority-queue-based request scheduling that sheds low-priority requests first under load, (3) backpressure signaling (HTTP 429 with Retry-After, queue depth headers) to inform upstream callers to reduce dispatch rate, (4) automatic scaling triggers that initiate scale-out before shedding thresholds are reached where infrastructure permits.", "steps": [ "Configure token bucket rate limiters at the AI gateway layer with per-client, per-tier, and global rate limits; return HTTP 429 with Retry-After headers and structured error bodies enabling clients to implement adaptive backoff.", "Implement priority-based request scheduling in the AI inference queue, assigning priority tiers based on request classification (e.g., real-time interactive > batch > background); under capacity pressure, shed lowest-priority requests first while protecting higher-priority tiers.", "Emit backpressure signals through response headers (X-Queue-Depth, X-Capacity-Utilization) and webhook notifications when capacity utilization exceeds defined thresholds, enabling upstream callers to proactively reduce dispatch rate before hard rejection begins.", "Define and test capacity ceiling triggers that initiate automatic scale-out actions (add inference workers, expand quota, failover to secondary region) before load shedding activation thresholds are reached under normal load growth." ], "site_reliability": { "summary": "Load shedding is an intentional reliability mechanism, not a failure state. SREs must be able to observe shedding activation, verify priority tiers are honored, and confirm that shedding prevents broader service degradation for accepted requests.", "actions": [ "Instrument the load shedding layer with metrics for shed rate by priority tier, 429 response rate, queue depth, and capacity utilization; set SLO error budget burn rate alerts for unexpected shedding patterns.", "Conduct quarterly load tests that drive the AI service to the shedding activation threshold and verify that shed requests are rejected cleanly while non-shed requests maintain their target latency SLOs.", "Define thresholds in runbooks at which shedding frequency indicates a capacity planning failure requiring immediate escalation rather than normal operational response." ], "failure_signals": [ "Load shedding is not instrumented; capacity exhaustion is only discovered when the service becomes unresponsive.", "Shedding discards high-priority requests at the same rate as low-priority requests due to missing priority classification.", "Upstream callers continue dispatching at full rate despite sustained 429 responses, indicating backpressure signals are not being acted upon." ] }, "security_architect": { "summary": "Load shedding mechanisms can be exploited to perform targeted denial-of-service against specific clients if the priority tier assignment is manipulable, or to reveal internal capacity state through shedding patterns. The shedding configuration must be hardened against these attack vectors.", "actions": [ "Review priority tier assignment logic to confirm it cannot be manipulated by a caller claiming elevated priority in request headers or metadata to avoid shedding.", "Ensure capacity utilization metrics are not exposed to unauthenticated callers in a form that reveals exploitable internal capacity state.", "Confirm that rate limiting is applied per authenticated identity to prevent one client from consuming shared quota headroom to induce disproportionate shedding against other clients." ], "failure_signals": [ "Callers can self-report priority tier in request headers to avoid shedding under load.", "Capacity utilization data is exposed in health endpoints accessible without authentication.", "Rate limiting is applied per source IP rather than per authenticated client identity." ] }, "it_operations": { "summary": "Operations teams must understand the difference between expected shedding under normal peak load and shedding that indicates a capacity planning failure or infrastructure incident requiring escalation.", "actions": [ "Create runbooks that define expected shedding rates for normal operations, elevated-but-acceptable conditions, and capacity emergency thresholds requiring immediate escalation to engineering.", "Train operations staff on how to read capacity utilization dashboards and interpret backpressure signals from AI inference infrastructure without requiring developer knowledge of the implementation.", "Establish a capacity planning review cadence that uses shedding frequency data and trends to inform AI inference capacity provisioning decisions before capacity emergencies occur." ], "failure_signals": [ "Operations cannot distinguish expected peak-load shedding from a capacity emergency requiring escalation.", "Shedding events are escalated as incidents unnecessarily due to absent context in operational runbooks.", "Capacity planning decisions are made without incorporating shedding frequency trend data." ] }, "grc_auditor": { "summary": "Load shedding implementation demonstrates that the organization manages AI service capacity proactively, satisfying capacity management and operational resilience requirements under applicable continuity and operational resilience frameworks.", "actions": [ "Verify that load shedding configurations are documented and have been reviewed within the past year.", "Request load test records confirming shedding mechanisms perform as designed under peak load conditions representative of production traffic patterns.", "Confirm shedding activation thresholds are aligned with capacity planning targets and do not breach contractual SLA availability floors." ], "metrics": [ "Load shedding configuration documentation: current and reviewed within 12 months.", "Load tests demonstrating shedding behavior under realistic peak load: at minimum annually.", "Shedding activations without corresponding scale-out within SLA window: target 0 per quarter." ], "failure_signals": [ "Load shedding configurations undocumented or not reviewed within 12 months.", "No load test evidence demonstrating shedding works as designed under realistic peak conditions.", "Shedding is categorized as a system failure in incident reports rather than an intentional capacity control." ] }, "business_continuity": { "summary": "Load shedding decides which business work survives overload. Priority classes must reflect business criticality \u2014 set by process owners, not inferred by engineers under pressure.", "actions": [ "Assign shedding priority classes to AI workloads with business-owner sign-off.", "Review shed events with process owners to confirm the right work was deferred.", "Include overload scenarios in continuity planning for peak business periods." ], "failure_signals": [ "Revenue-critical requests are shed while batch analytics keep running.", "Business owners learn about shedding policies only after an overload event.", "Peak-period capacity planning ignores documented shedding thresholds." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Most AI services implement basic rate limiting but few implement priority-aware shedding or proactive backpressure signaling; overload handling typically defaults to uniform latency degradation until service failure." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "multi-tenant" ], "implementers": [ "Platform Engineering", "Site Reliability Engineering", "AI/ML Engineering" ], "frameworks": [ { "framework": "google_sre", "requirement_id": "Ch. 22", "fit": "direct", "rationale": "Google SRE (SRE Book, 2016) Chapter 22's Load Shedding and Graceful Degradation section prescribes dropping lower-priority load early to preserve capacity for critical work during overload. Priority-based shedding for AI inference implements this guidance (Chapter 21, Handling Overload, covers the per-request criticality model it builds on).", "normative_force": "best-practice", "reviewed_on": "2026-07-02", "source_version": "2016", "basis": "anchored", "relation": "informs" }, { "framework": "aws_reliability", "requirement_id": "REL 5", "fit": "direct", "rationale": "AWS Well-Architected Reliability Pillar REL 5 includes throttling requests and shedding excess load among its best practices for withstanding failure and overload. Load shedding and backpressure for AI serving tiers are direct implementations (REL 10 covers fault isolation, not load shedding).", "normative_force": "best-practice", "reviewed_on": "2026-07-02", "source_version": "2024", "basis": "anchored", "relation": "informs" }, { "framework": "cloudflare_resilience", "requirement_id": "Rate limiting and DDoS protection", "fit": "direct", "rationale": "Cloudflare's DDoS protection and rate-limiting products enforce request limits at the CDN and API-gateway layer, providing the first tier of load shedding for AI endpoints before traffic reaches origin infrastructure. This is a product-capability citation; the product documentation has no normative section numbering.", "normative_force": "best-practice", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "dora", "requirement_id": "Art. 7", "fit": "partial", "rationale": "EU DORA Article 7 requires financial entities to use ICT systems, protocols and tools that are reliable and equipped with sufficient capacity for peak demand. Load shedding and backpressure are the runtime mechanisms that keep AI services within capacity limits when demand exceeds them; detailed capacity-and-performance management requirements sit in RTS (EU) 2024/1774 Article 9.", "normative_force": "binding-law", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_csf", "requirement_id": "PR.IR-04", "fit": "adjacent", "rationale": "NIST CSF 2.0 PR.IR-04 requires that adequate resource capacity to ensure availability is maintained. Load shedding and backpressure protect that availability by preventing resource exhaustion when demand exceeds provisioned capacity (there is no 'PR.DS availability' outcome in CSF 2.0).", "normative_force": "voluntary-standard", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://resilience/controls/FO-06", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "When the AI system's inbound request rate reaches the configured capacity threshold, the system must shed excess requests with structured rejection responses and propagate backpressure signals to upstream clients before resource exhaustion occurs. Shed requests must resolve within the rejection_latency_budget without consuming compute resources proportional to accepted requests.", "evidence_required": [ "rate_limiter_configuration_artifact specifying token bucket capacity, refill rate, and client-specific quota assignments per API endpoint and priority tier", "load_test_report documenting system behavior at 100%, 120%, and 150% of rated capacity, with shedding_activation_threshold, shed_request_count, and p99_latency_for_accepted_requests recorded for each level", "backpressure_signal_log showing Retry-After or equivalent headers returned to clients for each shedding event with capacity_utilization_percent at time of shed", "priority_queue_configuration artifact showing request priority tier definitions and shed_order_policy confirming lower-priority tiers are shed before higher-priority tiers", "capacity_utilization_trend_chart covering at least 30 days of production traffic with shedding event markers and tier-level breakdown" ], "machine_tests": [ "Send requests at 2x rated capacity sustained for 60 seconds \u2192 assert system maintains latency SLO for priority-tier-1 requests and sheds at least 40% of tier-3 requests with 429 or 503 status codes", "Exceed token bucket quota for a single client \u2192 assert client receives 429 response with Retry-After header value within \u00b110% of configured token_refill_interval_seconds", "Simulate queue depth reaching 95% of max_queue_depth \u2192 assert backpressure signal is emitted to upstream load balancer before queue reaches 100% and new requests are shed before queuing", "Restore request rate to 80% of rated capacity after a shedding event \u2192 assert previously shed-tier requests are accepted within recovery_window_seconds without requiring manual intervention" ], "human_review": [ "Review priority queue tier assignments to confirm they reflect current operational importance and are not arbitrary groupings inherited from a prior architecture without revalidation", "Assess whether shedding thresholds are calibrated against measured production peak traffic with headroom sufficient to absorb sudden burst events without premature or late activation", "Verify that backpressure signals carry enough structured information (status code, Retry-After, error detail) for upstream clients to implement backpressure-aware retry without guessing retry timing" ], "blocking_effect": "blocks-deployment", "normative_status": "best-practice", "anti_patterns": [ "Implementing rate limiting only at the application layer without enforcement at the API gateway, allowing overloaded backends to receive the full request volume before shedding begins", "Configuring a single global rate limit that ignores request priority tiers, causing high-value inference requests to be shed at the same rate as low-priority background tasks", "Returning generic 500 Internal Server Error responses during overload instead of structured 429 or 503 responses with Retry-After headers, preventing clients from implementing adaptive retry strategies", "Calibrating shedding thresholds against theoretical rated capacity rather than measured production load, leading to activation that is either premature at normal peaks or too late during real overload", "Relying on autoscaling as the sole overload response without any load shedding, creating unavailability windows during scale-out lag that affect all requests regardless of priority" ], "update_status": "current", "layer_code": "FO" }, { "id": "FO-07", "layer": "FO", "plane": "lifecycle", "name": "Fault Tolerance Test Coverage", "plain": "Every fault tolerance mechanism implemented for an AI system must have corresponding test coverage that exercises the mechanism under realistic failure conditions. Fault tolerance test coverage must be tracked, maintained, and reviewed as part of the AI system's release and operations lifecycle to ensure mechanisms function as designed and have not regressed.", "threat": { "tags": [ "untested-failure-modes", "resilience-assumption-failure", "chaos-gap", "silent-fault" ], "desc": "Fault tolerance mechanisms that have never been tested under realistic failure conditions provide false assurance. Circuit breakers may be misconfigured and never trigger; fallback behaviors may contain code defects that prevent them from activating; load shedding thresholds may be calibrated incorrectly for actual production load patterns. Silent faults \u2014 where the fault tolerance mechanism activates but produces incorrect results without raising an error \u2014 are discoverable only through explicit test coverage that validates output quality under failure conditions, not just mechanism activation." }, "standard": [ { "id": "google_sre", "section": "Ch. 33", "title": "Lessons Learned from Other Industries" }, { "id": "aws_reliability", "section": "REL 12", "title": "Test reliability using game days, fault injection, and chaos engineering" }, { "id": "nist_800_160_v2", "section": "\u00a73.2", "title": "Cyber resiliency analysis" }, { "id": "iso_22301", "section": "\u00a78.5", "title": "Exercise and testing of continuity plans" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/FO-07 Fault Tolerance Test Coverage control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/FO-07 Fault Tolerance Test Coverage control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Implement a structured fault tolerance test program with three tiers: (1) unit-level tests that inject failure conditions into individual fault tolerance mechanisms, (2) integration-level chaos experiments that inject infrastructure failures into running AI services in production-like environments, (3) game days that simulate production failure scenarios end-to-end including operational response. Maintain a fault tolerance test matrix mapping each FO control mechanism to at least one test at each tier.", "steps": [ "Create and maintain a fault tolerance test matrix that inventories every fault tolerance mechanism (circuit breakers, fallbacks, load shedding, input validation) and maps each to unit tests, integration chaos tests, and game day scenarios.", "Implement automated unit tests for all fault tolerance mechanisms that inject the failure conditions each mechanism is designed to handle and assert that the mechanism activates correctly and produces the expected degraded output within contract parameters.", "Implement integration-level chaos experiments using fault injection tooling that simulates dependency failures, latency spikes, resource exhaustion, and malformed inputs against the running AI service in a production-like environment, run at minimum quarterly per AI service.", "Conduct annual game days simulating production-scale failure scenarios including multi-component failures, extended outages, and cascade failure conditions, with post-game-day reviews whose findings feed back into the fault tolerance improvement backlog." ], "site_reliability": { "summary": "Fault tolerance testing is a core SRE practice. The absence of fault tolerance test coverage is itself a reliability risk that must be tracked and remediated through the reliability backlog, not deferred indefinitely.", "actions": [ "Maintain a fault tolerance test matrix in the team's reliability backlog and track coverage percentage as a standing reliability metric in engineering reviews.", "Integrate chaos experiments into the CI/CD pipeline for AI services so that fault tolerance regressions \u2014 such as a misconfigured circuit breaker threshold \u2014 are caught before production deployment.", "Publish game day findings as reliability postmortems with action items tracked to completion on a defined SLA." ], "failure_signals": [ "Fault tolerance mechanisms are deployed to production with no corresponding automated test coverage.", "Chaos experiments have not been run for an AI service in more than 90 days.", "Game day findings are repeatedly deferred in the engineering backlog without resolution." ] }, "security_architect": { "summary": "Fault tolerance testing must include adversarial test cases that verify security controls remain effective under failure conditions. Security regression testing during failure states is a distinct requirement from functional correctness testing.", "actions": [ "Include adversarial inputs in fault tolerance test cases to verify security controls (input validation, output filtering, access control) remain active when fault tolerance mechanisms are engaged.", "Test that circuit-open and fallback states do not expose internal system information through error responses, leaking stack traces, hostnames, or model identifiers.", "Verify that load shedding priority tier logic cannot be bypassed through adversarial input patterns during chaos experiment execution." ], "failure_signals": [ "Fault tolerance test cases do not include security regression assertions for controls active on the primary path.", "Adversarial inputs are absent from chaos experiment test case libraries.", "Game day scenario scripts do not include explicit verification of security control status under failure conditions." ] }, "grc_auditor": { "summary": "Fault tolerance test coverage provides the evidence base for asserting that fault tolerance mechanisms work as designed. Without test records, fault tolerance control attestations are assertions of intent without substantiation for audit purposes.", "actions": [ "Request the fault tolerance test matrix and verify it provides coverage for all implemented FO-layer mechanisms with records showing when each test was last executed.", "Review chaos experiment and game day records to confirm tests were conducted within required frequency windows and that findings were tracked to closure.", "Verify that fault tolerance test findings do not remain open indefinitely and that open items have documented risk acceptance or remediation timelines." ], "metrics": [ "Fault tolerance mechanism test coverage: target 100% of implemented mechanisms have at least one automated test.", "Chaos experiments conducted in past 90 days per AI service: target 100%.", "Annual game days conducted per AI service portfolio: target at least one." ], "failure_signals": [ "Fault tolerance mechanisms in production without any corresponding test coverage in the test matrix.", "No chaos experiment records for an AI service in more than 90 days.", "Annual game day not conducted despite material changes to the AI service architecture." ] }, "business_continuity": { "summary": "Fault tolerance test results validate that the technical controls supporting business continuity objectives actually function as designed. BCM teams should incorporate fault tolerance test outcomes into their confidence assessment for AI-dependent continuity plans rather than relying solely on design documentation.", "actions": [ "Review fault tolerance test records as part of the annual BCP review cycle to confirm that technical controls supporting continuity objectives have been tested and are functioning as expected.", "Incorporate game day observations into BCP revision cycles, using observed failure behavior and recovery timelines to update continuity procedures and RTO estimates.", "Require fault tolerance test completion as a release gate for AI service changes that materially affect business-critical capabilities." ], "failure_signals": [ "BCP reviews rely on design documentation rather than fault tolerance test outcomes for AI-dependent services.", "Continuity RTO estimates are not updated to reflect game day observations when they differ from design assumptions.", "AI service changes affecting business-critical capabilities proceed to production without fault tolerance test verification." ] }, "it_operations": { "summary": "Operations executes the fault-injection program safely: scheduled game days, controlled blast radius, and clean rollback of injected faults \u2014 with findings feeding the operational backlog.", "actions": [ "Run scheduled game days per the test calendar with pre-approved blast-radius limits and abort procedures.", "Verify every injected fault is fully reverted and systems return to steady state after each test.", "Convert test findings into tracked operational fixes and re-test on the next cycle." ], "failure_signals": [ "Game days keep slipping and coverage decays below the declared cadence.", "An injected fault lingers after a test window closes.", "The same fault class fails in consecutive test cycles with no fix in between." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Fault tolerance testing for AI systems is underdeveloped across the industry; most teams rely on production incident discovery rather than systematic chaos engineering to validate that resilience mechanisms work as designed." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "eu-high-risk-ai" ], "implementers": [ "Site Reliability Engineering", "AI/ML Engineering", "Quality Assurance" ], "frameworks": [ { "framework": "google_sre", "requirement_id": "Ch. 33", "fit": "direct", "rationale": "Google SRE (SRE Book, 2016) Chapter 33, Lessons Learned from Other Industries, draws on other industries' practice of proactive, structured failure testing and preparedness exercises as a model for reliability engineering. It supports the game-day discipline FO-07 requires; chaos-engineering specifics are industry practice layered on top rather than a chapter of the book.", "normative_force": "best-practice", "reviewed_on": "2026-07-02", "source_version": "2016", "basis": "anchored", "relation": "informs" }, { "framework": "aws_reliability", "requirement_id": "REL 12", "fit": "direct", "rationale": "AWS Well-Architected Reliability Pillar REL 12 (Test reliability) requires game days, fault injection, and chaos engineering to validate that resilience mechanisms work under failure conditions before they are needed. Fault-tolerance test coverage operationalizes this for AI systems.", "normative_force": "best-practice", "reviewed_on": "2026-07-02", "source_version": "2024", "basis": "anchored", "relation": "informs" }, { "framework": "nist_800_160_v2", "requirement_id": "\u00a73.2", "fit": "direct", "rationale": "NIST SP 800-160 Vol 2 Rev 1 \u00a73.2 describes cyber resiliency analysis \u2014 determining, with evidence, whether a system's resilience properties actually hold rather than assuming them from design documentation. Fault-tolerance test coverage produces exactly that evidence for AI systems.", "normative_force": "voluntary-standard", "reviewed_on": "2026-07-02", "source_version": "Rev 1", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_22301", "requirement_id": "\u00a78.5 \u2014 Business Continuity Exercises", "fit": "partial", "rationale": "ISO 22301:2019 Section 8.5 requires organizations to test business continuity plans through exercises that validate the plans function as expected under realistic conditions. FO-07's game day requirement is the AI system-specific implementation of ISO 22301 exercise requirements, validating that technical fault tolerance mechanisms support the continuity objectives documented in organizational BCP. The partial fit reflects that ISO 22301 focuses on organizational continuity exercises while FO-07 focuses on technical mechanism validation.", "normative_force": "certification-standard", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 26", "fit": "partial", "rationale": "EU DORA Article 26 requires advanced testing by way of threat-led penetration testing on critical live systems for significant financial entities. Comprehensive fault-injection coverage of critical AI systems is the operational-resilience counterpart such advanced testing programs build on (baseline digital operational resilience testing obligations sit in Articles 24-25).", "normative_force": "binding-law", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" } ], "canonical_id": "apeiris://resilience/controls/FO-07", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every fault tolerance mechanism deployed in the AI system must have at least one executed test case that exercises it under a realistic failure condition, and the aggregate test suite must achieve the documented fault_coverage_target with no failure mode from the threat model remaining untested. Test results must be recorded and traceable by mechanism ID.", "evidence_required": [ "fault_coverage_matrix mapping each deployed fault tolerance mechanism to at least one test case with fields test_id, failure_condition_injected, mechanism_under_test, and pass_fail outcome", "chaos_experiment_run_record for each experiment showing target_component, failure_type, blast_radius_actual, steady_state_metric_before, and steady_state_metric_after", "game_day_report with scenario_narrative, expected_behavior, observed_behavior, and delta_actions_required for each conducted exercise", "fault_injection_ci_run_log from the CI/CD pipeline showing automated fault injection test execution and pass/fail result per deployment artifact", "remediation_tracking_record for any test-discovered gaps with status, owner, and target_resolution_date" ], "machine_tests": [ "Execute fault injection test suite in CI pipeline for a representative deployment artifact \u2192 assert all mechanism-to-test mappings in fault_coverage_matrix return pass and coverage_percentage meets or exceeds fault_coverage_target", "Kill a randomly selected agent node during an active multi-step workflow \u2192 assert the system invokes the documented failover mechanism within failover_timeout_seconds and steady-state is restored without data loss", "Inject network partition between inference endpoint and model serving layer \u2192 assert graceful degradation activates and returns a structured fallback response rather than hanging beyond response_timeout_ms", "Introduce a new fault tolerance mechanism into the architecture without a corresponding fault_coverage_matrix entry \u2192 assert deployment gate validation rejects the artifact with error=uncovered_fault_tolerance_mechanism" ], "human_review": [ "Review the fault coverage matrix for completeness, verifying every mechanism in the current architecture has a corresponding test case and no known failure mode from the threat model is classified as untested without a documented waiver", "Assess chaos experiment scenarios for realism, confirming that injected failure conditions reflect plausible production failure patterns for the deployed infrastructure rather than purely synthetic synthetic conditions that do not occur in practice", "Verify that test-discovered gaps are tracked through remediation and do not persist across multiple consecutive test cycles without an accepted risk decision or remediation commitment" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Asserting fault tolerance coverage based on design documentation and architecture reviews alone without any executed test cases that verify mechanisms actually fire under failure conditions", "Running fault injection tests only in isolated unit test environments that do not reflect production dependency graphs, shared resource contention, or realistic concurrency levels", "Allowing fault_coverage_matrix entries with status=untested to persist across consecutive deployment cycles without a documented risk acceptance or remediation plan with a deadline", "Treating production incidents as the primary discovery mechanism for fault tolerance gaps rather than proactively injecting failures in controlled environments before production exposure", "Scoping chaos game days to a single team's services without including upstream and downstream dependent services, leaving cross-boundary failure propagation untested" ], "update_status": "current", "layer_code": "FO" }, { "id": "FO-08", "layer": "FO", "plane": "lifecycle", "name": "Fault Tolerance Evidence Package", "plain": "The organization must compile and maintain a structured evidence package demonstrating that FO-layer fault tolerance controls (FO-01 through FO-07) are designed, implemented, operationally active, and tested. This package serves as the compliance attestation artifact for the FO layer and must be renewed at least annually or after any material change to the AI system's fault tolerance architecture.", "threat": { "tags": [ "evidence-gap", "audit-exposure", "unverified-resilience", "compliance-assertion-failure" ], "desc": "Without a compiled evidence package, fault tolerance attestations are unsubstantiated assertions of intent. Regulators, auditors, and enterprise procurement teams increasingly require evidence-based proof of operational resilience, not documentation of design plans. An organization unable to produce a current and complete FO evidence package faces audit findings, procurement disqualification, and regulatory exposure under DORA and equivalent frameworks that require demonstrated resilience testing with reviewable records." }, "standard": [ { "id": "dora", "section": "Art. 25 & 26", "title": "DORA operational resilience testing and reporting requirements" }, { "id": "iso_22301", "section": "\u00a79.1", "title": "Performance evaluation and monitoring evidence requirements" }, { "id": "nist_csf", "section": "GV.OC-04", "title": "Organizational resilience requirements and evidence" }, { "id": "cobit_dss04", "section": "DSS04.04", "title": "Exercise, test and review the BCP and DRP \u2014 test evidence" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/FO-08 Fault Tolerance Evidence Package control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/FO-08 Fault Tolerance Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/FO-08 Fault Tolerance Evidence Package control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Compile the FO evidence package as a structured artifact collection indexed by control ID (FO-01 through FO-07). Each entry must include: design documentation, implementation artifact references, operational monitoring evidence (metric exports, alert records), and test records (unit test reports, chaos experiment results, game day postmortems). The package must be versioned, access-controlled, and linked to the canonical Apeiris FO-08 attestation record.", "steps": [ "Create and maintain an FO evidence index document that maps FO-01 through FO-07 to their corresponding evidence artifacts, with artifact location references, collection dates, and review status for each control.", "Establish an evidence collection cadence: design documentation reviewed at system design time, implementation artifacts collected at deployment, operational monitoring evidence exported monthly, and test records compiled after each chaos experiment and game day.", "Assign a named evidence owner for the FO layer who is responsible for maintaining the evidence index, triggering re-collection when material system changes occur, and presenting the package for internal and external audits.", "Conduct an annual FO evidence package review with the GRC team and AI system owners to assess completeness, identify gaps, initiate remediation actions, and issue the updated FO-08 attestation before the prior attestation expires." ], "site_reliability": { "summary": "SREs are the primary producers of FO layer operational evidence. Monitoring exports, chaos experiment reports, and game day postmortems must be structured for audit consumption \u2014 not just operational use \u2014 to satisfy evidence package requirements.", "actions": [ "Format monitoring metric exports and alert records in a structured, exportable format (JSON, CSV with defined schema) that audit reviewers can consume without access to live operational tooling.", "Tag chaos experiment and game day reports with the FO control IDs they exercise to enable automated evidence tracing in the evidence index.", "Maintain a running fault tolerance activation log (circuit breaker opens, fallback activations, load shedding events) with timestamps and durations retained for the evidence collection period." ], "failure_signals": [ "Operational monitoring evidence is only accessible via live dashboard and cannot be exported for audit review.", "Chaos experiment reports are not tagged to FO control IDs, requiring manual correlation during audit.", "Fault tolerance activation logs have gaps due to undetected logging configuration changes." ] }, "grc_auditor": { "summary": "The FO evidence package is the primary artifact for FO-layer compliance attestation. GRC teams own the evidence review and gap assessment process and must validate completeness before attestation is issued.", "actions": [ "Review the FO evidence index against the FO control matrix to confirm all seven FO controls have corresponding current evidence artifacts before issuing attestation.", "Verify that evidence artifacts are within the defined freshness window for each artifact type and have not been altered since collection, using hash verification where available.", "Document any evidence gaps identified during review and assign remediation owners with target completion dates; do not issue attestation with blocking gaps without formal risk acceptance." ], "metrics": [ "FO evidence index completeness: target 100% of FO-01 through FO-07 covered.", "Evidence artifact freshness compliance: target 100% of artifacts within defined freshness window at attestation time.", "Open blocking evidence gaps at attestation: target 0.", "Annual FO evidence review completion before attestation renewal: target 100%." ], "failure_signals": [ "FO controls without any corresponding evidence artifacts at the time of attestation issuance.", "Evidence artifacts presented that are older than the defined freshness window without documented justification.", "Evidence gaps carried into attestation without documented risk acceptance from an authorized approver." ] }, "business_continuity": { "summary": "The FO evidence package provides BCM teams with a unified view of fault tolerance control health across all AI-dependent systems, enabling informed continuity planning, regulatory reporting, and confident RTO commitments.", "actions": [ "Incorporate the FO evidence package review into the annual BCP review cycle to ensure continuity plans reflect current fault tolerance control status rather than original design assumptions.", "Use FO evidence package findings \u2014 particularly game day observations \u2014 to update AI service recovery time estimates in continuity plans when test results differ from design assumptions.", "Retain FO evidence packages for the regulatory retention period applicable to the organization's sector and jurisdiction, with documented retention schedules per artifact type." ], "failure_signals": [ "BCP review cycle does not incorporate FO evidence package findings for AI-dependent services.", "FO evidence packages are not retained for the minimum required regulatory period.", "BCM team is unaware of the FO evidence package or its role in supporting continuity planning decisions." ] }, "it_operations": { "summary": "Operations teams contribute operational evidence artifacts to the FO package. Evidence collection processes must be automated where possible to reduce manual burden and ensure evidence currency without depending on manual collection triggered by upcoming audit deadlines.", "actions": [ "Automate the export of operational monitoring evidence (metric exports, alert history, activation logs) on a defined schedule and deposit exports to the designated evidence artifact store automatically.", "Create and maintain runbooks documenting evidence collection procedures for each FO control artifact type, enabling consistent collection without specialist knowledge of the underlying systems.", "Test evidence collection procedures annually to confirm automated exports are complete, accessible, and correctly formatted; remediate any collection failures before the annual evidence review." ], "failure_signals": [ "Evidence collection relies entirely on manual processes with no automation, creating dependency on individual availability during audit preparation.", "Automated evidence exports fail silently and gaps are discovered only during audit review.", "Evidence collection procedures are undocumented and depend on tribal knowledge that is unavailable during staff transitions." ] }, "security_architect": { "summary": "Fault-tolerance evidence often exposes weaknesses; the package needs integrity protection and access control, and should include evidence that security controls held during injected failures.", "actions": [ "Include security-control behavior under fault conditions (auth, logging continuity) in the evidence scope.", "Protect the evidence package with integrity verification and restricted, logged access.", "Review evidence for findings that double as security vulnerabilities and route them to the security backlog." ], "failure_signals": [ "Evidence shows auth or logging failed during fault tests but no security ticket exists.", "The package is stored in a broadly writable share.", "Security has never reviewed fault-tolerance test outcomes." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Structured evidence packages for AI fault tolerance controls are rarely implemented proactively; most organizations rely on ad hoc artifact collection during audit preparation rather than continuous evidence maintenance." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "GRC / Compliance", "Site Reliability Engineering", "AI Risk Management" ], "frameworks": [ { "framework": "dora", "requirement_id": "Art. 25\u201326 \u2014 Resilience Testing and Reporting", "fit": "direct", "rationale": "EU DORA Articles 25 and 26 require financial entities to conduct and document operational resilience testing programs, with testing results reviewable by competent authorities on request. FO-08's structured evidence package directly satisfies DORA's documentation and reporting requirements by providing a complete, versioned record of AI fault tolerance control design, implementation, and testing. For financial entities, FO-08 is the primary mechanism for demonstrating DORA compliance for AI system resilience.", "normative_force": "binding-law", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "iso_22301", "requirement_id": "\u00a79.1 \u2014 Monitoring, Measurement, Analysis and Evaluation", "fit": "direct", "rationale": "ISO 22301:2019 Section 9.1 requires organizations to evaluate business continuity management system performance through monitoring, measurement, and evidence collection, with documented results retained and available for review. FO-08's evidence package is the AI fault tolerance implementation of ISO 22301's performance evaluation requirement. Certification to ISO 22301 requires current evidence of continuity control performance, which FO-08 provides in structured form.", "normative_force": "certification-standard", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "GV.OC-04 \u2014 Organizational Resilience Requirements", "fit": "partial", "rationale": "NIST CSF 2.0 Govern function GV.OC-04 requires organizations to document their resilience requirements and maintain evidence that those requirements are being met across the organization's risk posture. FO-08 provides the evidence structure satisfying GV.OC-04's documentation and evidence requirements for AI fault tolerance specifically, creating a reviewable record aligned with CSF governance obligations. The partial fit reflects that GV.OC-04 addresses organizational resilience broadly while FO-08 scopes to AI fault tolerance.", "normative_force": "voluntary-standard", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.04", "fit": "partial", "rationale": "COBIT 2019 DSS04.04 requires continuity plans to be exercised and tested with results documented and reviewed. FO-08's evidence package preserves those test artifacts; training records remain governed by DSS04.06.", "normative_force": "industry-framework", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.6", "fit": "partial", "rationale": "NIST SP 800-34 Rev 1 \u00a73.6 (Plan maintenance) requires contingency documentation to be kept current, with test results and updates retained as evidence. The fault-tolerance evidence package demonstrates that currency for AI systems (\u00a73.5 defines the TT&E program itself).", "normative_force": "voluntary-standard", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "REL 12", "fit": "direct", "rationale": "AWS Well-Architected Reliability Pillar REL 12 (Test reliability) requires documented reliability testing \u2014 game days, fault injection, chaos experiments \u2014 with results used to improve the workload. FO-08 packages that documentation for AI systems.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Production readiness review: fault tolerance criteria", "fit": "direct", "rationale": "Google SRE production readiness reviews require documented evidence of fault tolerance mechanisms before a service advances to production status. FO-08's evidence package directly maps to the fault tolerance section of the SRE PRR checklist, providing the structured artifacts reviewers need to assess resilience readiness.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "BCDR documentation: fault tolerance evidence", "fit": "direct", "rationale": "Microsoft Azure BCDR guidance requires documented evidence of fault tolerance patterns and tested recovery procedures. FO-08 produces the evidence package fulfilling this documentation requirement, capturing redundancy configurations, failover test results, and validated recovery time objectives for Azure-hosted workloads.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/FO-08", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "A structured evidence package indexed by control ID (FO-01 through FO-07) must exist, be current within the defined evidence_staleness_limit, and contain at minimum one design artifact, one implementation artifact, and one test artifact per control. The package must be self-contained and independently reviewable without supplementary explanation from the control owner.", "evidence_required": [ "evidence_package_index artifact mapping each of FO-01 through FO-07 to at least one design artifact, one implementation artifact, and one test artifact, each with artifact_id, artifact_type, and collected_at timestamp", "evidence_freshness_report confirming all artifacts are within the staleness_limit defined for their artifact_type, with any stale entries flagged and disposition documented", "assessor_review_record from the most recent independent evidence review with reviewer_id, review_date, gaps_identified, and resolution_status for each gap", "control_coverage_summary confirming all 7 FO-layer controls have non-empty evidence entries across all three required artifact types", "evidence_package_integrity_record with package_sha256_hash and signed_at timestamp ensuring tamper-evidence of the compiled package contents" ], "machine_tests": [ "Parse evidence_package_index \u2192 assert all 7 control IDs (FO-01 through FO-07) are present and each has at least three artifact references covering design, implementation, and test artifact_type values", "Check collected_at timestamps for each artifact against its artifact_type staleness_limit \u2192 assert no artifact exceeds its defined staleness_limit and all stale entries have a documented disposition", "Recompute SHA-256 hash of package contents and compare against evidence_package_integrity_record.package_sha256_hash \u2192 assert hashes match and signed_at is within the attestation_validity_window", "Query coverage API for FO-layer \u2192 assert coverage_percentage equals 100.0 with zero controls returning missing_artifact_types in the response" ], "human_review": [ "Review the evidence index for artifact quality, verifying that referenced artifacts contain sufficient procedural and measurement detail to allow an independent assessor to reproduce the assessment conclusion without access to the control owner", "Assess the evidence freshness report to confirm that stale artifact classifications are accurate and that artifacts flagged as stale have documented risk-acceptance decisions or active remediation commitments", "Verify that gaps identified in the most recent assessor review have been fully remediated or carry documented risk-acceptance decisions with named owners and expiry dates" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Assembling the evidence package as an ad hoc artifact collection at audit time rather than maintaining it continuously as a living record updated after each control change or test cycle", "Including design documentation alone without corresponding operational monitoring artifacts and test results, making the package unable to demonstrate that controls are actively enforced in production", "Omitting collected_at timestamps from artifact references, making it impossible to assess evidence freshness or enforce staleness limits programmatically", "Referencing artifacts by informal file paths or email thread attachments without stable, versioned identifiers that survive system migrations and repository restructuring", "Treating the FO evidence package as a one-time compilation deliverable that is archived after initial attestation rather than an artifact requiring continuous update as control states change" ], "update_status": "current", "layer_code": "FO" }, { "id": "RE-01", "layer": "RE", "plane": "control", "name": "High Availability Architecture for AI Systems", "plain": "AI inference endpoints, model serving infrastructure, and supporting data pipelines must be deployed with redundancy, load balancing, and automated failover to ensure continuous availability meeting defined SLA targets.", "threat": { "tags": [ "single-point-of-failure", "availability-degradation", "cascading-outage", "inference-unavailability" ], "desc": "AI systems without high availability design become single points of failure in production workflows. An unplanned outage of a model serving endpoint can halt autonomous agent operations, disrupt downstream business processes, and erode confidence in AI-driven decision-making. Without automated failover, recovery depends on manual intervention, extending mean time to recovery and violating availability SLAs." }, "standard": [ { "id": "aws_reliability", "section": "REL 10", "title": "Use fault isolation to protect your workload \u2014 deploy to multiple locations" }, { "id": "google_sre", "section": "Ch. 3", "title": "Embracing Risk and SLO-based availability management" }, { "id": "nist_800_160_v2", "section": "\u00a72.1.3", "title": "Redundancy and Diversity (cyber resiliency techniques)" }, { "id": "nist_csf", "section": "RC.RP-01", "title": "Recovery plan is executed during or after a cybersecurity incident" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RE-01 High Availability Architecture for AI Systems control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RE-01 High Availability Architecture for AI Systems control.", "reviewed_on": "2026-07-02" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RE-01 High Availability Architecture for AI Systems control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RE-01 High Availability Architecture for AI Systems control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RE-01 High Availability Architecture for AI Systems control.", "reviewed_on": "2026-07-01" }, { "id": "cloudflare_resilience_2024", "title": "Cloudflare DDoS Protection", "authority": "Cloudflare, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.cloudflare.com/ddos/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "cloudflare_resilience_2024", "relationship": "informative_reference", "rationale": "Establishes Cloudflare DDoS Protection requirements informing the apeiris://resilience/controls/RE-01 High Availability Architecture for AI Systems control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Deploy AI inference services across multiple availability zones or regions with a global load balancer, health-check-driven failover, and auto-scaling groups. Define SLOs (e.g., 99.9% availability) and enforce them via SLI pipelines.", "steps": [ "Identify all AI inference endpoints and model-serving components and map their single-points-of-failure.", "Deploy each serving component across at least two availability zones behind a load balancer with health checks.", "Configure automated failover with maximum 30-second detection and 60-second recovery targets.", "Implement auto-scaling policies triggered by latency and error-rate thresholds, not just CPU utilization.", "Define SLOs per endpoint and instrument SLI dashboards for continuous availability monitoring." ], "site_reliability": { "summary": "HA architecture is the foundation of AI service reliability. Engineer multi-AZ deployments and define SLOs before launching any production workloads.", "actions": [ "Map all AI service dependencies and identify single-point-of-failure nodes.", "Deploy serving infrastructure across multiple AZs with automated failover configured.", "Define and instrument SLOs; set error budget burn-rate alerts at 5% and 10% consumption thresholds." ], "failure_signals": [ "Single-AZ deployments for any production AI endpoint.", "No automated failover \u2014 recovery requires manual paging.", "SLO breaches not triggering error-budget alerts within 5 minutes." ] }, "security_architect": { "summary": "HA design must not trade availability for security. Redundant endpoints must be equally hardened and access-controlled as primary instances.", "actions": [ "Verify that failover replicas carry identical security configurations: TLS version, auth middleware, and egress controls.", "Ensure load balancer WAF rules apply uniformly across all backend nodes including standby replicas.", "Review that HA topology does not introduce cross-zone data residency violations for regulated data classes." ], "failure_signals": [ "Failover replicas with weaker TLS configuration or missing auth middleware.", "Load balancer bypasses that expose backends directly without WAF.", "Cross-region failover configuration violating documented data residency policy." ] }, "it_operations": { "summary": "Operations teams must be able to monitor, alert on, and respond to availability events in real time for all AI endpoints.", "actions": [ "Instrument all AI endpoints with availability and latency metrics exported to the centralized monitoring platform.", "Define and test runbooks for common HA failover and recovery scenarios covering at least AZ failure and load balancer failure modes.", "Conduct monthly failover drills and document results in the evidence store." ], "failure_signals": [ "No availability metrics for AI endpoints visible in the monitoring platform.", "Runbooks not tested within the last 90 days.", "Manual recovery steps consistently taking longer than defined RTO in drills." ] }, "grc_auditor": { "summary": "HA architecture must be documented, tested, and mapped to availability SLA commitments and applicable regulatory requirements.", "actions": [ "Request architecture diagrams and confirm multi-AZ or multi-region deployment for all Tier 1 AI services.", "Review SLO definitions and validate actual availability against targets over the prior quarter.", "Verify failover test records exist and are within the required 90-day recurrence interval." ], "metrics": [ "Target availability per Tier 1 AI endpoint: \u2265 99.9% measured monthly.", "Failover test success rate: 100% of scheduled tests completed.", "Time to detect failure: < 30 seconds for monitored endpoints." ], "failure_signals": [ "SLO availability below 99.9% for two consecutive months without a documented remediation plan.", "No failover test evidence in the last 90 days.", "Architecture diagrams not reflecting current deployed topology." ] }, "business_continuity": { "summary": "HA architecture directly protects business-critical AI capabilities from unplanned outages. Ensure all AI services are included in BCP scope with defined RTO and RPO.", "actions": [ "Include Tier 1 AI services in the Business Impact Analysis and assign RTO and RPO targets.", "Confirm that SLO breach thresholds align with BCP maximum tolerable downtime impact thresholds.", "Validate that HA test results feed into the annual BCP review and update cycle." ], "failure_signals": [ "AI services not listed in the BIA or BCP scope.", "RTO targets not defined or misaligned with business impact analysis findings.", "HA test results not reviewed during the annual BCP review cycle." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Many enterprises deploy AI models on single-region infrastructure without formal SLOs or automated failover." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "eu-high-risk-ai" ], "implementers": [ "Platform Engineering", "Site Reliability Engineering", "Infrastructure Team" ], "frameworks": [ { "framework": "aws_reliability", "requirement_id": "REL 10", "fit": "direct", "rationale": "AWS Well-Architected Reliability Pillar REL 10 (Use fault isolation to protect your workload) requires deploying workloads across multiple Availability Zones or Regions so that a location failure does not take down the service. Multi-AZ deployment with automated failover for AI serving implements this directly (REL 6 covers workload monitoring).", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Ch. 3", "fit": "direct", "rationale": "Google SRE Chapter 3 establishes the SLO-based framework for managing availability risk and error budgets. Defining and instrumenting SLOs for AI serving endpoints is a direct application of this framework. Error budget burn-rate alerting provides the operational trigger for HA remediation actions.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "nist_800_160_v2", "requirement_id": "\u00a72.1.3", "fit": "direct", "rationale": "NIST SP 800-160 Vol 2 Rev 1 \u00a72.1.3 defines Redundancy and Diversity among its cyber resiliency techniques, elaborated by Appendix D design principles such as 'Maintain redundancy' and 'Plan and manage diversity'. High-availability AI architecture is the application of those techniques to serving infrastructure.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "RC.RP-01", "fit": "partial", "rationale": "NIST CSF 2.0 Recover function RC.RP-01 requires that recovery plans be executed during or after incidents. HA architecture is a prerequisite that reduces the need for recovery plan activation by keeping services operational through automated failover. The Recover function provides governance context for validating HA effectiveness through drill evidence.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 11", "fit": "partial", "rationale": "EU DORA Article 11 requires financial entities to maintain redundant ICT capabilities to ensure continuity of critical functions. AI systems supporting financial workflows must meet HA requirements as part of DORA ICT business continuity policy. HA architecture documentation and test records are required audit artifacts for DORA supervisory review.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "microsoft_azure_resil", "requirement_id": "Azure Resiliency \u2014 Availability zone design and geo-redundancy for mission-critical workloads", "fit": "direct", "rationale": "Microsoft Azure Resiliency documentation defines availability zone design and geo-redundancy as the primary implementation pattern for the redundancy, load balancing, and automated failover that the High Availability Architecture for AI Systems control requires. Azure BCDR guidance prescribes deploying stateless inference workloads across at least two availability zones with Traffic Manager-based health-check failover \u2014 a direct implementation path for the SLA-backed availability posture this control mandates. The recovery objective alignment section of Azure resiliency documentation maps availability zone primitives to RTO and RTA targets, providing an implementation-level correspondence to this control's SLO enforcement requirements.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "cloudflare_resilience", "requirement_id": "Cloudflare Resilience \u2014 Load balancing and failover, DDoS protection layers for AI inference endpoint availability", "fit": "direct", "rationale": "Cloudflare's load balancing and failover capabilities implement active-active and active-passive configurations with health-check-driven routing that directly realize the automated failover requirements of the High Availability Architecture for AI Systems control at the network ingress layer. Cloudflare's DDoS protection layers ensure that inference endpoint availability is maintained even under volumetric attack conditions, providing an additional availability guarantee beyond what infrastructure-layer redundancy alone can deliver. Cloudflare's edge resilience patterns specifically address AI inference API availability through rate limiting and anycast routing, making this framework a direct implementation reference for this control's SLA protection requirements.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "SRE Workbook Ch. 2 \u2014 Implementing SLOs", "fit": "partial", "rationale": "The Google SRE Workbook Chapter 2 provides operational guidance for SLO-based availability architecture; RE-01 HA architecture should adopt SLO error-budget management to balance reliability investment for AI systems.", "normative_force": "best-practice", "source_version": "2018", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RE-01", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "AI inference endpoints and model serving infrastructure must be deployed across at least two independent availability zones with automated health-check-driven failover, and the system must sustain the defined availability SLO under a simulated single-AZ failure without manual intervention. Failover must complete within the documented RTO and result in zero data loss beyond the RPO.", "evidence_required": [ "infrastructure_topology_diagram annotating AZ placement for all inference endpoints, load balancers, and supporting data services with redundancy tier labels and single-AZ failure blast radius boundaries", "slo_definition_document specifying availability_target_percentage, RTO_seconds, and RPO_seconds for each AI service component signed by the service owner", "failover_test_report from the most recent AZ-failure simulation showing time_to_failover_seconds, affected_request_count, error_count_during_failover, and data_loss_events", "auto_scaling_policy_record showing scale-in and scale-out triggers, cooldown_seconds, and minimum_healthy_instances constraint per deployment tier", "availability_monitoring_export covering 90 days of uptime data with SLO burn rate annotations and incident markers" ], "machine_tests": [ "Terminate all instances in one availability zone \u2192 assert traffic fails over to remaining AZ within rto_seconds and subsequent requests return 2xx with latency within slo_latency_p99_ms", "Scale inference endpoint to zero instances \u2192 assert auto-scaling policy triggers new instance provisioning within scale_out_trigger_seconds and health checks pass before traffic is routed", "Send 10,000 requests during a simulated AZ failover window \u2192 assert fewer than error_budget_requests return 5xx errors and no request hangs beyond 2x the normal response timeout", "Verify load balancer health check configuration against slo_definition_document \u2192 assert health_check_interval_seconds, unhealthy_threshold, and healthy_threshold match documented values with zero drift" ], "human_review": [ "Review the infrastructure topology diagram for single points of failure not addressed by the AZ redundancy design, including shared database endpoints, DNS providers, certificate authorities, and model artifact storage backends", "Assess SLO definitions for alignment with business continuity requirements, confirming RTO and RPO values are derived from stakeholder impact analysis rather than infrastructure-determined defaults", "Verify that failover test scenarios reflect the most likely failure modes for the deployed infrastructure provider and are re-executed after each major infrastructure change rather than treated as one-time certifications" ], "blocking_effect": "blocks-deployment", "normative_status": "voluntary-standard", "anti_patterns": [ "Deploying AI inference endpoints to a single availability zone without automated failover, creating a hard dependency that makes the entire AI capability unavailable during zone-level outages", "Defining SLOs based on infrastructure vendor default uptime commitments rather than deriving them from measured user impact and business continuity requirements", "Configuring health checks with unhealthy_threshold values so high that the load balancer continues routing traffic to a degraded instance for multiple minutes before failover triggers", "Relying on manual runbook execution for AZ failover rather than automated health-check-driven promotion, introducing human reaction time into the RTO calculation", "Treating initial deployment across multiple AZs as sufficient without periodic failover testing, allowing failover automation to silently drift out of operational readiness over time" ], "update_status": "current", "layer_code": "RE" }, { "id": "RE-02", "layer": "RE", "plane": "control", "name": "Stateless Agent Design and Session Recovery", "plain": "AI agents must be designed to externalize session state to durable stores, enabling any agent instance to resume interrupted tasks from a checkpoint without requiring the original process to remain alive.", "threat": { "tags": [ "session-state-loss", "non-idempotent-execution", "workflow-corruption", "agent-unavailability" ], "desc": "Stateful agents that hold task context in process memory lose progress on any node failure or deployment event. Re-running incomplete tasks from scratch risks double-execution of side effects such as financial transactions or data writes. Without explicit checkpoint and recovery design, partial failures produce inconsistent system states that are difficult to detect and remediate in production." }, "standard": [ { "id": "google_sre", "section": "Ch. 26", "title": "Data Integrity \u2014 durable state and recovery" }, { "id": "aws_reliability", "section": "REL 12", "title": "Test reliability \u2014 failure injection validates stateless recovery" }, { "id": "nist_800_160_v2", "section": "\u00a72.1.3", "title": "Substantiated Integrity (cyber resiliency technique)" }, { "id": "microsoft_azure_resil", "section": "Checkpoint/Restart", "title": "Durable Function checkpoint and replay pattern" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RE-02 Stateless Agent Design and Session Recovery control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RE-02 Stateless Agent Design and Session Recovery control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RE-02 Stateless Agent Design and Session Recovery control.", "reviewed_on": "2026-07-01" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RE-02 Stateless Agent Design and Session Recovery control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RE-02 Stateless Agent Design and Session Recovery control.", "reviewed_on": "2026-07-02" }, { "id": "cloudflare_resilience_2024", "title": "Cloudflare DDoS Protection", "authority": "Cloudflare, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.cloudflare.com/ddos/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "cloudflare_resilience_2024", "relationship": "informative_reference", "rationale": "Establishes Cloudflare DDoS Protection requirements informing the apeiris://resilience/controls/RE-02 Stateless Agent Design and Session Recovery control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Externalize all agent task state to a durable key-value or workflow store (e.g., Redis, DynamoDB, Azure Durable Functions). Use idempotency keys and at-least-once delivery with deduplication to safely replay interrupted steps.", "steps": [ "Audit all production AI agents and identify where ephemeral process state is used to track task progress.", "Refactor agents to write checkpoint records to an external durable store at each meaningful step boundary.", "Implement idempotency keys for all side-effecting operations including API calls, database writes, and message publishes.", "Build a recovery harness that detects interrupted tasks via heartbeat timeout and resumes from the last durable checkpoint.", "Test recovery by injecting node terminations during active agent runs and validating state consistency post-recovery." ], "site_reliability": { "summary": "Stateless design is the single most impactful architectural decision for AI agent resilience. Every agent that holds process state is a reliability liability.", "actions": [ "Map all agent process-state dependencies and prioritize refactoring to external state stores.", "Implement heartbeat and timeout-based dead task detection with configurable escalation.", "Run chaos experiments (node kill, network partition) against agent workflows to validate checkpoint-based recovery." ], "failure_signals": [ "Agent tasks not resumable after unexpected node termination.", "No checkpoint records observable in the external state store during active agent runs.", "Recovery tests not scheduled or not passing in the last 90 days." ] }, "security_architect": { "summary": "External state stores must be protected with equivalent controls to the agent runtime \u2014 encryption at rest, fine-grained access control, and audit logging.", "actions": [ "Verify that state stores enforce encryption at rest (AES-256) and in transit (TLS 1.2+) for all checkpoint data.", "Apply least-privilege IAM policies so only the owning agent service can read and write its checkpoints.", "Ensure checkpoint records do not persist sensitive payloads such as PII or credentials beyond their required task lifetime." ], "failure_signals": [ "State stores accessible without authentication or over unencrypted connections.", "Checkpoint records retaining PII or secrets past task completion.", "No audit log configured for state store access." ] }, "it_operations": { "summary": "Operations must be able to monitor agent heartbeats and detect stalled or crashed tasks before they affect business outcomes.", "actions": [ "Deploy heartbeat monitoring for all long-running agent tasks with configurable timeout thresholds.", "Alert on tasks that have exceeded expected duration without a checkpoint update.", "Maintain runbooks for manually resuming or abandoning stalled agent tasks with defined escalation paths." ], "failure_signals": [ "No heartbeat monitoring for agent tasks running longer than 5 minutes.", "Stalled tasks not detected until end-user reports a problem.", "No runbook for manual task recovery tested within the last quarter." ] }, "grc_auditor": { "summary": "Stateless design must be verified through architecture review and validated via recovery test evidence. Idempotency controls are required for all side-effecting operations.", "actions": [ "Request agent architecture diagrams and verify external state store usage for all Tier 1 agent workflows.", "Review recovery test logs for evidence of successful checkpoint-based resumption after injected failures.", "Confirm idempotency controls are documented and tested for all side-effecting agent actions." ], "metrics": [ "Percentage of Tier 1 agent workflows with verified external state store: target 100%.", "Recovery test success rate: \u2265 95% of injected failure scenarios resolved without data loss.", "Mean time to recover from agent node failure: < 2 minutes per defined RTO." ], "failure_signals": [ "Tier 1 agent workflows using in-process state with no recovery mechanism.", "No recovery test evidence in the last quarter.", "Agent tasks producing duplicate side effects after recovery in test scenarios." ] }, "business_continuity": { "summary": "Stateless agent design protects business workflows from AI infrastructure disruptions. Interrupted agent tasks should recover automatically without requiring business escalation.", "actions": [ "Include AI agent workflow recovery scenarios in BCP tabletop exercises.", "Define acceptable task recovery time targets aligned with business impact thresholds.", "Confirm that stateless design evidence is reviewed in the annual BCP review cycle." ], "failure_signals": [ "AI agent failures consistently requiring manual business escalation for every occurrence.", "No defined recovery time target for interrupted agent tasks.", "AI agent workflow recovery not included in BCP test scenarios." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Most AI agent deployments are prototype-era and retain significant in-process state; explicit checkpoint design is uncommon." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "cloud-native", "federated-enterprise", "multi-tenant", "high-risk-sector" ], "implementers": [ "Platform Engineering", "AI/ML Engineering", "Site Reliability Engineering" ], "frameworks": [ { "framework": "google_sre", "requirement_id": "Ch. 26", "fit": "partial", "rationale": "Google SRE (SRE Book, 2016) Chapter 26, Data Integrity, covers keeping durable state trustworthy and recoverable. Stateless agent design externalizes session and task state into durable stores, placing recovery of agent context squarely under data-integrity practice; the SRE Book has no chapter on agent behavioral consistency itself.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "aws_reliability", "requirement_id": "REL 12", "fit": "direct", "rationale": "AWS Well-Architected Reliability Pillar REL 12 (Test reliability) covers fault-injection testing, the primary validation mechanism for stateless agent design: chaos experiments that terminate agent nodes verify that sessions resume from externalized state (REL 9 covers data backup).", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "nist_800_160_v2", "requirement_id": "\u00a72.1.3", "fit": "direct", "rationale": "NIST SP 800-160 Vol 2 Rev 1 \u00a72.1.3 defines Substantiated Integrity \u2014 confirming that critical state has not been corrupted \u2014 among its cyber resiliency techniques (\u00a73.2 covers the analysis process, not this construct). Externalized, idempotent, checkpointed agent state is how AI agents maintain provably correct state across failures.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "microsoft_azure_resil", "requirement_id": "Durable Functions checkpoint", "fit": "direct", "rationale": "Microsoft Azure Durable Functions implements the checkpoint-and-replay pattern that directly exemplifies the stateless agent design principle. The framework provides concrete implementation guidance for externalized workflow state with automatic recovery on host restart or failure. Azure's durable orchestration guidance is directly applicable to cloud-native AI agent architectures.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "iso_22301", "requirement_id": "\u00a78.4.4", "fit": "adjacent", "rationale": "ISO 22301 \u00a78.4.4 requires that business continuity plans address the resumption of activities at a defined minimum service level. Stateless agent design is the technical mechanism that enables AI-driven activities to resume meeting this requirement after infrastructure failure. Evidence of checkpoint-based recovery directly supports ISO 22301 audit demonstrations.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cloudflare_resilience", "requirement_id": "Cloudflare Resilience \u2014 Edge resilience patterns: stateless worker design enabling transparent load balancing and failover", "fit": "partial", "rationale": "Cloudflare's edge resilience patterns explicitly favor stateless worker designs where request and task context is externalized to durable stores, enabling load balancing to distribute agent invocations across any available instance \u2014 a deployment model that directly reinforces the Stateless Agent Design and Session Recovery control's requirement for agent instances to be interchangeable. Cloudflare's load balancing and failover capabilities route agent requests to healthy instances after node failures and the edge resilience documentation guides how stateless workers should be designed to support transparent failover without session continuity dependencies. The alignment between Cloudflare's stateless edge architecture guidance and this control's externalizing-state mandate makes Cloudflare a relevant implementation reference for AI agents deployed at the edge.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RE-02", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "AI agent task state must be externalized to a durable store at each checkpoint such that any agent instance can resume an interrupted workflow from the latest checkpoint without requiring the original process to remain alive. Checkpoint write must be confirmed durable before the agent proceeds to the next non-idempotent action.", "evidence_required": [ "state_externalization_design_document identifying every in-process state variable for each agent type and mapping each to its durable store key, schema, and checkpoint_frequency", "idempotency_key_audit_report confirming all non-idempotent agent actions are gated on idempotency keys stored in a durable deduplication log with ttl and dedup_window_seconds", "checkpoint_restore_test_record showing agent resumption from checkpoint after simulated node failure with fields task_id, checkpoint_id, state_fields_restored, and resume_latency_ms", "workflow_store_durability_configuration artifact showing replication_factor, write_concern, and persistence_guarantee for the chosen durable store", "agent_session_recovery_log from production or staging showing at least 5 successful task resumptions from checkpoint with no duplicate action execution" ], "machine_tests": [ "Start a multi-step agent task, kill the agent process mid-task after checkpoint write confirmation \u2192 assert a new agent instance resumes from the latest checkpoint within recovery_timeout_seconds without re-executing completed steps", "Submit the same agent action twice with identical idempotency keys within dedup_window_seconds \u2192 assert the second submission returns the cached result without executing the action a second time", "Force a durable store write failure before checkpoint confirmation \u2192 assert the agent halts the current step and retries rather than proceeding to the next non-idempotent action", "Simulate durable store unavailability during agent execution \u2192 assert the agent suspends task execution and emits a structured status=suspended event rather than accumulating unexternalized state" ], "human_review": [ "Review the state externalization design document for completeness, verifying every in-process state variable is mapped to a durable store key and no agent type retains task-critical state solely in memory", "Assess the idempotency key audit report for coverage, confirming all non-idempotent actions are gated on idempotency keys and the deduplication window is long enough to cover the expected retry interval", "Verify that the checkpoint_frequency setting balances recovery granularity against durable store write overhead, and that the chosen value is validated against measured task execution patterns rather than set arbitrarily" ], "blocking_effect": "blocks-deployment", "normative_status": "voluntary-standard", "anti_patterns": [ "Retaining task context, tool call history, and intermediate results solely in agent process memory without writing checkpoints to a durable store, causing total task loss on any node failure", "Using in-memory caches or local disk as the checkpoint store instead of a replicated durable store, making checkpoint recovery dependent on the availability of the original node", "Proceeding to the next non-idempotent action before receiving durable write confirmation from the checkpoint store, creating a window where task state and execution diverge", "Implementing checkpoint restore without idempotency keys on downstream tool calls, causing non-idempotent actions to execute multiple times when a recovered agent replays the workflow from checkpoint", "Treating stateless design as a deployment-time label rather than verifying through checkpoint restore tests that all agent types can actually resume from externalized state after node failure" ], "update_status": "current", "layer_code": "RE" }, { "id": "RE-03", "layer": "RE", "plane": "data", "name": "Data Redundancy and Backup Governance", "plain": "All AI system data \u2014 training datasets, model weights, inference logs, vector indexes, and operational state \u2014 must be protected by redundant storage, scheduled backups, and verified restore tests to meet defined RPO targets.", "threat": { "tags": [ "data-loss", "backup-corruption", "unverified-restore", "rpo-breach" ], "desc": "AI systems depend on curated datasets and trained model artifacts that may take weeks to recreate. Backup processes that are never tested may fail silently, leaving organizations with no recoverable copies at the moment of disaster. Ransomware or storage corruption affecting production AI data without a verified backup path can halt AI operations for days or permanently destroy proprietary training investments." }, "standard": [ { "id": "iso_22301", "section": "\u00a78.4.5", "title": "Recovery \u2014 data restoration procedures" }, { "id": "cis_controls_v8", "section": "Control 11", "title": "Data Recovery \u2014 backup and restore processes" }, { "id": "nist_sp800_34", "section": "\u00a73.4", "title": "Backup storage and data recovery" }, { "id": "nist_csf", "section": "RC.RP-02", "title": "Recovery actions are selected, scoped, prioritized, and performed" } ], "sources": [ { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RE-03 Data Redundancy and Backup Governance control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RE-03 Data Redundancy and Backup Governance control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RE-03 Data Redundancy and Backup Governance control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RE-03 Data Redundancy and Backup Governance control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RE-03 Data Redundancy and Backup Governance control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Implement a 3-2-1 backup strategy: three copies on two media types with one offsite. Enforce immutable backup storage for AI artifacts. Schedule automated restore tests monthly and record results as compliance evidence.", "steps": [ "Classify all AI data assets \u2014 training data, model weights, inference logs, vector indexes \u2014 by criticality and assign RPO targets.", "Implement automated backups on a schedule aligned to RPO (e.g., hourly snapshots for production model state, daily for training data).", "Configure at least one backup destination as immutable object storage to resist ransomware deletion or modification.", "Deploy offsite or cross-region backup replication for all Tier 1 AI data assets.", "Automate monthly restore tests and publish results to a compliance evidence store.", "Alert on backup job failures within 15 minutes and escalate to on-call within 1 hour if unresolved." ], "site_reliability": { "summary": "Backup without tested restore is not a backup. Automate restore verification as rigorously as the backup job itself.", "actions": [ "Build backup pipelines as code and deploy them through CI/CD with the same review gates as production services.", "Instrument backup success and failure metrics and set alerts on missed schedules or restore test failures.", "Run quarterly full-restore drills for the highest-criticality AI data assets." ], "failure_signals": [ "Backup jobs failing silently with no alert triggered.", "Last verified restore test older than 30 days for any Tier 1 AI asset.", "Model weights or training data with no backup coverage." ] }, "it_operations": { "summary": "Operations teams own backup monitoring and must ensure backup health is visible in the operational dashboard.", "actions": [ "Onboard all AI backup jobs to centralized monitoring with success and failure dashboards visible to on-call staff.", "Maintain an inventory of all AI data assets with their backup schedule and last verified restore date.", "Execute and document monthly restore tests per the defined schedule and retain records for audit." ], "failure_signals": [ "AI backup jobs not listed in centralized backup monitoring.", "Restore tests not documented in the evidence store.", "Backup schedule drift beyond the defined RPO window without a change record." ] }, "grc_auditor": { "summary": "Backup governance requires evidence of scheduled execution, integrity verification, and restore testing \u2014 not just backup configuration records.", "actions": [ "Request backup job execution logs for the prior quarter and confirm all scheduled runs completed successfully.", "Review restore test records and verify that tested backup artifacts match production versions by hash.", "Confirm immutable storage is configured for at least one backup destination per Tier 1 AI asset." ], "metrics": [ "Backup job success rate: \u2265 99% of scheduled jobs completed without error.", "Restore test coverage: 100% of Tier 1 AI assets tested at least monthly.", "Recovery time in restore test: within defined RTO per asset class." ], "failure_signals": [ "Backup job success rate below 99% with no remediation record.", "No restore test evidence for any Tier 1 AI data asset in the last 30 days.", "Backup destinations lacking immutability configuration for ransomware-susceptible data." ] }, "business_continuity": { "summary": "AI data loss is a business continuity event. Ensure AI data assets are in scope for BIA and that RPO targets reflect actual business impact.", "actions": [ "Include AI data assets in the Business Impact Analysis with quantified recovery cost and data loss impact.", "Confirm RPO targets for AI data align with the maximum tolerable data loss defined in BCP documentation.", "Validate that backup and restore test results are reviewed in the annual BCP test and update cycle." ], "failure_signals": [ "AI training data and model weights not included in the BIA.", "RPO target for AI data looser than the business-defined maximum tolerable data loss.", "Backup governance outcomes not reviewed as part of the BCP annual exercise." ] }, "security_architect": { "summary": "Backups of models and AI data are high-value targets and a ransomware last line. Enforce immutability, encryption, access separation, and restore-time integrity verification in backup governance.", "actions": [ "Require immutable/offline copies and separate credentials for backup infrastructure.", "Encrypt backups with keys managed outside the backed-up environment; test key recovery.", "Verify integrity (hashes, signatures) at backup and restore time for model artifacts and training data." ], "failure_signals": [ "Backup credentials are reachable from the production environment they protect.", "Restore tests skip integrity verification of model artifacts.", "Backup encryption keys would be lost in the same incident as the data." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "AI-specific data assets such as model weights and vector indexes are often excluded from standard enterprise backup programs." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "cloud-native" ], "implementers": [ "Infrastructure Team", "IT Operations", "Data Engineering" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a78.4.5", "fit": "direct", "rationale": "ISO 22301:2019 \u00a78.4.5 (Recovery) requires documented recovery procedures \u2014 including restoring information and data \u2014 with defined recovery objectives and tested restore paths. This directly governs backup governance for AI data assets (\u00a78.4.3 covers warning and communication).", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cis_controls_v8", "requirement_id": "Control 11", "fit": "direct", "rationale": "CIS Controls v8 Control 11 defines data recovery best practices including the 3-2-1 backup strategy, immutable storage, and tested restoration procedures. These requirements apply directly to AI data assets and provide measurable safeguards for audit validation. The CIS benchmark implementation groups allow graduated adoption aligned to organization maturity.", "normative_force": "best-practice", "source_version": "v8", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.4", "fit": "direct", "rationale": "NIST SP 800-34 \u00a73.4 provides contingency planning guidance for backup storage including offsite replication and backup testing requirements. The guidance defines RPO and RTO derivation from BIA, which maps directly to the AI data classification and backup schedule design in this control. NIST 800-34 is referenced by federal AI governance frameworks for contingency planning requirements.", "normative_force": "voluntary-standard", "source_version": "Rev.1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 12", "fit": "partial", "rationale": "EU DORA Article 12 requires financial entities to maintain backup policies with defined backup frequency, testing schedules, and demonstrated recovery capability. AI systems supporting financial services must include their data assets in DORA-compliant backup governance programs. Restore test records are required audit artifacts under DORA supervisory examination.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_csf", "requirement_id": "RC.RP-02", "fit": "adjacent", "rationale": "NIST CSF 2.0 RC.RP-02 requires recovery actions to be selected, scoped, prioritized, and performed. Backup restore-test results determine which data-recovery actions are viable and how they should be prioritized during real incidents.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "AWS Well-Architected Reliability Pillar \u2014 Failure management: backup strategy, data replication, and recovery validation procedures", "fit": "direct", "rationale": "The AWS Well-Architected Reliability Pillar's failure management pillar prescribes backup strategies, cross-region data replication patterns, and automated recovery validation procedures that map directly to the 3-2-1 backup requirement and monthly verified restore testing mandate of the Data Redundancy and Backup Governance control. AWS reliability guidance specifies S3 Object Lock for immutable backup protection of AI artifacts, cross-region replication for offsite backup copies, and automated restore validation pipelines \u2014 the precise implementation patterns underpinning this control's RPO target alignment and backup integrity requirements. AWS also provides specific guidance on protecting machine learning artifacts including model weights, training datasets, and vector indexes under the Reliability Pillar's data protection practices.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Google SRE \u2014 Postmortem practices and error budget policies: data loss event analysis and backup governance improvement cycles", "fit": "partial", "rationale": "Google SRE postmortem practices mandate structured analysis of data loss and recovery failure events with findings fed back into backup governance processes \u2014 the same continuous improvement cycle that the Data Redundancy and Backup Governance control requires through its verified restore test scheduling and RPO target deviation tracking. The SRE error budget framework provides a mechanism for encoding backup verification failures as reliability budget consumption, creating operational incentives for maintaining the restore validation cadence this control mandates rather than treating backup governance as a compliance checkbox. SRE practices also prescribe the runbook documentation for data recovery procedures that this control requires as evidence of backup governance effectiveness.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "Azure Resiliency & BCDR \u2014 Geo-redundant storage design, Azure Backup policies, and recovery objective alignment for AI/ML workloads", "fit": "direct", "rationale": "Microsoft Azure Resiliency and BCDR guidance defines geo-redundant storage configurations, Azure Backup policies with immutability settings, and RPO-aligned recovery validation frameworks that directly implement the redundant storage, scheduled backup, and verified restore requirements of the Data Redundancy and Backup Governance control. Azure BCDR documentation specifically addresses AI and ML workload data protection including Azure Machine Learning model registry backup, vector index replication across regions, and automated restore validation \u2014 all of which correspond to the AI-specific data assets (model weights, vector indexes, training datasets) this control requires to be covered. The control's monthly restore test mandate maps to Azure BCDR guidance on regular recovery validation exercises needed to verify backup integrity.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RE-03", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "All AI system data assets including training datasets, model weights, inference logs, vector indexes, and operational state must be covered by a documented backup policy with defined RPO and RTO, and restore procedures must be verified through scheduled restore tests producing recorded outcomes that confirm data integrity and recovery time within policy.", "evidence_required": [ "backup_policy_document enumerating each AI data asset class with assigned backup_frequency, retention_period, storage_tier, rpo_hours, and rto_hours", "backup_configuration_audit_record confirming immutable storage is enabled for AI artifacts, replication_factor meets the policy minimum, and retention rules are enforced on the storage backend", "restore_test_record from the most recent scheduled test for each asset class with fields asset_class, backup_timestamp, restore_start_time, restore_completion_time, integrity_check_method, and integrity_check_result", "backup_monitoring_alert_log confirming backup job failures and RPO breach alerts are routed to on-call within alert_response_sla_minutes", "data_asset_inventory listing all AI data assets in scope for backup governance with coverage_status and last_backup_verified_at for each" ], "machine_tests": [ "Trigger a restore of the most recent model weights backup to an isolated environment \u2192 assert restore completes within rto_hours and SHA-256 hash of restored artifact matches the hash recorded at backup time", "Verify backup job execution timestamps for all asset classes against backup_frequency policy \u2192 assert no asset class has a gap between consecutive backups exceeding rpo_hours", "Attempt to delete or overwrite an artifact in immutable backup storage \u2192 assert the storage backend rejects the operation with an immutability error and the original artifact remains intact", "Simulate backup job failure for a vector index \u2192 assert a monitoring alert fires within alert_response_sla_minutes and the failure is logged in backup_monitoring_alert_log with asset_class and failure_reason" ], "human_review": [ "Review the data asset inventory for completeness, verifying that AI-specific asset classes such as model weights, fine-tuned adapters, vector indexes, and RLHF feedback datasets are explicitly enumerated and not excluded from backup governance as non-standard data types", "Assess restore test records for rigor, confirming that tests use actual production backups rather than synthetic test datasets, and that integrity checks validate semantic correctness beyond file-size comparison", "Verify that RPO and RTO values in the backup policy are derived from business impact analysis rather than set to convenient round numbers, and that restore test outcomes consistently meet the documented objectives" ], "blocking_effect": "blocks-deployment", "normative_status": "certification-standard", "anti_patterns": [ "Applying standard enterprise backup policies to AI systems without enumerating AI-specific data asset classes, leaving model weights, vector indexes, and training datasets outside backup governance scope", "Relying on storage-layer replication alone as a substitute for backup, which does not protect against logical corruption, accidental deletion, or ransomware affecting the primary storage tier", "Scheduling backup restore tests as annual events rather than as recurring scheduled tests, allowing restore procedures to drift out of operational readiness between audit cycles", "Recording backup job completion as evidence of successful backup without performing integrity checks on the backup artifact to verify it is not corrupted or incomplete", "Storing backups in the same storage account or blast radius as the primary data, such that an account-level compromise or regional outage affects both the primary data and the backup simultaneously" ], "update_status": "current", "layer_code": "RE" }, { "id": "RE-04", "layer": "RE", "plane": "control", "name": "Infrastructure-as-Code for Reproducible Recovery", "plain": "All AI system infrastructure must be defined as versioned code in a source control repository, enabling complete environment recreation from a known-good state within defined RTO targets without reliance on manual runbooks or tribal knowledge.", "threat": { "tags": [ "configuration-drift", "snowflake-server", "recovery-delay", "unrecoverable-environment" ], "desc": "Manually configured AI infrastructure drifts from its intended state over time, making recovery unpredictable and error-prone. When a production environment must be rebuilt after a disaster, undocumented configuration differences cause failures that extend recovery time far beyond RTO targets. Without IaC, each rebuild is a unique engineering exercise dependent on individual knowledge rather than a deterministic reproducible process." }, "standard": [ { "id": "nist_sp800_34", "section": "\u00a73.3", "title": "Alternate site and system reconstitution procedures" }, { "id": "aws_reliability", "section": "REL 12", "title": "Test reliability \u2014 exercise automated recovery procedures" }, { "id": "nist_800_160_v2", "section": "\u00a72.1.3", "title": "Adaptive Response (cyber resiliency technique)" }, { "id": "cobit_dss04", "section": "DSS04.03", "title": "Develop and implement a business continuity response" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RE-04 Infrastructure-as-Code for Reproducible Recovery control.", "reviewed_on": "2026-07-01" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RE-04 Infrastructure-as-Code for Reproducible Recovery control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RE-04 Infrastructure-as-Code for Reproducible Recovery control.", "reviewed_on": "2026-07-02" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RE-04 Infrastructure-as-Code for Reproducible Recovery control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RE-04 Infrastructure-as-Code for Reproducible Recovery control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Store all AI infrastructure definitions \u2014 compute, networking, storage, IAM \u2014 in a version-controlled IaC repository. Enforce plan-before-apply policies in CI/CD. Conduct quarterly IaC-only recovery drills in an isolated environment.", "steps": [ "Audit current AI infrastructure and identify all manually configured components not represented in IaC.", "Migrate all manual configurations to IaC (Terraform, Pulumi, or cloud-native CDK) within a defined remediation timeline.", "Enforce branch protection and peer review on all IaC changes; block direct console modifications via SCP or IAM deny policy.", "Tag all IaC modules with the control version and last tested recovery date.", "Conduct quarterly IaC recovery drills by deploying a complete environment from scratch in an isolated account and measuring RTO achieved.", "Record IaC drill results including RTO achieved, configuration gaps found, and remediation actions taken with owners." ], "site_reliability": { "summary": "IaC is the only reliable path to deterministic recovery. Any infrastructure not in code is a liability when rebuilding under time pressure during an incident.", "actions": [ "Achieve 100% IaC coverage for all production AI infrastructure; track coverage as a metric reported to engineering leadership.", "Automate drift detection via Terraform plan in read-only mode on a schedule and alert on detected drift.", "Run IaC recovery drill quarterly and record RTO achieved against the defined target." ], "failure_signals": [ "Production AI infrastructure components not represented in any IaC module.", "Drift detected between IaC state and live environment without a remediation ticket open.", "IaC recovery drill not completed in the last 90 days." ] }, "security_architect": { "summary": "IaC enforces security baseline by design. Any security configuration not captured in code can be silently undone by manual change or configuration drift.", "actions": [ "Require security policy-as-code gates (OPA, Sentinel) in IaC CI/CD pipelines to prevent non-compliant deployments.", "Ensure IAM and network ACL configurations are exclusively managed via IaC with no console override path permitted.", "Review IaC modules for hardcoded secrets and enforce use of secrets management references throughout." ], "failure_signals": [ "Security groups or IAM policies modified directly in the cloud console without a corresponding IaC commit.", "Hardcoded credentials found in IaC module source during review.", "Policy-as-code gate not present in IaC CI/CD pipeline." ] }, "it_operations": { "summary": "Operations teams benefit from IaC by having a documented, tested procedure for environment recreation that does not depend on any single engineer's knowledge.", "actions": [ "Maintain an IaC module inventory with owner, last tested date, and RTO achievement per module.", "Train at least two operations staff on IaC deployment procedures so recovery is not dependent on a single person.", "Document the end-to-end recovery procedure in the runbook with references to the IaC repository and deployment commands." ], "failure_signals": [ "Recovery procedure depends on knowledge held by fewer than two trained staff members.", "IaC module inventory not maintained or more than 30 days out of date.", "Operations staff unable to execute IaC recovery without developer assistance during drill." ] }, "grc_auditor": { "summary": "IaC coverage and recovery drill results are the primary evidence that environments are recoverable to a known-good state within defined RTO targets.", "actions": [ "Request IaC repository access and verify all Tier 1 AI infrastructure is represented with current module versions.", "Review recovery drill records for the prior two quarters and confirm RTO targets were met.", "Verify IaC change history and confirm all production changes originated from IaC commits with peer review." ], "metrics": [ "IaC coverage of Tier 1 AI infrastructure: target 100%.", "IaC recovery drill RTO achieved vs. target: \u2264 defined RTO in 90% of drills.", "Drift incidents resolved within defined SLA: target 100% within 24 hours of detection." ], "failure_signals": [ "IaC coverage below 80% for Tier 1 AI infrastructure.", "Recovery drill RTO exceeding defined target with no remediation plan documented.", "Production changes without corresponding IaC commits identified in the prior quarter." ] }, "business_continuity": { "summary": "IaC recovery drills are the most rigorous test of business continuity for AI infrastructure. Include drill results in BCP reporting to executive governance.", "actions": [ "Include IaC recovery drill results in the annual BCP review and board operational resilience reporting.", "Confirm RTO targets demonstrated in IaC drills match or exceed BCP commitments for AI-critical services.", "Ensure the IaC repository itself is backed up and accessible from a DR site independent of primary production." ], "failure_signals": [ "IaC recovery drill results not reviewed in the annual BCP cycle.", "IaC repository itself not backed up or accessible from DR site.", "BCP RTO targets more aggressive than what IaC drills demonstrate is actually achievable." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "AI infrastructure is frequently provisioned via console click-ops during rapid experimentation phases and never migrated to IaC before production launch." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "federated-enterprise" ], "implementers": [ "Platform Engineering", "DevOps / Infrastructure Engineering", "Site Reliability Engineering" ], "frameworks": [ { "framework": "nist_sp800_34", "requirement_id": "\u00a73.3", "fit": "direct", "rationale": "NIST SP 800-34 \u00a73.3 covers alternate site activation and system reconstitution, requiring documented and tested procedures to rebuild systems from known-good configurations. IaC provides the mechanism for deterministic reconstitution that NIST 800-34 requires. Recovery drill results serve as the contingency plan test evidence required by this standard.", "normative_force": "voluntary-standard", "source_version": "Rev.1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "REL 12", "fit": "partial", "rationale": "AWS Well-Architected Reliability Pillar REL 12 (Test reliability) requires recovery procedures \u2014 including IaC-driven environment rebuilds \u2014 to be exercised regularly so automation works when needed; prescriptive runbook guidance lives in the Operational Excellence pillar. IaC recovery drills implement this testing discipline (REL 8 covers change implementation).", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "nist_800_160_v2", "requirement_id": "\u00a72.1.3", "fit": "direct", "rationale": "NIST SP 800-160 Vol 2 Rev 1 \u00a72.1.3 defines Adaptive Response \u2014 taking timely, appropriate action in response to adverse conditions, including reconfiguration and restoration from known-good definitions \u2014 among its cyber resiliency techniques. IaC-based environment reconstruction is a direct realization of that technique.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.03", "fit": "partial", "rationale": "COBIT 2019 DSS04.03 requires developing and implementing a business continuity response, including documented recovery procedures for IT services. IaC-based rebuild procedures are the modern implementation of those documented IT recovery procedures (DSS04.05 covers plan review, not development).", "normative_force": "best-practice", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 11(6)", "fit": "partial", "rationale": "EU DORA Article 11(6) requires financial entities to test their ICT business continuity plans and ICT response and recovery plans at least yearly, covering critical functions. IaC recovery drills that rebuild AI environments from code are a rigorous way to satisfy that testing requirement.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "google_sre", "requirement_id": "Google SRE \u2014 Production readiness review and change management: controlled, auditable infrastructure change processes", "fit": "partial", "rationale": "Google SRE production readiness review criteria include verification that service infrastructure can be reliably reproduced and that all changes are made through controlled, auditable processes \u2014 requirements the Infrastructure-as-Code for Reproducible Recovery control addresses by mandating version-controlled IaC with CI/CD plan-before-apply enforcement as the only permitted infrastructure change mechanism. SRE change management practices rely on the deterministic, repeatable infrastructure recreation that IaC enables, supporting PRR-gated deployment gates and quarterly recovery drills to validate that the AI production environment can be rebuilt from code alone within defined RTO targets. The SRE postmortem practice of identifying configuration drift as a contributing factor to incidents is directly mitigated by the IaC-mandated configuration consistency this control enforces.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "Azure Resiliency & BCDR \u2014 infrastructure-as-code recovery (ARM templates and Bicep)", "fit": "direct", "rationale": "Microsoft Azure Resiliency and BCDR design guidance explicitly recommends infrastructure-as-code \u2014 specifically Azure Resource Manager templates and Bicep definitions \u2014 as the implementation mechanism for automated disaster recovery environment provisioning, directly prescribing the version-controlled infrastructure approach the Infrastructure-as-Code for Reproducible Recovery control mandates for AI systems. Azure BCDR documentation prescribes IaC-driven failover environment creation as the standard for meeting defined RTO targets, since manual environment rebuilds introduce variability that cannot reliably satisfy recovery time objectives. The control's requirement for complete AI environment recreation from code within RTO targets aligns directly with Azure BCDR failover configuration guidance that uses IaC as the authoritative environment specification.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RE-04", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "All Tier 1 AI system infrastructure must be 100% defined in a version-controlled IaC repository with zero unresolved drift between IaC state and live configuration. Quarterly IaC-only recovery drills must demonstrate successful environment recreation from code alone within the defined RTO target using no manual steps.", "evidence_required": [ "IaC coverage report listing every Tier 1 AI infrastructure component mapped to an IaC module with module name, version tag, and last-tested-recovery date", "Drift detection scan report (Terraform plan read-only output or equivalent) confirming zero unresolved drift findings between IaC state and deployed configuration at time of evidence collection", "IaC recovery drill record for the prior quarter including isolated environment name, start timestamp, end timestamp, RTO achieved vs. target, configuration gaps discovered, and remediation actions with named owners", "CI/CD pipeline policy configuration confirming plan-before-apply enforcement and branch protection rules with peer-review requirement; plus SCP or IAM deny policy blocking direct console modifications", "IaC change history export for the prior quarter confirming all production infrastructure changes originated from IaC commits with peer-review approval" ], "machine_tests": [ "Run terraform plan in read-only mode against production AI environment \u2192 assert exit code indicates zero drift findings between IaC state and live configuration", "Attempt direct console modification of an IaC-managed AI resource \u2192 assert SCP or IAM deny policy blocks the action and generates a CloudTrail or audit log alert within 5 minutes", "Deploy full Tier 1 AI environment from IaC in an isolated account with timer running \u2192 assert environment reaches operational state and RTO target is met or exceeded", "Submit IaC pull request containing a hardcoded credential string \u2192 assert policy-as-code gate (OPA or Sentinel) rejects the PR before merge" ], "human_review": [ "Review IaC module inventory for completeness against the current production AI infrastructure topology and verify all Tier 1 components have a named IaC owner, current module version, and last-tested recovery date within 90 days", "Assess IaC recovery drill records from the prior two quarters to confirm RTO targets were met, configuration gaps were documented, and remediation plans have named owners and realistic timelines", "Verify that the IaC repository itself is stored in a system that is backed up and accessible from a DR site independent of primary production infrastructure" ], "blocking_effect": "blocks-deployment", "normative_status": "voluntary-standard", "anti_patterns": [ "Provisioning AI inference infrastructure via cloud console click-ops during rapid experimentation and promoting it to production without ever migrating the configuration to IaC", "Maintaining IaC definitions that have drifted from live configuration because emergency hotfixes were applied directly in the console without a corresponding IaC commit or drift remediation ticket", "Treating IaC recovery drills as documentation walkthroughs rather than timed, isolated environment recreation tests where RTO achievement is measured and recorded as evidence", "Permitting a single engineer to hold exclusive knowledge of the IaC deployment procedure, creating a recovery single point of failure whenever that individual is unavailable during a disaster", "Allowing IaC modules to reference secrets as hardcoded literal values rather than parameterized secrets management system references, leaving credentials exposed in version control history" ], "update_status": "current", "layer_code": "RE" }, { "id": "RE-05", "layer": "RE", "plane": "control", "name": "Dependency Isolation and Bulkhead Patterns", "plain": "AI system components must be isolated from each other and from shared infrastructure using bulkhead patterns, circuit breakers, and resource quota enforcement so that failure or overload in one component cannot propagate to others.", "threat": { "tags": [ "cascading-failure", "resource-exhaustion", "blast-radius-expansion", "noisy-neighbor" ], "desc": "Tightly coupled AI system components share thread pools, database connections, or API rate limits in ways that allow one degraded component to exhaust shared resources and bring down adjacent healthy services. A single slow model inference call holding a shared connection pool can cascade into a full-service outage. Without explicit isolation boundaries, the blast radius of any failure expands unpredictably across unrelated services." }, "standard": [ { "id": "google_sre", "section": "Ch. 21", "title": "Handling overload with load shedding and bulkheads" }, { "id": "aws_reliability", "section": "REL 11", "title": "Design your workload to withstand component failures" }, { "id": "nist_800_160_v2", "section": "\u00a72.1.3", "title": "Segmentation (cyber resiliency technique)" }, { "id": "microsoft_azure_resil", "section": "Bulkhead pattern", "title": "Isolate elements to prevent cascading failures" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RE-05 Dependency Isolation and Bulkhead Patterns control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RE-05 Dependency Isolation and Bulkhead Patterns control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RE-05 Dependency Isolation and Bulkhead Patterns control.", "reviewed_on": "2026-07-01" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RE-05 Dependency Isolation and Bulkhead Patterns control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RE-05 Dependency Isolation and Bulkhead Patterns control.", "reviewed_on": "2026-07-02" }, { "id": "cloudflare_resilience_2024", "title": "Cloudflare DDoS Protection", "authority": "Cloudflare, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.cloudflare.com/ddos/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "cloudflare_resilience_2024", "relationship": "informative_reference", "rationale": "Establishes Cloudflare DDoS Protection requirements informing the apeiris://resilience/controls/RE-05 Dependency Isolation and Bulkhead Patterns control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Implement bulkheads as separate thread pools, container resource quotas, and dedicated connection pools per service tier. Deploy circuit breakers at all inter-service boundaries with configurable failure thresholds and half-open recovery probes.", "steps": [ "Map all AI system inter-service dependencies and identify shared resource pools including thread pools, DB connections, and API quotas.", "Assign each service tier its own resource quota \u2014 CPU, memory, connections \u2014 enforced at the container or process level.", "Deploy circuit breakers at all cross-service call sites with failure threshold, timeout, and half-open probe configuration.", "Implement fallback behaviors for each circuit-broken dependency including graceful degradation, cached response, or explicit error.", "Test bulkhead isolation by injecting artificial latency and error rates into one service and confirming adjacent services remain healthy.", "Monitor circuit breaker open and close state and set alerts on sustained open state indicating dependency health degradation." ], "site_reliability": { "summary": "Bulkheads are the primary tool for bounding blast radius in AI systems. Every inter-service dependency is a potential failure vector that requires a circuit breaker.", "actions": [ "Audit all inter-AI-service dependencies and confirm circuit breakers are deployed at each call site.", "Define and enforce resource quotas per service in Kubernetes namespace or equivalent isolation boundary.", "Run failure injection tests quarterly to validate bulkhead isolation effectiveness under realistic load." ], "failure_signals": [ "Inter-service calls without circuit breaker protection identified during audit.", "Shared thread pools or connection pools visible across multiple AI service tiers.", "Failure injection in one AI service causing unrelated services to degrade." ] }, "security_architect": { "summary": "Dependency isolation is also a security control \u2014 it limits lateral movement and privilege escalation pathways through service compromise.", "actions": [ "Verify that network policies enforce bulkhead boundaries at the network layer in addition to the application layer.", "Ensure that a compromised service cannot exceed its resource quota to deny service to co-located components.", "Review that circuit breaker fallback behaviors do not expose sensitive data or internal system details in error responses." ], "failure_signals": [ "No network-layer policy enforcing service isolation boundaries between AI components.", "A compromised service able to consume unlimited shared resources without quota enforcement.", "Circuit breaker fallback responses containing internal error details or stack traces." ] }, "it_operations": { "summary": "Operations must monitor bulkhead health and circuit breaker state to detect isolation failures before they cascade into business impact.", "actions": [ "Instrument all circuit breakers and export state (closed, open, half-open) to monitoring dashboards visible to on-call staff.", "Alert when any Tier 1 service circuit breaker has been in open state for more than 5 minutes.", "Maintain a dependency map updated whenever new inter-service AI integrations are deployed to production." ], "failure_signals": [ "Circuit breaker state not visible in monitoring dashboards.", "Sustained circuit-open state discovered reactively after business impact rather than by alert.", "Dependency map not updated after recent service additions." ] }, "grc_auditor": { "summary": "Bulkhead and circuit breaker design should be evidenced through architecture review records and failure injection test results.", "actions": [ "Request architecture diagrams and verify bulkhead patterns are shown at all cross-service AI boundaries.", "Review failure injection test records confirming isolation effectiveness under load.", "Confirm resource quota enforcement is configured and visible in deployment manifests or namespace definitions." ], "metrics": [ "Percentage of Tier 1 inter-service dependencies with circuit breakers: target 100%.", "Failure injection test coverage: all Tier 1 bulkhead boundaries tested at least annually.", "Time to detect sustained circuit-open state: < 5 minutes via monitoring alert." ], "failure_signals": [ "Tier 1 AI service dependencies without circuit breaker protection identified in review.", "No failure injection test evidence for bulkhead design in the last 12 months.", "Resource quotas not enforced at the container or process level in production." ] }, "business_continuity": { "summary": "Bulkhead design protects business-critical AI services from collateral damage when adjacent systems fail. Include isolation architecture assumptions in BCP impact scenarios.", "actions": [ "Include cascading failure scenarios in BCP tabletop exercises to test bulkhead assumptions.", "Confirm that BCP identifies which AI services have isolation guarantees and which are shared-fate with other systems.", "Review bulkhead test results as part of the annual BCP review cycle." ], "failure_signals": [ "BCP not accounting for cascading failure scenarios across AI service boundaries.", "No distinction in BCP between isolated and shared-fate AI services.", "Bulkhead test results not reviewed in the annual BCP exercise." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Circuit breaker and bulkhead patterns are well-understood in general software architecture but are rarely applied systematically to AI service architectures." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "cloud-native", "high-risk-sector", "federated-enterprise", "multi-tenant" ], "implementers": [ "Platform Engineering", "Site Reliability Engineering", "AI/ML Engineering" ], "frameworks": [ { "framework": "google_sre", "requirement_id": "Ch. 21", "fit": "direct", "rationale": "Google SRE Chapter 21 covers handling overload through load shedding, bulkheads, and circuit breakers as primary resilience patterns for production services. The chapter provides the operational definitions and implementation guidance that underpin this control. SRE error budget policies create the incentive structure for investing in bulkhead design.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "aws_reliability", "requirement_id": "REL 11", "fit": "direct", "rationale": "AWS Well-Architected Reliability Pillar REL 11 (Design your workload to withstand component failures) recommends bulkhead architectures and fault-isolated boundaries so failures are contained rather than propagated. Dependency isolation for AI pipelines implements this requirement.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "nist_800_160_v2", "requirement_id": "\u00a72.1.3", "fit": "direct", "rationale": "NIST SP 800-160 Vol 2 Rev 1 \u00a72.1.3 defines Segmentation among its cyber resiliency techniques, with Appendix D structural principles ('Contain and exclude behaviors', 'Layer defenses and partition resources') giving the design rules. Bulkhead patterns create exactly those failure-containment boundaries.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "microsoft_azure_resil", "requirement_id": "Bulkhead pattern", "fit": "direct", "rationale": "Microsoft Azure cloud design patterns documentation provides the canonical definition and implementation guidance for the bulkhead pattern including thread pool isolation, semaphore-based bulkheads, and container resource quotas. The Azure guidance covers concrete implementation mechanisms applicable to cloud-native AI architectures. This framework provides the reference implementation specification for this control.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "dora", "requirement_id": "Art. 9(4)", "fit": "adjacent", "rationale": "EU DORA Article 9(4) requires financial entities to implement controls to prevent ICT risks from propagating across ICT systems and networks. Bulkhead isolation patterns are the architectural mechanism that implements this propagation prevention requirement for AI system components. DORA operational resilience testing must validate that isolation boundaries contain failures as designed without cross-service cascade.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "cloudflare_resilience", "requirement_id": "Cloudflare Resilience \u2014 Rate limiting for API protection and DDoS protection layers: per-client resource quota enforcement for AI inference endpoints", "fit": "direct", "rationale": "Cloudflare's rate limiting capabilities enforce per-client and per-endpoint resource quotas at the network edge that implement bulkhead-equivalent isolation between AI inference API consumers, directly complementing the Dependency Isolation and Bulkhead Patterns control's requirement to prevent any single consumer from exhausting shared inference capacity. Cloudflare DDoS protection layers provide a distributed enforcement mechanism for the blast-radius containment principle that the bulkhead pattern implements internally \u2014 ensuring that volumetric attack traffic against one AI inference endpoint does not exhaust resources for other endpoints serving legitimate requests. Cloudflare's edge resilience patterns for AI API protection specifically address the noisy-neighbor and resource-exhaustion failure modes that this control's circuit breaker and resource quota requirements are designed to prevent.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RE-05", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every cross-service call path in the AI system must have an enforced circuit breaker with configured failure threshold, timeout, and half-open probe, and each service tier must have container-level resource quotas preventing shared-resource exhaustion. Failure injection tests must confirm that sustained overload or error injection in one component does not degrade adjacent healthy services.", "evidence_required": [ "Inter-service dependency map listing every AI service call path with circuit breaker configuration record showing failure threshold, timeout duration, half-open probe interval, and fallback behavior per dependency", "Container resource quota manifest confirming CPU, memory, and connection pool limits are enforced per service tier in production namespaces or equivalent isolation boundaries", "Failure injection test report confirming artificial latency and error rate injection into one AI inference service did not degrade adjacent services, including service-level metrics before, during, and after injection with timestamps", "Circuit breaker monitoring alert configuration showing sustained open-state alerts with sub-5-minute detection SLA, plus sample alert delivery confirmation from the last test" ], "machine_tests": [ "Inject 100% error rate into a single AI inference service for 60 seconds \u2192 assert circuit breaker opens within configured failure threshold and adjacent services maintain error rate within SLO budget", "Set one AI service to consume 100% of its defined CPU and memory quota \u2192 assert container resource enforcement prevents consumption from spilling into adjacent service namespaces and those services maintain availability", "Attempt to deploy an AI service without a circuit breaker configured on its outbound dependency calls \u2192 assert policy gate blocks the deployment and generates a finding", "Simulate AI model inference latency at 5x normal response time \u2192 assert circuit breaker opens before shared thread pool exhaustion and defined fallback behavior is invoked within configured timeout" ], "human_review": [ "Review the inter-service dependency map for completeness against the current production AI system topology and confirm circuit breakers are deployed at all Tier 1 cross-service call sites including any recently added AI service integrations", "Assess circuit breaker fallback behaviors for each Tier 1 dependency and confirm they provide graceful degradation without exposing internal stack traces or system details in error responses", "Evaluate failure injection test results for realism and confirm test scenarios represent production-representative load levels rather than trivial synthetic failure conditions that would not reveal shared-resource contention" ], "blocking_effect": "blocks-deployment", "normative_status": "voluntary-standard", "anti_patterns": [ "Sharing thread pools and database connection pools across multiple AI service tiers so that a slow or hung inference call can exhaust connections for unrelated services co-located on the same infrastructure", "Implementing circuit breakers only at the external API gateway layer and omitting them at individual service-to-service call sites within the AI system where failure propagation actually originates", "Configuring circuit breaker failure thresholds so conservatively high that they never open under realistic production failure conditions, rendering the protection mechanism effectively inactive", "Treating container resource quotas as advisory soft limits rather than enforced hard limits, allowing AI inference services to burst beyond their allocation during peak load and starve adjacent services", "Performing failure injection tests only with minimal synthetic traffic rather than under production-representative concurrent load that would reveal shared-resource contention patterns invisible at low concurrency" ], "update_status": "current", "layer_code": "RE" }, { "id": "RE-06", "layer": "RE", "plane": "both", "name": "DORA Compliance Architecture for AI Systems", "plain": "AI systems deployed in or supporting financial services must be architecturally designed to meet EU Digital Operational Resilience Act (DORA) requirements for ICT risk management, operational resilience testing, third-party risk governance, and incident reporting.", "threat": { "tags": [ "regulatory-non-compliance", "ict-risk-failure", "third-party-dependency-failure", "supervisory-action" ], "desc": "DORA imposes binding obligations on financial entities and their ICT third-party service providers regarding AI system resilience that, if unmet, expose organizations to supervisory sanctions and mandatory remediation orders. AI systems that lack DORA-compliant architecture create audit gaps discoverable during supervisory examinations. Third-party AI providers not assessed under DORA's subcontracting rules introduce unquantified ICT concentration risk that the enterprise may not discover until an outage occurs." }, "standard": [ { "id": "dora", "section": "Art. 6-16", "title": "ICT risk management framework requirements" }, { "id": "iso_22301", "section": "\u00a78.4", "title": "Business continuity procedures and testing" }, { "id": "cobit_dss04", "section": "DSS04.02", "title": "Determine and ensure business continuity requirements" }, { "id": "nist_sp800_34", "section": "\u00a72.1", "title": "IT contingency planning policy and objectives" } ], "sources": [ { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RE-06 DORA Compliance Architecture for AI Systems control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RE-06 DORA Compliance Architecture for AI Systems control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RE-06 DORA Compliance Architecture for AI Systems control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RE-06 DORA Compliance Architecture for AI Systems control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RE-06 DORA Compliance Architecture for AI Systems control.", "reviewed_on": "2026-07-01" }, { "id": "cloudflare_resilience_2024", "title": "Cloudflare DDoS Protection", "authority": "Cloudflare, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.cloudflare.com/ddos/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "cloudflare_resilience_2024", "relationship": "informative_reference", "rationale": "Establishes Cloudflare DDoS Protection requirements informing the apeiris://resilience/controls/RE-06 DORA Compliance Architecture for AI Systems control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Perform a DORA gap assessment against the AI system architecture. Map all DORA requirements \u2014 ICT risk management, TLPT, incident reporting, third-party risk \u2014 to specific architectural controls. Document the DORA compliance posture in the ICT risk management framework.", "steps": [ "Identify all AI systems in scope for DORA by determining which support or are integrated with financial entity operations.", "Conduct a DORA Article 6-16 gap assessment for each in-scope AI system, documenting findings and assigning remediation owners.", "Map AI system architecture to DORA's ICT risk management functions in Articles 8-14: identification (Art. 8), protection and prevention (Art. 9), detection (Art. 10), response and recovery (Art. 11), backup and restoration (Art. 12), learning and evolving (Art. 13), and communication (Art. 14).", "Assess all AI third-party providers \u2014 model APIs, cloud infrastructure, data services \u2014 under DORA Article 28-30 third-party risk rules.", "Design and schedule DORA-required threat-led penetration testing (TLPT) for Tier 1 AI systems including AI-specific attack scenarios.", "Establish DORA-compliant major ICT incident classification criteria for AI system failures and test the end-to-end reporting pipeline.", "Document DORA compliance architecture in a register reviewed by the CISO and Board Risk Committee at least annually." ], "site_reliability": { "summary": "DORA requires tested operational resilience with evidence. SRE teams must design AI systems with DORA-compliant RTO and RPO targets and participate in TLPT exercises.", "actions": [ "Map AI service SLOs to DORA recovery time objectives for critical functions and confirm alignment.", "Participate in TLPT design by providing AI system architecture diagrams and operationally relevant threat scenarios.", "Implement DORA-required incident detection and response logging for all AI system events meeting classification thresholds." ], "failure_signals": [ "AI system SLOs not mapped to DORA recovery time objectives for critical functions.", "SRE team not involved in TLPT planning for AI systems.", "Incident detection pipeline not meeting DORA classification and reporting timeline requirements." ] }, "security_architect": { "summary": "DORA mandates security architecture controls across the full ICT risk lifecycle for AI systems operating in financial services contexts.", "actions": [ "Ensure AI system security architecture covers all DORA Article 9 protection requirements including access control, encryption, and patch management.", "Design DORA-compliant network segmentation for AI components supporting critical financial functions.", "Review third-party AI provider contracts for DORA Article 30 required contractual provisions on resilience and audit access." ], "failure_signals": [ "AI system architecture not reviewed against DORA Article 9 protection requirements.", "Third-party AI provider contracts lacking DORA Article 30 mandatory provisions.", "Network segmentation for AI critical functions not documented or tested against DORA requirements." ] }, "grc_auditor": { "summary": "DORA compliance requires documented evidence across all five ICT risk management pillars for AI systems in scope. Internal audit must assess DORA posture at least annually.", "actions": [ "Conduct annual internal audit of AI system DORA compliance posture using EBA and ESMA supervision guidelines as the audit standard.", "Review TLPT scheduling and results for Tier 1 AI systems and confirm gaps are remediated within required timelines.", "Examine third-party AI provider risk assessments for completeness against DORA Article 28-30 requirements." ], "metrics": [ "DORA critical gap findings remediated within agreed SLA: target 100%.", "TLPT completion on schedule for Tier 1 AI systems: 100% within the three-year test cycle.", "Third-party AI provider risk assessments completed annually: 100% of significant ICT providers." ], "failure_signals": [ "Critical DORA gap findings open beyond agreed remediation SLA without documented risk acceptance.", "TLPT not completed for any Tier 1 AI system within the required three-year cycle.", "Third-party AI provider risk assessments not completed for any significant provider in the last 12 months." ] }, "business_continuity": { "summary": "DORA BCP requirements for AI systems mandate tested plans, defined recovery objectives, and crisis communication procedures aligned with supervisory reporting timelines.", "actions": [ "Ensure AI-related business continuity plans are DORA-compliant and aligned with the financial entity's overall ICT BCP.", "Test AI system BCP scenarios annually and document results for DORA supervisory reporting availability.", "Map AI failure scenarios to DORA major incident reporting obligations and test the reporting pipeline end to end." ], "failure_signals": [ "AI system BCP not aligned with or referenced in the financial entity's DORA-compliant ICT BCP.", "No annual BCP test for AI-critical workflows supporting financial services.", "DORA major incident reporting pipeline for AI failures not tested within the last 12 months." ] }, "it_operations": { "summary": "Operations turns DORA architecture into evidence: incident classification, test execution, and register upkeep are operational routines that must run on cadence, not at audit time.", "actions": [ "Operate the DORA-aligned incident classification and reporting pipeline as a standing procedure with drills.", "Execute and record scheduled resilience tests for in-scope AI systems per the compliance calendar.", "Keep the compliance register updated as architecture and third-party dependencies change." ], "failure_signals": [ "Incident classification against DORA criteria happens retroactively during reporting panic.", "The compliance register lags months behind actual architecture.", "Scheduled resilience tests are skipped in busy quarters with no exception record." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "DORA became applicable in January 2025; most AI system deployments predate DORA and lack explicit compliance architecture documentation." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "eu-high-risk-ai", "high-risk-sector", "universal-enterprise" ], "implementers": [ "GRC / Compliance Team", "Security Architecture", "Platform Engineering" ], "frameworks": [ { "framework": "dora", "requirement_id": "Art. 6-16", "fit": "direct", "rationale": "EU DORA Articles 6-16 define the complete ICT risk management framework requirements that financial entities must implement. This control maps AI system architecture directly to these binding regulatory requirements across all five ICT risk management pillars. DORA is the primary normative driver for this control, and compliance is mandatory for in-scope financial entities from January 2025.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "iso_22301", "requirement_id": "\u00a78.4", "fit": "partial", "rationale": "ISO 22301 \u00a78.4 provides the business continuity procedure framework that aligns with DORA's ICT BCP requirements. Using ISO 22301 as the BCP methodology provides a recognized standard that satisfies DORA's requirement for structured continuity planning. Joint ISO 22301 and DORA compliance can be demonstrated through a single integrated audit program.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.02", "fit": "partial", "rationale": "COBIT 2019 DSS04.02 provides governance practices for determining and ensuring business continuity requirements that complement DORA's regulatory mandates. COBIT's structured approach to BIA, BCP development, and testing provides an audit-friendly methodology for demonstrating DORA compliance. Many financial entities already use COBIT governance frameworks, enabling incremental DORA integration without duplicative framework overhead.", "normative_force": "best-practice", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a72.1", "fit": "adjacent", "rationale": "NIST SP 800-34 \u00a72.1 defines contingency planning policy objectives that parallel DORA's ICT risk management requirements. While not directly applicable to EU regulation, NIST 800-34 provides a technically rigorous implementation methodology that satisfies DORA's principle-based requirements. Organizations with existing NIST 800-34 compliance can leverage that posture toward DORA without rebuilding documentation from scratch.", "normative_force": "voluntary-standard", "source_version": "Rev.1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "AWS Well-Architected Reliability Pillar \u2014 Resilience design principles, failure management, and testing reliability: implementation patterns for DORA operational resilience obligations", "fit": "direct", "rationale": "The AWS Well-Architected Reliability Pillar provides the primary cloud infrastructure design patterns for meeting DORA's operational resilience requirements, with resilience design principles, failure management procedures, and reliability testing frameworks that directly satisfy the ICT risk management obligations in DORA Articles 6-16 for AI systems hosted on AWS. AWS reliability testing guidance \u2014 including chaos engineering and failure injection testing under REL 9 \u2014 implements the threat-led penetration testing and operational resilience validation that DORA Article 26 requires for significant ICT third-party service providers. AWS publishes explicit DORA compliance alignment documentation for its Well-Architected Reliability Pillar, providing a structured mapping between AWS resilience practices and DORA's ICT requirements that the DORA Compliance Architecture control can directly reference.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Google SRE \u2014 Incident management, postmortem practices, and SLO management: operational resilience evidence for DORA supervisory reporting", "fit": "partial", "rationale": "Google SRE incident management and postmortem practices directly address DORA Article 19's ICT-related incident management requirements, providing structured operational frameworks for detecting, classifying, escalating, and reporting incidents at the granularity DORA's supervisory reporting obligations require from financial entities. SRE SLO/SLA management establishes the service availability measurement and error budget tracking that DORA's operational resilience testing requirements expect as baseline evidence of ICT system health, while SRE's production readiness review provides pre-deployment gate documentation that can be incorporated into the DORA compliance architecture evidence required under Article 8. The DORA Compliance Architecture for AI Systems control's requirement for third-party risk governance is supported by SRE production readiness reviews that evaluate third-party AI service dependencies before onboarding.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "Azure Resiliency & BCDR \u2014 design guidance for regulated workloads", "fit": "partial", "rationale": "Microsoft Azure Resiliency and BCDR design guidance defines availability-zone architecture, failover configuration, and recovery-objective planning for Azure-hosted workloads, giving financial entities concrete implementation patterns for the ICT risk management and continuity obligations DORA imposes. It is vendor guidance; it does not itself constitute or attest DORA compliance.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "cloudflare_resilience", "requirement_id": "Cloudflare Resilience \u2014 DDoS protection and load balancing for financial services AI inference endpoints: DORA third-party ICT risk controls", "fit": "partial", "rationale": "Cloudflare's Resilience and DDoS Mitigation framework provides edge-layer operational resilience capabilities \u2014 DDoS protection, load balancing, WAF, and rate limiting \u2014 that address DORA's ICT third-party risk management requirements for AI inference endpoints exposed to financial services infrastructure. When Cloudflare is deployed as an ICT third-party service provider for AI inference availability, the DORA Compliance Architecture for AI Systems control must include Cloudflare's operational resilience commitments and DORA-aligned SLA terms within its third-party risk governance framework under DORA Article 28. Cloudflare's DDoS protection layers and failover capabilities also satisfy the availability requirements DORA imposes on ICT services supporting critical financial functions, though this framework covers the network and edge resilience layer only and does not address DORA's full ICT risk management scope.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RE-06", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every AI system in scope for DORA must have a completed gap assessment against Articles 6-16 with all critical findings remediated or risk-accepted, a documented architecture mapping to all five ICT risk management pillars, third-party AI providers assessed under Articles 28-30, and TLPT scheduled and completed within the required three-year cycle for Tier 1 AI systems.", "evidence_required": [ "DORA gap assessment report for each in-scope AI system documenting findings against Articles 6-16 with severity ratings, remediation owners, target completion dates, and current status for each finding", "AI system architecture mapping document cross-referencing design components to each of the five DORA ICT risk management pillars (identify, protect, detect, respond, recover) with named controls for each pillar", "Third-party AI provider risk assessment records under DORA Articles 28-30 including contract review summary, mandatory Article 30 provision confirmation, ICT concentration risk classification, and assessment completion date for each significant provider", "TLPT scheduling record and results for each Tier 1 AI system confirming completion within the three-year cycle, AI-specific attack scenarios included, and remediation plan for findings with owners and timelines", "DORA major incident classification criteria document for AI system failures and a reporting pipeline end-to-end test record confirming the pipeline produces a compliant draft supervisory report within required timelines" ], "machine_tests": [ "Submit a simulated major AI system incident event to the classification and reporting pipeline \u2192 assert the incident is classified, escalated to the designated function, and a draft supervisory report artifact is generated within the DORA-required reporting timeline", "Query the third-party AI provider register for all significant ICT providers \u2192 assert each entry has a completed Article 30 mandatory provision checklist with no mandatory provisions marked absent", "Run automated DORA pillar coverage check against the AI system architecture diagram \u2192 assert all five ICT risk management pillars have at least one mapped and documented control with an evidence artifact reference" ], "human_review": [ "Review DORA gap assessment findings for each in-scope AI system and assess whether the remediation owners, timelines, and resourcing are realistic given the severity of identified gaps and regulatory deadlines", "Evaluate third-party AI provider risk assessments for completeness and methodological independence, confirming that ICT concentration risk across cloud infrastructure, model API, and data service providers has been quantified and disclosed", "Assess TLPT scope and test design for Tier 1 AI systems to confirm AI-specific attack scenarios including model inference abuse, adversarial prompt injection, and AI component availability attacks are included alongside traditional ICT test scenarios" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Applying a generic IT DORA compliance assessment template to AI systems without addressing AI-specific failure modes such as model inference timeouts, hallucination under load, context window overflow, and model API provider outages", "Treating third-party AI model API providers as excluded from DORA third-party risk assessment because they are consumed as SaaS rather than as traditional ICT outsourcing subject to Articles 28-30 sub-contracting rules", "Scoping TLPT exercises to cover only traditional ICT infrastructure while excluding AI inference systems, model serving pipelines, and external model API dependencies from the threat-led penetration test scope", "Producing an AI system architecture mapping that cross-references DORA pillars at a conceptual level only, without identifying specific controls, control owners, and evidence artifacts for each pillar required by supervisory examination", "Deferring major incident reporting pipeline testing until a real DORA-reportable AI system incident forces it, discovering reporting procedure gaps under supervisory time pressure" ], "update_status": "current", "layer_code": "RE" }, { "id": "RE-07", "layer": "RE", "plane": "lifecycle", "name": "Resilience Engineering Review in Development Lifecycle", "plain": "AI system development and deployment pipelines must include formal resilience design reviews at defined lifecycle gates \u2014 architecture review, pre-production readiness, and post-incident \u2014 to ensure resilience is engineered in from the start rather than retrofitted.", "threat": { "tags": [ "resilience-debt", "unreviewed-failure-modes", "deployment-without-rto-validation", "lifecycle-gap" ], "desc": "AI systems shipped without resilience reviews accumulate resilience debt that becomes progressively more expensive to remediate in production. Novel AI failure modes \u2014 model drift, inference timeout, context window overflow, hallucination under load \u2014 are not covered by traditional application resilience reviews, leaving gaps in failure mode analysis. Without lifecycle gates, resilience requirements are treated as optional post-launch improvements rather than engineering constraints blocking deployment." }, "standard": [ { "id": "nist_800_160_v2", "section": "\u00a72.2", "title": "Cyber resiliency in the system life cycle" }, { "id": "cobit_dss04", "section": "DSS04.04", "title": "Exercise, test and review the BCP and DRP" }, { "id": "google_sre", "section": "Ch. 32", "title": "The Evolving SRE Engagement Model \u2014 Production Readiness Review" }, { "id": "nist_csf", "section": "GV.OC-01", "title": "Organizational mission is understood and informs cybersecurity risk management" } ], "sources": [ { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RE-07 Resilience Engineering Review in Development Lifecycle control.", "reviewed_on": "2026-07-02" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RE-07 Resilience Engineering Review in Development Lifecycle control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RE-07 Resilience Engineering Review in Development Lifecycle control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RE-07 Resilience Engineering Review in Development Lifecycle control.", "reviewed_on": "2026-07-01" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RE-07 Resilience Engineering Review in Development Lifecycle control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Define resilience review checkpoints at three lifecycle gates: architecture review board for new AI systems, production readiness review before first deployment, and post-incident review after any Severity 1 or 2 event. Maintain a resilience review checklist covering all RE-layer controls.", "steps": [ "Define and publish a resilience review checklist covering RE-01 through RE-07 controls, adapted for each AI system tier.", "Integrate resilience review as a mandatory blocking gate in the Architecture Review Board process for all new AI systems.", "Create a Production Readiness Review (PRR) template that includes resilience sign-off by an SRE or resilience lead.", "Mandate post-incident reviews for all Sev1 and Sev2 AI system incidents with a dedicated resilience finding section.", "Track open resilience review findings in the risk register with assigned owners and remediation SLAs by severity.", "Publish quarterly resilience metrics \u2014 review completion rate, open findings by severity, days to remediation \u2014 to engineering leadership." ], "site_reliability": { "summary": "PRR is the SRE team's primary gate for ensuring AI systems meet resilience standards before they accept production load. Do not approve production launch without a signed PRR.", "actions": [ "Own and maintain the Production Readiness Review template; update it as new AI-specific failure modes are identified.", "Block production deployment of Tier 1 AI systems without a completed and signed PRR resilience section.", "Lead post-incident resilience reviews and ensure findings feed back into the RE control checklist updates." ], "failure_signals": [ "Tier 1 AI systems deployed to production without a completed PRR.", "Post-incident resilience findings not tracked or closed within defined SLA.", "PRR template not updated to cover AI-specific failure modes identified in the last 12 months." ] }, "security_architect": { "summary": "Architecture review board gates ensure resilience and security requirements are co-reviewed and do not conflict. Resilience design choices must not create security exceptions.", "actions": [ "Include the resilience checklist review in architecture review board meeting agendas for all new AI systems.", "Verify that resilience design choices such as multi-region failover do not introduce data residency or access control violations.", "Ensure resilience review findings with security implications are tracked in the security risk register." ], "failure_signals": [ "New AI system architectures approved by ARB without resilience checklist review.", "Resilience design introducing security exceptions not documented with a risk acceptance record.", "Security and resilience review processes operating independently without cross-referencing findings." ] }, "it_operations": { "summary": "Operations teams must participate in PRR to confirm runbooks, monitoring coverage, and operational readiness are in place before AI systems go live.", "actions": [ "Participate in PRR sign-off to confirm operational readiness: monitoring, alerting, runbooks, and on-call rotation coverage.", "Maintain a registry of AI systems that have passed PRR and their current operational readiness status.", "Escalate AI systems in production that have not completed PRR for retroactive review within the current quarter." ], "failure_signals": [ "AI systems in production without documented operational readiness sign-off.", "Operations team not included in the PRR process.", "Runbooks not completed or tested before production deployment." ] }, "grc_auditor": { "summary": "Resilience review completion rates and open finding aging are the primary metrics for assessing whether lifecycle resilience governance is functioning effectively.", "actions": [ "Request PRR completion records for all Tier 1 AI systems deployed in the prior 12 months.", "Review post-incident resilience finding records and confirm closed status with supporting evidence.", "Assess architecture review board records for resilience checklist completion rate." ], "metrics": [ "PRR completion rate before first production deployment: target 100% for Tier 1 AI systems.", "Post-incident resilience review completion rate: 100% for Sev1 and Sev2 events within 5 business days.", "Open critical resilience findings older than 30 days without documented risk acceptance: target zero." ], "failure_signals": [ "Tier 1 AI systems deployed without PRR in the audit period.", "Post-incident reviews for Sev1 events not completed within 5 business days.", "Critical resilience findings open beyond 30 days without documented risk acceptance record." ] }, "business_continuity": { "summary": "Embedding resilience reviews in the development lifecycle ensures that BCP coverage keeps pace with new AI system deployments before they enter production.", "actions": [ "Require that BCP impact assessment be completed as part of AI system architecture review.", "Confirm that new AI systems supporting critical business functions are added to BCP scope before production launch.", "Review PRR resilience sign-offs as part of the annual BCP update cycle." ], "failure_signals": [ "New AI systems supporting critical functions launched without a corresponding BCP scope update.", "BIA not completed for AI systems before production deployment.", "PRR resilience records not reviewed during the annual BCP review cycle." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most enterprises have generic architecture review processes that do not include AI-specific resilience criteria or PRR checklists with SRE sign-off gates." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "Site Reliability Engineering", "Platform Engineering", "GRC / Compliance Team" ], "frameworks": [ { "framework": "nist_800_160_v2", "requirement_id": "\u00a72.2", "fit": "direct", "rationale": "NIST SP 800-160 Vol 2 Rev 1 \u00a72.2 places cyber resiliency within the system life cycle, requiring resilience considerations to be addressed through the engineering process stages rather than bolted on at the end. Lifecycle resilience review gates implement that discipline for AI system development.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "google_sre", "requirement_id": "Ch. 32", "fit": "direct", "rationale": "Google SRE (SRE Book, 2016) Chapter 32, The Evolving SRE Engagement Model, defines the Production Readiness Review through which services must demonstrate reliability standards before SRE-supported launch. RE-07's pre-production resilience gate directly implements the PRR model.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.04", "fit": "partial", "rationale": "COBIT 2019 DSS04.04 requires the business continuity plan and disaster response plan to be exercised, tested and reviewed on a regular basis. Reviewing resilience requirements at each AI lifecycle gate extends that review cadence to system change events (DSS04.06 covers training only).", "normative_force": "best-practice", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "GV.OC-01", "fit": "adjacent", "rationale": "NIST CSF 2.0 GV.OC-01 requires that the organizational mission is understood and informs cybersecurity risk management. Embedding resilience review gates into the AI development lifecycle is one way mission-criticality understanding is operationalized in engineering practice.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 6(5)", "fit": "partial", "rationale": "EU DORA Article 6(5) requires financial entities to continuously update their ICT risk management framework in response to lessons learned and changes in the ICT environment. Lifecycle resilience reviews are the mechanism that satisfies this continuous update obligation. Post-incident resilience reviews feed directly into DORA ICT risk management framework updates ensuring the framework remains current.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "aws_reliability", "requirement_id": "AWS Well-Architected Framework \u2014 Well-Architected Review: structured resilience assessment at architecture and pre-production lifecycle gates", "fit": "partial", "rationale": "The AWS Well-Architected Framework provides a structured review methodology \u2014 the Well-Architected Review \u2014 that directly implements the architecture review board gate the Resilience Engineering Review in Development Lifecycle control requires for new AI systems, with the Reliability Pillar providing the resilience-specific evaluation criteria for that review. AWS reliability foundations guidance specifies resilience design principles to validate at the architecture stage, and the AWS reliability testing pillar defines pre-production readiness criteria that align with this control's production readiness review gate, covering RTO/RPO definition, failure mode analysis, and chaos engineering test plans. The AWS Well-Architected Tool provides an automated mechanism for conducting and documenting resilience reviews at defined lifecycle checkpoints, producing structured findings that satisfy this control's review documentation requirements.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "Azure Resiliency & BCDR \u2014 Design lifecycle integration: BCDR planning and pre-production readiness review for Azure workloads", "fit": "partial", "rationale": "Microsoft Azure Resiliency guidance integrates BCDR planning into the service design lifecycle as a standard practice, prescribing BCDR review at the architecture stage for Azure workloads \u2014 a lifecycle integration pattern the Resilience Engineering Review in Development Lifecycle control mandates specifically for AI systems. Azure's pre-production readiness guidance includes verification of failover configuration, RTO and RPO target definition, and recovery procedure testing before AI workloads go live, aligning directly with this control's three-stage lifecycle gate structure covering architecture review, production readiness review, and post-incident review. Azure's guidance on embedding business continuity requirements in the design phase specifically addresses the resilience debt risk that this control targets \u2014 noting that retrofitting resilience post-deployment is significantly more costly than designing it in from the architecture stage.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RE-07", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every Tier 1 AI system must have a completed and signed Production Readiness Review with a resilience section before its first production deployment. Post-incident resilience reviews for all Severity 1 and 2 events must be completed within 5 business days, with all findings tracked in the risk register and no critical finding open beyond 30 days without documented risk acceptance.", "evidence_required": [ "Production Readiness Review completion record for each Tier 1 AI system including signed resilience section, named SRE or resilience lead reviewer, completion date before first production deployment, and list of any open findings with assigned owners and remediation timelines", "Architecture Review Board meeting record for each new AI system confirming the resilience checklist was completed as a blocking gate item, with checklist results and any findings documented in the ARB decision record", "Post-incident resilience review records for all Sev1 and Sev2 AI system events in the prior 12 months showing review completion date, findings identified, root cause classification, remediation owners, and closure evidence", "Resilience finding tracker export showing all open findings categorized by severity and age, with evidence that every critical finding older than 30 days has a documented risk acceptance record signed by a named authority" ], "machine_tests": [ "Query production AI service registry for all systems deployed in the last 90 days \u2192 assert each service has a corresponding signed PRR record with a resilience section completion date before the service's first production deployment date", "Query the incident register for all Sev1 and Sev2 AI system incidents in the last 6 months \u2192 assert 100% have a post-incident review record with completion date within 5 business days of incident declaration", "Check resilience finding tracker for critical-severity findings \u2192 assert zero findings are older than 30 days without a documented risk acceptance record containing a named approver and expiry date" ], "human_review": [ "Review the PRR template for currency against known AI-specific failure modes and confirm it explicitly covers model drift detection, inference timeout handling, context window overflow, graceful degradation when the model API is unavailable, and AI-specific recovery time validation", "Assess post-incident resilience review records for analytical depth, confirming findings identify specific root causes and actionable control improvements rather than superficial observations that do not change the control posture", "Evaluate the resilience finding tracker to confirm findings are progressing toward closure within stated timelines and that any change of owner has been accompanied by updated risk acceptance and priority classification" ], "blocking_effect": "blocks-deployment", "normative_status": "voluntary-standard", "anti_patterns": [ "Deploying Tier 1 AI systems to production without a completed PRR by treating the PRR as a documentation formality that can be completed retrospectively after the system is already serving production traffic", "Maintaining a PRR resilience checklist that covers only traditional infrastructure failure modes without AI-specific criteria such as model inference latency degradation, context window overflow, prompt injection under load, or model API provider outage", "Completing post-incident resilience reviews as checkbox exercises that document the timeline of events but do not identify specific control failures, root causes, or concrete actions that would prevent recurrence", "Tracking open resilience review findings in isolated team documents rather than a centralized risk register, making it impossible to report on aggregate finding aging or demonstrate systematic closure to auditors", "Treating the Architecture Review Board resilience checklist as an advisory input rather than a mandatory blocking gate, allowing systems to proceed to production with open resilience findings pending a future review" ], "update_status": "current", "layer_code": "RE" }, { "id": "RE-08", "layer": "RE", "plane": "lifecycle", "name": "Resilience Engineering Evidence Package", "plain": "Compliance evidence for the RE layer must be compiled into a structured evidence package demonstrating that high availability, stateless design, backup governance, IaC recovery, dependency isolation, DORA architecture, and lifecycle resilience reviews are all in place and operating effectively.", "threat": { "tags": [ "incomplete-evidence", "audit-failure", "resilience-attestation-gap", "unverifiable-compliance-posture" ], "desc": "Without a structured evidence package, resilience claims cannot be verified by auditors, regulators, or downstream consumers of the ResilienceAttestation. Fragmented evidence scattered across multiple teams creates gaps that are only discovered during supervisory examinations. An inability to produce timely, complete resilience evidence can trigger regulatory findings or invalidate attestations that depend on RE-layer controls being demonstrably in place." }, "standard": [ { "id": "iso_22301", "section": "\u00a79.1", "title": "Monitoring, measurement, analysis and evaluation" }, { "id": "dora", "section": "Art. 19", "title": "Reporting of major ICT-related incidents" }, { "id": "nist_sp800_34", "section": "\u00a73.5", "title": "Plan testing, training, and exercises" }, { "id": "cobit_dss04", "section": "DSS04.08", "title": "Conduct post-resumption review" } ], "sources": [ { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RE-08 Resilience Engineering Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RE-08 Resilience Engineering Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RE-08 Resilience Engineering Evidence Package control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RE-08 Resilience Engineering Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RE-08 Resilience Engineering Evidence Package control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Establish a quarterly evidence collection cycle that gathers artifacts from each RE control (RE-01 through RE-07), validates completeness against a defined evidence schema, and packages them for attestation. Store the evidence package in an immutable evidence store referenced by the ResilienceAttestation (RG-08) artifact.", "steps": [ "Define the RE evidence schema: required artifacts per control, acceptable formats, freshness requirements, and coverage thresholds.", "Assign evidence collection owners for each RE control (RE-01 through RE-07) and document in the RACI.", "Automate evidence collection where possible: pull HA SLO reports, backup job logs, IaC drift scan results, and circuit breaker metrics from monitoring systems.", "Collect manually-produced evidence \u2014 architecture diagrams, PRR sign-offs, DORA gap assessment, and test records \u2014 on a defined quarterly schedule.", "Validate completeness of the collected evidence package against the schema and generate a gap report for missing or stale artifacts.", "Package validated evidence into a signed, immutable bundle referenced by the ResilienceAttestation (RG-08) artifact.", "Present the evidence package to the CISO and GRC function quarterly and to the Board Risk Committee annually." ], "site_reliability": { "summary": "SRE teams own the operational evidence for RE-01 through RE-05. Automate export of SLO reports, chaos test results, circuit breaker state logs, and IaC drill records into the evidence pipeline.", "actions": [ "Build automated pipelines that export SLO compliance reports, failover test results, and IaC drill records on the evidence collection schedule.", "Tag all chaos engineering and failure injection test results with the evidence collection run ID for traceability.", "Review completeness reports and remediate missing SRE-owned evidence artifacts before the evidence package deadline." ], "failure_signals": [ "SRE-owned evidence artifacts missing from two consecutive quarterly evidence packages.", "SLO reports not exported in machine-readable format compatible with evidence collection.", "Chaos test results not linked to the evidence package through a traceable run ID." ] }, "security_architect": { "summary": "The evidence package must include architecture diagrams and security review sign-offs confirming that resilience controls do not introduce security gaps.", "actions": [ "Provide signed architecture review records confirming HA, IaC, and bulkhead design was reviewed for security compliance.", "Confirm DORA third-party risk assessment evidence includes security architecture review outputs.", "Review the evidence package schema annually to ensure security-relevant artifacts remain required." ], "failure_signals": [ "Architecture review records missing from the evidence package.", "Security sign-off on HA or IaC design not present in the evidence bundle.", "Evidence package schema not reviewed annually by security architecture." ] }, "it_operations": { "summary": "IT Operations owns backup job logs, restore test records, and operational runbook evidence. These must be production-quality records created at time of execution, not retroactively reconstructed documentation.", "actions": [ "Maintain backup job execution logs and restore test records in a format suitable for evidence export throughout the quarter.", "Provide operational runbook evidence and on-call response records as RE-03 and RE-01 evidence artifacts on schedule.", "Confirm all IT-owned evidence is submitted by the evidence collection deadline each quarter." ], "failure_signals": [ "Backup job logs not retained in exportable format for the required evidence retention period.", "Restore test records created retroactively rather than at time of test execution.", "IT operations evidence submitted after the quarterly evidence package deadline." ] }, "grc_auditor": { "summary": "The GRC function validates evidence completeness and integrity, issues the ResilienceAttestation, and owns the evidence register. Evidence must be timely, accurate, and traceable to source systems.", "actions": [ "Validate the evidence package against the RE evidence schema before each attestation cycle.", "Cross-reference automated evidence artifacts against source systems to confirm integrity and freshness.", "Issue the ResilienceAttestation (RG-08) only when evidence completeness exceeds the defined threshold and no critical gaps exist." ], "metrics": [ "Evidence completeness per RE control: target \u2265 95% coverage of required artifacts at attestation.", "Evidence freshness: all artifacts within defined staleness threshold at time of attestation.", "Attestation issuance on schedule: 100% of quarterly attestation cycles completed on time." ], "failure_signals": [ "Evidence completeness below 95% for any RE control at attestation time without a documented waiver.", "ResilienceAttestation issued with known critical evidence gaps without documented risk acceptance.", "Evidence package integrity not verified against source systems before attestation." ] }, "business_continuity": { "summary": "The evidence package demonstrates to board, regulators, and counterparties that AI systems are built to withstand disruption. This is a strategic governance artifact, not merely a compliance checkbox.", "actions": [ "Present the annual RE evidence package summary to the Board Risk Committee as part of operational resilience reporting.", "Use evidence package findings to drive the annual BCP review and update cycle.", "Ensure the evidence package is accessible to supervisory authorities within defined response timelines for regulatory requests." ], "failure_signals": [ "RE evidence package not presented to board-level governance in the annual reporting cycle.", "Evidence package findings not driving concrete BCP update actions.", "Evidence not available for supervisory review within required response timeframe." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Structured resilience evidence packages are rare; most organizations can produce isolated test records but cannot assemble them into a coherent compliance artifact on demand." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "GRC / Compliance Team", "Site Reliability Engineering", "CISO Office" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a79.1", "fit": "direct", "rationale": "ISO 22301 \u00a79.1 requires monitoring, measurement, analysis, and evaluation of the business continuity management system including the production of documented evidence for management review. The RE evidence package is the primary artifact that satisfies this evaluation requirement for AI systems. ISO 22301 certification auditors will examine the evidence package as the core audit artifact demonstrating BCMS effectiveness.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 19", "fit": "direct", "rationale": "EU DORA Article 19 requires financial entities to report major ICT-related incidents to competent authorities within defined windows. The RE evidence package preserves the incident, testing and recovery records that make timely, accurate reporting \u2014 and after-the-fact supervisory scrutiny \u2014 supportable.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.5", "fit": "direct", "rationale": "NIST SP 800-34 Rev 1 \u00a73.5 requires contingency plan tests, training, and exercises with documented results retained as evidence of plan effectiveness. The RE evidence package operationalizes that retention for AI resilience engineering.", "normative_force": "voluntary-standard", "source_version": "Rev.1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.08", "fit": "direct", "rationale": "COBIT 2019 DSS04.08 (Conduct post-resumption review) requires lessons from continuity events to be captured and used to improve plans. The RE evidence package is the artifact repository that enables that lessons-learned loop.", "normative_force": "best-practice", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "ID.IM-03", "fit": "adjacent", "rationale": "NIST CSF 2.0 ID.IM-03 requires improvements to be identified from execution of operational processes, procedures, and activities. The evidence package's cross-cycle review mechanism institutionalizes that improvement loop for AI resilience engineering.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_27031", "requirement_id": "Cl. 13", "fit": "adjacent", "rationale": "ISO/IEC 27031:2025 Clause 13 requires management evaluation of ICT readiness performance, based on documented monitoring, testing, and review results. The RE evidence package assembles the documented record that evaluation consumes.", "normative_force": "voluntary-standard", "source_version": "2025", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "AWS Well-Architected Reliability Pillar \u2014 Testing reliability documentation and failure management evidence: Well-Architected Review findings and remediation tracking", "fit": "partial", "rationale": "The AWS Well-Architected Reliability Pillar prescribes continuous evidence collection for reliability testing outcomes, failure management procedures, and recovery validation results that map directly to the RE-layer artifact categories required by the Resilience Engineering Evidence Package control. AWS reliability testing guidance mandates documented outcomes from chaos engineering exercises, failover validation tests, and backup restore verifications \u2014 all required artifact categories in the evidence package this control defines for the ResilienceAttestation. AWS Well-Architected Review findings and remediation tracking provide a structured evidence trail demonstrating that HA design, dependency isolation, and IaC recovery controls meet AWS reliability standards, directly contributing to the completeness evidence this control must demonstrate.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Google SRE \u2014 Postmortem practices and production readiness review: structured post-incident and pre-deployment resilience evidence artifacts", "fit": "partial", "rationale": "Google SRE postmortem practices generate structured post-incident analyses and production readiness reviews produce documented pre-deployment resilience assessments \u2014 both primary evidence artifact categories that the Resilience Engineering Evidence Package control requires for demonstrating RE-layer control effectiveness to auditors and attestation consumers. SRE SLO measurement data constitutes an ongoing evidence stream demonstrating that HA availability targets defined in RE-01 are being met, providing the continuous monitoring evidence this control must include in its quarterly evidence collection cycle. SRE postmortems also produce the lessons-learned documentation required by the RE-07 lifecycle review artifact category in the evidence package, completing the chain from incident detection through resilience improvement to attestation.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "Azure Resiliency & BCDR \u2014 BCDR documentation, failover test reports, and recovery objective verification records as resilience compliance evidence", "fit": "partial", "rationale": "Microsoft Azure Resiliency and BCDR documentation defines the artifact categories \u2014 BCDR plans, failover test reports, geo-redundancy configuration records, and recovery objective verification results \u2014 that constitute evidence of operational resilience compliance and map to the RE-layer evidence schema the Resilience Engineering Evidence Package control requires. Azure Advisor recommendations and Azure reliability dashboard data provide structured, platform-generated evidence inputs that can be incorporated into the quarterly evidence collection cycle this control mandates, reducing manual evidence assembly effort. Azure compliance attestation artifacts and availability zone health reports provide the verifiable third-party evidence that the ResilienceAttestation's evidence completeness requirements expect, complementing organization-produced artifacts in the evidence package.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RE-08", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "A structured evidence package covering all RE-layer controls (RE-01 through RE-07) must be compiled on a quarterly cadence, with completeness exceeding 95% of required artifacts per control and all artifacts within defined freshness thresholds. The package must be validated against the RE evidence schema before the ResilienceAttestation (RG-08) is issued.", "evidence_required": [ "RE evidence completeness report showing artifact coverage percentage per control (RE-01 through RE-07) against the defined schema, with gap list and staleness flags for any artifact exceeding the freshness threshold", "Signed evidence package manifest listing every artifact included, its source system, collection timestamp, SHA-256 hash, and the evidence collection run ID used for traceability", "Automated evidence pipeline execution log confirming SLO compliance reports, backup job logs, IaC drift scan results, chaos test records, and circuit breaker metrics were collected from source systems rather than reconstructed manually", "CISO and GRC function quarterly review record confirming the evidence package was presented, completeness was assessed, and the ResilienceAttestation issuance decision was made against documented criteria", "Immutable evidence store write confirmation showing the validated package was committed with a content hash and is retrievable for supervisory review within defined response timelines" ], "machine_tests": [ "Run evidence completeness check against RE evidence schema for the current quarter package \u2192 assert coverage is at or above 95% for every RE control and zero artifacts exceed the defined staleness threshold", "Verify evidence package manifest integrity by recomputing SHA-256 hashes of all package artifacts \u2192 assert all computed hashes match the manifest values, confirming no post-collection modification", "Query immutable evidence store for the most recent quarterly package \u2192 assert the package is retrievable within the defined supervisory response time SLA and is readable by the GRC access account" ], "human_review": [ "Review the RE evidence schema annually to confirm required artifact types for each control remain current, freshness thresholds are appropriate for audit cycles, and any new RE control additions or modifications are reflected in the schema", "Cross-reference a sample of automated evidence artifacts against their source systems to confirm the evidence collection pipeline is pulling live data rather than cached or manually assembled records that could misrepresent the current control posture", "Assess the evidence package for coherence across RE controls, confirming that HA architecture evidence, IaC recovery records, and PRR sign-offs are internally consistent and that discrepancies between controls are identified and explained rather than silently omitted" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Assembling the RE evidence package by retroactively reconstructing documents after the attestation deadline rather than maintaining evidence artifacts as production-quality records generated at time of execution throughout the quarter", "Issuing the ResilienceAttestation (RG-08) with evidence completeness below the defined threshold by treating missing artifacts as implicit evidence of control operation rather than as documented gaps requiring risk acceptance", "Storing evidence artifacts in team file shares or email threads rather than an immutable, access-controlled evidence store, making it impossible to demonstrate chain of custody or retrieve artifacts on demand for supervisory review", "Treating automated evidence collection as a substitute for validation against the evidence schema, allowing pipeline-collected artifacts that are stale, incomplete, or from the wrong source system to inflate completeness metrics", "Presenting the evidence package to governance bodies as a summary slide deck without making the underlying artifact details available, preventing meaningful review of evidence quality by CISO, GRC, and board-level audiences" ], "update_status": "current", "layer_code": "RE" }, { "id": "RG-01", "layer": "RG", "plane": "control", "name": "Resilience Governance Structure", "plain": "The organization must establish a formal governance structure for AI operational resilience, including a designated committee, documented roles and accountability mapping, and named executive sponsorship. This structure provides the authority to mandate resilience requirements across AI system development and operations.", "threat": { "tags": [ "governance-vacuum", "unowned-risk", "accountability-gap", "program-fragmentation" ], "desc": "Without a formal governance structure, AI resilience responsibilities fragment across teams without clear ownership or escalation paths. When a high-availability AI system fails, the absence of defined accountability means recovery decisions are delayed, workarounds bypass resilience controls, and lessons learned are never institutionalized. Regulators including DORA supervisors expect named accountable roles at senior levels." }, "standard": [ { "id": "iso_22301", "section": "\u00a75.1", "title": "Leadership and commitment for business continuity" }, { "id": "cobit_dss04", "section": "DSS04.01", "title": "Define the business continuity policy, objectives and scope" }, { "id": "dora", "section": "Art. 5", "title": "ICT risk management governance and accountability" }, { "id": "nist_sp800_34", "section": "\u00a72.1", "title": "Roles and responsibilities for contingency planning" } ], "sources": [ { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RG-01 Resilience Governance Structure control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RG-01 Resilience Governance Structure control.", "reviewed_on": "2026-07-02" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RG-01 Resilience Governance Structure control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RG-01 Resilience Governance Structure control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Establish a Resilience Steering Committee with cross-functional membership (Engineering, Operations, Risk, Legal, Compliance). Define a RACI mapping resilience obligations to named executives. Publish a governance charter covering mandate, meeting cadence, escalation paths, and metrics review cycle.", "steps": [ "Draft and ratify a Resilience Governance Charter defining committee scope, membership, quorum rules, and decision authority.", "Map all AI system tiers to named executive sponsors and assign primary resilience owners to each production AI system.", "Establish a quarterly governance review cadence with documented agenda including RTO/RPO compliance rates, test results, and open remediation items.", "Integrate the resilience governance structure into the enterprise risk committee reporting hierarchy and link it to board-level risk appetite statements." ], "site_reliability": { "summary": "The governance structure defines which bodies have authority to accept resilience exceptions or approve architecture changes that affect RTO/RPO commitments. SRE teams must know the escalation path.", "actions": [ "Document the escalation chain from SRE on-call to resilience governance committee for AI system outages.", "Ensure SRE runbooks reference the governance approval gate for resilience architecture changes." ], "failure_signals": [ "Resilience exceptions approved informally outside the committee process.", "SRE teams unable to name the executive accountable for a given AI system's resilience commitments." ] }, "grc_auditor": { "summary": "The governance structure is the foundational artifact demonstrating that resilience is managed at an organizational level, not ad hoc. Auditors should verify the charter exists, is current, and that the committee actually meets.", "actions": [ "Request and review the Resilience Governance Charter for completeness: membership, mandate, escalation, and metrics scope.", "Sample at least two quarters of committee meeting minutes and verify action items are tracked to closure.", "Cross-reference the RACI against the current AI system inventory to confirm all production systems have a named owner." ], "metrics": [ "Percentage of production AI systems with a named executive resilience sponsor: target 100%.", "Governance committee meeting cadence compliance: target 100% of scheduled meetings held.", "Open resilience governance action items older than 90 days: target 0." ], "failure_signals": [ "Charter not reviewed or updated in over 12 months.", "Committee meetings missed for two or more consecutive quarters.", "AI systems in production without a named resilience owner in the RACI." ] }, "business_continuity": { "summary": "The governance structure authorizes the BCM function to mandate resilience requirements for AI systems and ensures continuity planning is resourced and enforced.", "actions": [ "Ensure BCM is represented on the Resilience Steering Committee with voting rights.", "Align AI system resilience governance cadence with enterprise BCM review cycle." ], "failure_signals": [ "BCM function excluded from AI system resilience decision-making.", "AI system BIAs not aligned with enterprise BIA review cycle." ] }, "it_operations": { "summary": "Operations gives the governance structure its ground truth: operational metrics, incident data, and capacity realities must flow into committee decisions, and committee decisions must land as operational priorities.", "actions": [ "Supply the resilience committee with standing operational reports (incidents, test results, capacity risks).", "Route committee decisions into the operational backlog with owners and dates.", "Escalate operational risks that exceed team authority through the defined governance path." ], "failure_signals": [ "Committee decisions are based on stale or anecdotal operational data.", "Governance actions never appear in operational work queues.", "Known operational risks die in team backlogs for lack of an escalation path." ] }, "security_architect": { "summary": "Resilience governance and security governance share failure modes and must not conflict: ensure the committee structure includes security representation and deconflicts with incident response and risk committees.", "actions": [ "Secure a standing security seat on the resilience governance committee.", "Map overlaps between resilience and security governance (incident authority, risk acceptance) and document precedence.", "Bring security-relevant resilience risks (recovery-path weaknesses, provider trust) onto the committee agenda." ], "failure_signals": [ "Resilience and security committees issue conflicting directives during incidents.", "Security learns of resilience architecture decisions after implementation.", "No forum exists where recovery-path security risks are owned." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most organizations lack formal governance structures specifically for AI system resilience; this is typically subsumed under general IT resilience governance without AI-specific accountability." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "federated-enterprise", "eu-high-risk-ai" ], "implementers": [ "CISO", "Chief Risk Officer", "VP Engineering" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a75.1", "fit": "direct", "rationale": "ISO 22301:2019 \u00a75.1 requires top management to demonstrate leadership and commitment to the BCMS, which directly maps to the executive sponsorship and committee structure required here. The standard mandates that roles, responsibilities, and authorities be assigned and communicated for business continuity.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 5", "fit": "direct", "rationale": "DORA Article 5 requires financial entities to have an internal governance and control framework for ICT risk that ensures the management body takes an active role. For EU-regulated entities operating AI systems, this makes a formal governance structure with named senior accountability a legal obligation, not a recommendation.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.01", "fit": "direct", "rationale": "COBIT DSS04.01 explicitly calls for defining the business continuity policy, objectives, and scope, with clear roles and responsibilities. The management practice directly supports the committee charter and RACI requirements of this control.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a72.1", "fit": "direct", "rationale": "NIST SP 800-34 \u00a72.1 establishes that a contingency planning coordinator must be designated and that senior management provide visible support. This maps directly to the named executive sponsor and governance committee requirements in this control.", "normative_force": "voluntary-standard", "source_version": "Rev. 1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "GV.RR", "fit": "direct", "rationale": "NIST CSF 2.0 Govern function GV.RR (Roles, Responsibilities, and Authorities) requires that cybersecurity roles and responsibilities are established and communicated. The Resilience Governance Structure fulfills this for the resilience domain.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "anthropic_rsp", "requirement_id": "Governance \u2014 Responsible Scaling Officer and internal accountability", "fit": "adjacent", "rationale": "The Responsible Scaling Policy (v3.3) binds Anthropic and includes a Governance section: a named Responsible Scaling Officer, internal review, and escalation paths for capability determinations. It is an informative analogue \u2014 a named-officer governance model \u2014 for the executive-sponsored resilience committee structure this control requires of deployers.", "normative_force": "best-practice", "source_version": "3.3", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "openai_preparedness", "requirement_id": "\u00a75.1 Internal governance \u2014 Safety Advisory Group", "fit": "adjacent", "rationale": "The OpenAI Preparedness Framework v2 \u00a75.1 establishes internal governance that binds OpenAI: a Safety Advisory Group reviews capability and safeguard evidence and advises leadership, who own deployment decisions. It offers an informative analogue for the governance committee and executive ownership this control requires.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Production readiness review", "fit": "adjacent", "rationale": "Google SRE's Production Readiness Review establishes a governance gate that assigns organizational accountability for system reliability before production launch, requiring designated owners and clear escalation paths. The Resilience Governance Structure formalizes the authority structures \u2014 committee, RACI, executive sponsor \u2014 that make PRR decisions binding and enforceable across AI system development and operations teams.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "Business continuity planning", "fit": "partial", "rationale": "Microsoft Azure BCDR's business continuity planning guidance requires organizations to establish governance structures with executive ownership, defined roles, and clear accountability before designing BCDR solutions for cloud-hosted workloads. The Resilience Governance Structure directly implements the organizational BCM governance layer \u2014 steering committee, RACI, executive sponsor, and governance charter \u2014 that Azure BCDR's business continuity planning approach depends on to be operationally effective.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RG-01", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "A Resilience Steering Committee must exist with a ratified governance charter, documented cross-functional membership, and named executive sponsors. Every Tier 1 production AI system must have a named resilience owner in the RACI, and the committee must have met on its defined quarterly cadence with documented action items tracked to closure within 90 days.", "evidence_required": [ "Resilience Governance Charter document with ratification date, committee membership list with named executives and their roles, defined quorum rules, quarterly meeting cadence, and escalation path to board-level risk committee", "RACI mapping showing every Tier 1 production AI system with a named executive resilience sponsor and a primary resilience owner, current as of the date of evidence collection", "Quarterly committee meeting minutes for the prior two quarters confirming meetings occurred, quorum was achieved, resilience metrics were reviewed, and action items were recorded with owners and due dates", "Action item tracker export confirming all committee action items are either closed with supporting evidence or have an open status with named owner and next-review date; no item older than 90 days without documented rationale for extension" ], "machine_tests": [ "Query production AI service registry \u2192 assert every Tier 1 system has a named executive resilience sponsor and primary resilience owner recorded in the RACI with entries current within the last 90 days", "Check committee meeting calendar records \u2192 assert no more than one scheduled quarterly meeting was missed in the trailing four quarters and that each meeting has a signed minutes record in the document management system", "Query action item tracker for resilience governance items \u2192 assert zero items are older than 90 days without either a closed status with evidence or an open status with a documented extension rationale signed by the committee chair" ], "human_review": [ "Review the Resilience Governance Charter for substantive completeness, confirming it specifies the committee's authority to mandate resilience requirements across AI development and operations teams and that this authority is reflected in actual committee decisions rather than serving as aspirational documentation", "Sample two quarters of committee meeting minutes and assess whether the agenda covers meaningful operational review of resilience metrics, test results, and open findings rather than pro forma approval of pre-agreed items", "Cross-reference the RACI against the current production AI system inventory and assess whether named resilience owners are organizationally positioned to be accountable for the systems assigned to them, or whether the assignments reflect paper ownership without operational authority" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Designating a resilience governance committee on paper to satisfy regulatory examination requirements while allowing all substantive resilience architecture and exception decisions to continue being made informally outside the committee process", "Assigning AI system executive resilience sponsorship to senior leaders who lack operational authority over the engineering and operations teams responsible for implementing resilience requirements, creating accountable owners with no power to mandate change", "Maintaining a RACI that maps AI systems to resilience owners who have since changed roles or departed, leaving named accountability with individuals who no longer have any relationship to the systems or teams they are listed as owning", "Running resilience committee meetings as status report presentations without documented decision authorities, leaving it unclear which body has the power to accept resilience exceptions or block non-compliant AI deployments", "Treating the governance committee's quarterly meeting as the only venue for resilience decision-making, with no defined escalation path for urgent resilience issues arising between meetings that require executive authority" ], "update_status": "current", "layer_code": "RG" }, { "id": "RG-02", "layer": "RG", "plane": "control", "name": "Resilience Policy Framework", "plain": "The organization must maintain a documented policy framework governing how AI systems are designed, tested, and maintained for resilience, including policies for recovery objective setting, chaos engineering, backup verification, and resilience exception management.", "threat": { "tags": [ "policy-gap", "inconsistent-standards", "ungoverned-recovery", "test-avoidance" ], "desc": "Without formal resilience policies, AI systems are built and operated to inconsistent standards. Teams make local recovery objective decisions without organizational alignment, resilience tests are deferred indefinitely, and exceptions accumulate without review. When a failure occurs, the absence of policy means there is no authoritative standard against which to measure compliance or identify the governance failure that permitted the gap." }, "standard": [ { "id": "iso_22301", "section": "\u00a76.2", "title": "Business continuity objectives and planning" }, { "id": "dora", "section": "Art. 6", "title": "ICT risk management framework policy requirements" }, { "id": "nist_sp800_34", "section": "\u00a73.1", "title": "Contingency planning policy statement" }, { "id": "cobit_dss04", "section": "DSS04.02", "title": "Implement business continuity strategies" } ], "sources": [ { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RG-02 Resilience Policy Framework control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RG-02 Resilience Policy Framework control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RG-02 Resilience Policy Framework control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RG-02 Resilience Policy Framework control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RG-02 Resilience Policy Framework control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Develop a tiered resilience policy hierarchy: a top-level Resilience Policy defining principles and scope, domain-specific standards for RTO/RPO by system tier, and operational procedures for testing, backup verification, and exception management. Policies must be reviewed annually and approved by the Resilience Steering Committee.", "steps": [ "Draft a top-level AI Resilience Policy covering scope, principles, system tiering criteria, and mandatory control requirements by tier.", "Develop subordinate standards documents: Recovery Objective Standard, Resilience Testing Standard, Backup and Restore Standard, and Exception Management Standard.", "Establish an annual policy review cycle tied to the enterprise policy management calendar; require committee approval for all substantive changes.", "Publish policies in the enterprise policy management system with version history, effective dates, and named policy owners for each document." ], "site_reliability": { "summary": "The policy framework defines the mandatory floor for resilience engineering decisions. SRE teams use the standards to determine which testing cadences are required and which RTO/RPO commitments are binding.", "actions": [ "Map each AI system tier to its applicable policy requirements and document the mapping in the service catalog.", "Ensure chaos engineering and DR test schedules reflect policy-mandated cadences, not ad hoc scheduling." ], "failure_signals": [ "AI systems in production whose resilience configuration does not meet the applicable tier standard.", "Test schedules that are less frequent than the mandated cadence in the Resilience Testing Standard." ] }, "grc_auditor": { "summary": "The policy framework is the primary evidence that resilience is governed with documented, approved standards. Auditors should verify completeness, currency, and that policies are operationally effective \u2014 not shelfware.", "actions": [ "Request the full resilience policy hierarchy and verify all required documents exist with current review dates.", "Cross-reference the policy exception register against active exceptions to confirm all are within approved terms.", "Sample 5 AI systems and verify their resilience configurations match the applicable tier standard." ], "metrics": [ "Percentage of resilience policy documents reviewed within the last 12 months: target 100%.", "Open policy exceptions beyond their approved expiry date: target 0.", "AI systems with documented policy compliance mapping: target 100% of production systems." ], "failure_signals": [ "Policy documents with no review date or last reviewed more than 18 months ago.", "Policy exceptions with no expiry date or no risk acceptance by a named owner.", "AI systems without a documented tier classification in the policy mapping." ] }, "business_continuity": { "summary": "The resilience policy framework should align with and extend the enterprise BCM policy to cover AI-specific risks and testing requirements that traditional BCM policies do not address.", "actions": [ "Review the enterprise BCM policy and identify gaps for AI system-specific requirements including model recovery and inference state restoration.", "Ensure the AI Resilience Policy is cross-referenced from the enterprise BCM policy with clear delineation of scope." ], "failure_signals": [ "AI resilience policy conflicts with or duplicates enterprise BCM policy requirements.", "BCM policy reviews do not include evaluation of AI-specific policy coverage gaps." ] }, "it_operations": { "summary": "Operations teams must understand which policy requirements apply to the systems they operate and maintain compliance evidence as part of normal operational practice.", "actions": [ "Ensure all runbooks reference the applicable resilience policy tier and document how compliance is maintained.", "Include policy compliance verification as a standing item in change advisory board reviews for AI systems." ], "failure_signals": [ "Change requests approved without resilience policy impact assessment for Tier 1 or Tier 2 AI systems.", "Operations runbooks that do not reference policy-mandated recovery procedures." ] }, "security_architect": { "summary": "The resilience policy framework must interlock with security policy: recovery, fallback, and continuity policies should reference security requirements rather than silently overriding them.", "actions": [ "Cross-reference resilience policies with security policies and resolve conflicts explicitly (e.g., recovery speed vs. verification).", "Require security review as a mandatory step in the resilience policy approval workflow.", "Define in policy which security controls remain non-negotiable during continuity operations." ], "failure_signals": [ "Resilience policies authorize actions security policy forbids, with no documented precedence.", "Policy updates ship without security review.", "Responders improvise security trade-offs because policy is silent." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Most organizations have general IT continuity policies that do not address AI-specific resilience requirements such as model state recovery, inference pipeline restoration, or AI system tier classification." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "Chief Risk Officer", "CISO", "BCM Team" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a76.2", "fit": "direct", "rationale": "ISO 22301:2019 \u00a76.2 requires organizations to establish business continuity objectives, determine what is needed to achieve them, and maintain documented information. This directly supports the requirement for a formal policy framework with measurable objectives.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 6", "fit": "direct", "rationale": "DORA Article 6 requires financial entities to have a comprehensive ICT risk management framework including policies, procedures, and protocols. For AI systems operated by DORA-regulated entities, a formal resilience policy framework is a binding legal requirement.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.1", "fit": "direct", "rationale": "NIST SP 800-34 Rev 1 \u00a73.1 requires a contingency planning policy statement defining objectives, scope, roles, and requirements \u2014 the foundational federal guidance on which an AI resilience policy framework should be modeled.", "normative_force": "voluntary-standard", "source_version": "Rev. 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.02", "fit": "partial", "rationale": "COBIT DSS04.02 addresses implementing business continuity strategies, which includes the policy framework needed to execute those strategies consistently. The partial fit reflects that COBIT addresses strategy implementation broadly while this control focuses on the governance policy layer.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "GV.PO", "fit": "direct", "rationale": "NIST CSF 2.0 GV.PO (Policy) requires that organizational cybersecurity policy is established, communicated, and enforced. The resilience policy framework is the primary vehicle for fulfilling this requirement within the resilience domain.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "Resilience design principles", "fit": "partial", "rationale": "The AWS Well-Architected Reliability Pillar's resilience design principles provide a structured set of policy-level commitments covering workload architecture, change management, and failure response that represent best practice for cloud-hosted AI systems. The Resilience Policy Framework should incorporate and formalize these principles as governance-approved standards within its policy hierarchy, ensuring engineering teams have documented, approved guidance rather than informal interpretations of AWS best practices.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Error budget policies", "fit": "partial", "rationale": "Google SRE's error budget policy concept formalizes how reliability targets are managed and what corrective actions are triggered when error budgets are exhausted, constituting a specific and actionable policy type within a resilience governance framework. Incorporating error budget policies within the Resilience Policy Framework for AI Operational Resilience provides a quantitative, operationally grounded policy mechanism for AI system reliability governance that bridges governance intent and engineering practice.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "BCDR design guidance", "fit": "partial", "rationale": "Microsoft Azure BCDR design guidance establishes architectural and operational policies for business continuity that must be formalized in organizational policy documents before they can be consistently applied to AI system design. The Resilience Policy Framework incorporates Azure BCDR design guidance as a domain-specific standard within its policy hierarchy for cloud-hosted AI systems, giving engineering teams governance-approved policy backing rather than advisory guidance.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RG-02", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "A tiered resilience policy hierarchy must exist covering recovery objective setting, resilience testing cadences, backup verification, and exception management. All policy documents must have been reviewed within the last 12 months with committee approval, and every production AI system must have a documented tier classification with a corresponding policy compliance mapping.", "evidence_required": [ "Resilience policy document inventory listing each policy and standard in the hierarchy (top-level AI Resilience Policy, Recovery Objective Standard, Resilience Testing Standard, Backup and Restore Standard, Exception Management Standard) with version, effective date, last-review date, committee approval record, and named policy owner", "AI system tier classification register mapping every production AI system to its applicable policy tier with the classification rationale documented and the classification date within the last 12 months", "Policy exception register listing all active exceptions with exception ID, AI system affected, policy requirement being excepted, approved exception scope, expiry date, and risk acceptance record signed by named authority", "Compliance verification sample records confirming at least 5 AI systems were checked against their applicable tier standard within the last audit cycle, with findings and remediation actions documented" ], "machine_tests": [ "Query the policy management system for all resilience policy documents \u2192 assert every required document in the hierarchy exists with a last-review date within the past 12 months and a committee approval record", "Query the policy exception register \u2192 assert zero exceptions have an open status with an expiry date in the past, and zero exceptions lack a named risk acceptance owner with an expiry date", "Query the production AI system inventory \u2192 assert every system has a tier classification entry in the tier classification register with a classification date within the past 12 months" ], "human_review": [ "Review the Recovery Objective Standard and verify that the RTO and RPO tiers defined for AI system categories reflect current business impact assessments, and that the tiering criteria clearly distinguish between AI inference systems supporting critical business functions and lower-priority AI workloads", "Assess the Exception Management Standard to confirm it specifies a maximum exception duration, requires documented risk acceptance from a named authority at the appropriate organizational level, and mandates periodic review rather than allowing silent exception perpetuation", "Evaluate whether the policy hierarchy meaningfully addresses AI-specific resilience requirements not covered by the enterprise BCM policy, such as model state recovery, inference pipeline restoration, model API provider concentration risk, and AI system tier reclassification triggers" ], "blocking_effect": "advisory", "normative_status": "binding-law", "anti_patterns": [ "Publishing a single generic AI Resilience Policy that references recovery objectives without defining tier-specific RTO and RPO thresholds, leaving engineering teams to interpret policy requirements inconsistently across different AI systems and risk profiles", "Allowing policy exceptions to accumulate without expiry dates or documented risk acceptance, effectively converting temporary approved exceptions into permanent ungoverned deviations from the resilience standard", "Maintaining the enterprise BCM policy as the only governing document for AI system resilience without creating AI-specific subordinate standards that address model state recovery, inference pipeline restoration, and AI system tier classification criteria", "Treating policy review as an annual document refresh exercise that updates version numbers and review dates without substantively evaluating whether the policy tiers and testing cadence requirements remain aligned with current business risk appetite and AI system criticality", "Classifying AI systems into resilience tiers at initial deployment and never reclassifying them as their usage, criticality, or organizational dependencies evolve, allowing high-criticality AI systems to remain in lower-tier policy scope as their business impact grows" ], "update_status": "current", "layer_code": "RG" }, { "id": "RG-03", "layer": "RG", "plane": "control", "name": "Senior Accountability for AI Operational Resilience", "plain": "A named executive must hold formal accountability for AI operational resilience commitments, with this accountability documented in role definitions, board-level risk registers, and regulatory disclosures where applicable. This individual must attest to the adequacy of the resilience program at least annually.", "threat": { "tags": [ "accountability-gap", "regulatory-exposure", "diffused-ownership", "untested-commitment" ], "desc": "Without named senior accountability, resilience commitments made to customers, regulators, or the board are unenforceable because no individual bears personal accountability for their fulfillment. During a major AI system failure, the absence of a designated accountable executive creates decision paralysis, slows regulatory communications, and prevents meaningful post-incident accountability. DORA and similar regimes are increasingly requiring named senior executives to attest to operational resilience adequacy." }, "standard": [ { "id": "dora", "section": "Art. 5(2)", "title": "Management body accountability for ICT risk" }, { "id": "iso_22301", "section": "\u00a75.3", "title": "Organizational roles, responsibilities and authorities" }, { "id": "cobit_dss04", "section": "DSS04.01", "title": "Define accountability for continuity outcomes" }, { "id": "nist_csf", "section": "GV.RR-02", "title": "Roles and responsibilities for senior leadership" } ], "sources": [ { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RG-03 Senior Accountability for AI Operational Resilience control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RG-03 Senior Accountability for AI Operational Resilience control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RG-03 Senior Accountability for AI Operational Resilience control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Document a named executive role (e.g., Chief Resilience Officer or designated member of the executive committee) with explicit accountability for AI operational resilience. Include this accountability in the role's formal job description, the enterprise risk register, and any regulatory submissions. Require an annual written attestation from this individual to the board or audit committee.", "steps": [ "Designate a named executive with formal accountability for AI operational resilience and document this in the role's job description and performance objectives.", "Register the accountability in the enterprise risk register and in the internal governance and control framework documentation that DORA Article 5 requires the management body to define, approve and oversee. DORA imposes no standing regulatory filing of this designation; the documentation must instead be available to competent authorities on request.", "Establish an annual attestation process: the accountable executive reviews the resilience program assessment and provides a written attestation of adequacy to the audit committee or board.", "Define a backup accountability mapping to ensure continuity of governance accountability during executive transitions or absences." ], "site_reliability": { "summary": "The named executive is the ultimate escalation point for decisions that cannot be resolved within engineering and operations. SRE leadership must know the escalation path and the executive must be reachable during declared incidents.", "actions": [ "Include the named accountable executive in the major incident escalation path for Tier 1 AI system outages.", "Provide the executive with quarterly briefings on resilience posture, open findings, and test results to ensure informed attestation." ], "failure_signals": [ "Executive accountable for resilience not included in P1 AI system incident escalation contacts.", "Executive attestation based on summary decks with no underlying evidence review." ] }, "grc_auditor": { "summary": "Named senior accountability is increasingly a regulatory requirement and a foundational control for demonstrating organizational commitment to resilience. Auditors should verify the designation is current, documented, and that attestations are substantive.", "actions": [ "Request the current role designation document and verify it names a specific individual, not just a role title.", "Review the most recent annual attestation letter for substantive content and evidence that the executive engaged with the underlying program assessment.", "Verify the accountability is registered in the enterprise risk register and reflected in regulatory submissions where required." ], "metrics": [ "Named executive accountability documented and current: binary (yes/no), target yes.", "Annual attestation completed within required timeframe: target 100%.", "Regulatory filings reflecting named accountability: target 100% where required." ], "failure_signals": [ "Executive role is vacant with no interim accountability designation.", "Attestation letters that are formulaic and unsupported by evidence of program review.", "Accountability not updated following executive departures or role changes." ] }, "business_continuity": { "summary": "BCM depends on senior executive sponsorship to mandate resilience requirements and secure resources. The named accountability role is the governance anchor for the BCM program's authority.", "actions": [ "Ensure the accountable executive chairs or sponsors the Resilience Steering Committee.", "Brief the executive on BCM program gaps and resource constraints as part of the annual attestation preparation process." ], "failure_signals": [ "BCM program lacks access to named accountable executive for resource or policy escalations.", "Executive unfamiliar with material resilience gaps at time of annual attestation." ] }, "it_operations": { "summary": "The accountable executive depends on operations for a truthful picture: operational attestation inputs (test results, incident metrics, unresolved risks) must be accurate, current, and unfiltered.", "actions": [ "Provide the accountable executive with regular operational resilience reports including failures and open risks.", "Maintain the operational evidence (test records, incident data) that backs each attestation cycle.", "Flag material operational risks to the accountable executive without dilution through management layers." ], "failure_signals": [ "Attestations rest on operational summaries that omit known failures.", "Evidence for the last attestation cannot be reproduced from operational records.", "Frontline risk warnings never reach the accountable executive." ] }, "security_architect": { "summary": "Senior accountability must cover the security dimension of AI resilience: the accountable executive should see recovery-path security risks, and attestations should not outrun security reality.", "actions": [ "Include security posture of recovery and continuity paths in the accountable executive's briefing pack.", "Review attestation language for claims that depend on unverified security controls.", "Establish a direct escalation route from security architecture to the accountable executive for resilience-relevant risks." ], "failure_signals": [ "Attestations assert resilience while known recovery-path security gaps remain open.", "The accountable executive has never been briefed on failover security posture.", "Security risk registers and resilience attestations tell different stories." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Named senior accountability for AI-specific operational resilience is rare; most organizations assign this within a broad technology or risk role without explicit AI resilience accountability." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "Board / Audit Committee", "Chief Risk Officer", "General Counsel" ], "frameworks": [ { "framework": "dora", "requirement_id": "Art. 5(2)", "fit": "direct", "rationale": "DORA Article 5(2) explicitly requires that the management body of financial entities define and approve strategies, policies, and procedures for ICT risk management and bear ultimate accountability. This makes named senior accountability a binding legal requirement for EU-regulated entities.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "iso_22301", "requirement_id": "\u00a75.3", "fit": "direct", "rationale": "ISO 22301:2019 \u00a75.3 requires that roles, responsibilities, and authorities relevant to the BCMS are assigned, documented, and communicated within the organization. Named senior accountability is the apex of this requirement.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.01", "fit": "partial", "rationale": "COBIT DSS04.01 includes defining accountability for continuity outcomes as part of the governance management practice. The partial fit reflects that COBIT addresses this within a broader continuity management context rather than AI-specific senior attestation.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "GV.RR-02", "fit": "direct", "rationale": "NIST CSF 2.0 GV.RR-02 requires that roles and responsibilities for cybersecurity risk management are established, communicated, and understood by senior leadership. The named executive accountability for AI resilience directly fulfills this subcategory.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a72.1", "fit": "partial", "rationale": "NIST SP 800-34 \u00a72.1 requires that senior management provide visible support for the contingency planning program. While it does not require the formal attestation this control mandates, it establishes the principle of senior leadership accountability.", "normative_force": "voluntary-standard", "source_version": "Rev. 1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "anthropic_rsp", "requirement_id": "Governance \u2014 Responsible Scaling Officer accountability", "fit": "adjacent", "rationale": "The Responsible Scaling Policy (v3.3) binds Anthropic and names a Responsible Scaling Officer accountable for policy implementation. That named-individual accountability model is an informative analogue for the senior executive accountability for AI operational resilience this control requires.", "normative_force": "best-practice", "source_version": "3.3", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "openai_preparedness", "requirement_id": "\u00a75.1 Internal governance \u2014 leadership risk ownership", "fit": "adjacent", "rationale": "The OpenAI Preparedness Framework v2 \u00a75.1 places deployment risk decisions with OpenAI leadership, advised by the Safety Advisory Group. The pattern of a named senior owner for operational risk decisions is the informative analogue to this control's named accountable executive.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "Business continuity planning", "fit": "adjacent", "rationale": "Microsoft Azure BCDR's business continuity planning guidance expects organizations to designate executive ownership of BCM programs as a prerequisite for effective BCDR program execution. The Senior Accountability for AI Operational Resilience control implements this executive designation formally \u2014 with documented role definition, risk register registration, and annual board attestation \u2014 creating the organizational accountability that Azure BCDR's business continuity planning approach assumes is in place.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RG-03", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "The organization must have a current, named executive with documented formal accountability for AI operational resilience, with this designation reflected in role definitions and the enterprise risk register, and an annual attestation letter substantively reviewed and delivered to the board or audit committee within the required cycle.", "evidence_required": [ "Executive role designation document naming a specific individual (not a role title) with explicit AI operational resilience accountability, including an effective date and last review date", "Enterprise risk register entry confirming the named individual's accountability is registered and current, with the most recent review timestamp", "Annual executive attestation letter addressed to the board or audit committee, with evidence of engagement with underlying program assessment (not a formulaic sign-off)", "Incident escalation contact list confirming the named executive is included in the Tier 1 AI system escalation path with current contact details", "Regulatory submission records (where DORA or equivalent applies) showing the named individual's accountability is disclosed to supervisory authorities" ], "machine_tests": [ "Query the role designation registry for the AI resilience accountability entry \u2192 assert a named individual (not a generic title) is returned with a designation date within the last 12 months", "Query the enterprise risk register for the AI operational resilience accountability record \u2192 assert record exists, individual name is populated, and last review date is within 12 months", "Query the attestation tracking system for the most recent annual executive attestation \u2192 assert submission date is within 12 months and status is board-acknowledged", "Query the Tier 1 incident escalation contact list \u2192 assert the named accountable executive appears with a current contact entry" ], "human_review": [ "Review the most recent annual attestation letter for substantive content confirming the executive engaged with the underlying program assessment, not a formulaic recitation of compliance", "Verify that role transitions or executive departures since the last review have not created an unaddressed accountability gap, and that an interim designation is documented if the role is being transitioned", "Assess whether the internal governance documentation (and any responses provided to competent authorities on request) that names ICT risk accountability accurately reflects the current designated individual \u2014 DORA Article 5 requires internal assignment and documentation, not a standing regulatory filing" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Assigning AI resilience accountability to a role title (e.g., 'Head of Technology Risk') without naming a specific individual, making the accountability unenforceable during transitions", "Treating the annual executive attestation as a rubber-stamp formality signed without the accountable individual reviewing the underlying program evidence or findings", "Failing to update the designation record and regulatory filings following executive departures, leaving the accountability unassigned for extended periods", "Assigning AI resilience accountability as an undocumented appendage to a broader IT risk or CISO role without reflecting it in the job description or performance objectives", "Excluding the named accountable executive from the major incident escalation path, defeating the purpose of the accountability designation during actual crisis events" ], "update_status": "current", "layer_code": "RG" }, { "id": "RG-04", "layer": "RG", "plane": "control", "name": "Resilience Risk Appetite and Threshold Setting", "plain": "The organization must define and formally approve a resilience risk appetite statement for AI systems, including quantitative RTO/RPO thresholds by system tier, acceptable failure rates, and maximum tolerable downtime. These thresholds must flow down to system-level SLOs and be reviewed at least annually.", "threat": { "tags": [ "undefined-tolerance", "uncalibrated-rto-rpo", "appetite-drift", "slo-misalignment" ], "desc": "Without formal risk appetite thresholds, AI system resilience targets are set by engineering teams without governance alignment, resulting in inconsistent commitments to stakeholders. Overly optimistic RTO/RPO commitments create regulatory exposure when they cannot be met in practice. Absence of tiered thresholds means critical AI systems and low-value systems are governed to the same standard \u2014 or no standard at all." }, "standard": [ { "id": "iso_22301", "section": "\u00a78.2.2", "title": "Business impact analysis and recovery objectives" }, { "id": "dora", "section": "Art. 11", "title": "ICT business continuity policy and recovery objectives" }, { "id": "nist_sp800_34", "section": "\u00a73.2", "title": "Recovery time and recovery point objectives" }, { "id": "cobit_dss04", "section": "DSS04.02", "title": "Recovery time and recovery point objectives" } ], "sources": [ { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RG-04 Resilience Risk Appetite and Threshold Setting control.", "reviewed_on": "2026-07-02" }, { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RG-04 Resilience Risk Appetite and Threshold Setting control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RG-04 Resilience Risk Appetite and Threshold Setting control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RG-04 Resilience Risk Appetite and Threshold Setting control.", "reviewed_on": "2026-07-01" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RG-04 Resilience Risk Appetite and Threshold Setting control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Define a resilience risk appetite statement at the enterprise level, then translate it into quantitative tier-specific thresholds. Tier 1 (mission-critical AI): RTO \u2264 4h, RPO \u2264 1h. Tier 2 (business-critical AI): RTO \u2264 8h, RPO \u2264 4h. Tier 3 (operational AI): RTO \u2264 24h, RPO \u2264 24h. Thresholds must be approved by the Resilience Steering Committee and reflected in system-level SLOs.", "steps": [ "Conduct a business impact analysis (BIA) for all production AI systems to determine their criticality tier and maximum tolerable period of disruption (MTPD).", "Draft a Resilience Risk Appetite Statement with quantitative RTO/RPO thresholds per tier, maximum tolerable failure rates, and data loss tolerance; obtain formal approval from the Resilience Steering Committee.", "Cascade thresholds to system-level SLOs and service commitments; document the mapping in the AI system inventory.", "Schedule an annual review of thresholds tied to the BIA refresh cycle and update commitments where business impact has changed." ], "site_reliability": { "summary": "Risk appetite thresholds define the SLO floor for AI systems. SRE teams use these to set error budgets and determine when a system requires architectural improvement to meet its governance commitment.", "actions": [ "Align each AI system's SLOs and error budgets to the RTO/RPO thresholds set by the governance-approved risk appetite.", "Flag any AI system whose current reliability metrics indicate it cannot meet its tier-mandated RTO/RPO without architectural change." ], "failure_signals": [ "AI systems with SLOs that are looser than the governance-approved tier threshold.", "RTO/RPO thresholds that have never been validated against actual recovery test results." ] }, "grc_auditor": { "summary": "Risk appetite and threshold documentation is the foundational evidence that resilience commitments are governance-approved and not ad hoc. Auditors must verify thresholds are approved, current, and flowed down to operational artifacts.", "actions": [ "Request the approved Resilience Risk Appetite Statement and verify it has been formally approved by the Resilience Steering Committee with a current review date.", "Sample 5 production AI systems and verify their SLOs reflect the tier-appropriate thresholds from the appetite statement.", "Review BIA documentation to confirm tier classifications are based on current business impact, not legacy categorizations." ], "metrics": [ "Percentage of production AI systems with RTO/RPO thresholds aligned to approved appetite statement: target 100%.", "Risk appetite statement reviewed within the last 12 months: binary (yes/no), target yes.", "BIA refresh completed within the last 12 months: target 100% of Tier 1 and Tier 2 systems." ], "failure_signals": [ "Risk appetite statement not formally approved or lacking a review date.", "AI systems with SLOs that exceed the approved tier thresholds without a documented exception.", "BIA documents more than 24 months old for Tier 1 AI systems." ] }, "business_continuity": { "summary": "Risk appetite thresholds are the quantitative expression of resilience governance. BCM is responsible for conducting BIAs that inform the thresholds and for ensuring the appetite is operationally realistic.", "actions": [ "Conduct annual BIA refresh for all Tier 1 and Tier 2 AI systems and provide results to the Resilience Steering Committee before the appetite review.", "Validate that proposed RTO/RPO thresholds are achievable given current architecture before presenting for governance approval." ], "failure_signals": [ "BIA process that does not cover AI-specific dependencies such as model inference pipelines and training data stores.", "Appetite thresholds approved without BCM validation of technical achievability." ] }, "it_operations": { "summary": "Thresholds set in governance must be enforceable in operations: RTO/RPO and availability appetites become alert thresholds, capacity plans, and escalation triggers.", "actions": [ "Translate approved thresholds into monitoring alerts and escalation triggers in operational tooling.", "Report threshold breaches and near-misses to governance with operational context.", "Validate through exercises that operational capability can actually meet the approved thresholds." ], "failure_signals": [ "Approved thresholds exist on paper but no alert fires when they are breached.", "Operations quietly runs to different targets than governance approved.", "Threshold breaches are discovered in retrospectives, not in real time." ] }, "security_architect": { "summary": "Risk appetite for resilience must price security trade-offs: aggressive recovery targets can pressure teams to bypass verification. Make the security cost of each threshold explicit at approval time.", "actions": [ "Annotate proposed RTO/RPO thresholds with the security steps they must accommodate before approval.", "Define appetite statements for security-relevant degraded states (running without full verification, emergency access).", "Review threshold exceptions for security impact before they are granted." ], "failure_signals": [ "Recovery targets are approved that are only achievable by skipping security validation.", "No appetite statement exists for operating with degraded security controls.", "Threshold exceptions accumulate without security review." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Formal AI-specific risk appetite statements with quantitative RTO/RPO tiers are uncommon; most organizations use informal or inherited IT thresholds without governance approval." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "cloud-native", "multi-tenant" ], "implementers": [ "Chief Risk Officer", "BCM Team", "CISO" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a78.2.2", "fit": "direct", "rationale": "ISO 22301:2019 \u00a78.2.2 requires organizations to conduct business impact analysis and determine recovery time objectives. This is the direct normative basis for RTO/RPO threshold setting as a governance requirement.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 11", "fit": "direct", "rationale": "DORA Article 11 requires financial entities to have an ICT business continuity policy that includes recovery time and recovery point objectives. Quantitative threshold-setting backed by governance approval is a direct legal requirement for covered entities.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.2", "fit": "direct", "rationale": "NIST SP 800-34 \u00a73.2 provides detailed guidance on determining RTO and RPO through BIA, making it the canonical federal guidance for the threshold-setting process this control mandates.", "normative_force": "voluntary-standard", "source_version": "Rev. 1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.02", "fit": "direct", "rationale": "COBIT DSS04.02 specifically addresses determining recovery time and recovery point objectives as part of implementing business continuity strategies. The management practice aligns directly with governance-approved threshold setting.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "google_sre", "requirement_id": "SRE Book Ch. 3", "fit": "partial", "rationale": "Google SRE practices establish the concept of error budgets derived from SLOs, which is the operational implementation of risk appetite in engineering terms. The partial fit reflects that SRE practices are operational guidance rather than governance requirements.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "anthropic_rsp", "requirement_id": "Capability thresholds \u2014 ASL-3 standards at defined capability levels", "fit": "adjacent", "rationale": "The Responsible Scaling Policy (v3.3) binds Anthropic: it defines capability thresholds beyond which ASL-3 deployment and security standards must be in place before scaling continues. Threshold-triggered obligations are an informative model for the quantified resilience risk-appetite thresholds this control requires deployers to set.", "normative_force": "best-practice", "source_version": "3.3", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "openai_preparedness", "requirement_id": "Capability thresholds \u2014 High and Critical levels", "fit": "adjacent", "rationale": "The OpenAI Preparedness Framework v2 defines High and Critical capability thresholds and requires safeguards before deployment proceeds at those levels. The threshold-and-consequence structure is an informative model for board-approved resilience thresholds (RTO/RPO, error budgets) in deploying organizations.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "aws_reliability", "requirement_id": "Failure management \u2014 RTO/RPO alignment", "fit": "direct", "rationale": "The AWS Well-Architected Reliability Pillar's failure management focus area provides detailed guidance on setting RTO and RPO objectives aligned to business requirements, covering BIA methodology, tier-based classification, and the translation of recovery objectives into architectural design decisions. The Resilience Risk Appetite and Threshold Setting control directly implements the governance layer above this technical guidance, ensuring RTO/RPO thresholds are formally approved by the Resilience Steering Committee and cascaded to AI system SLOs as binding organizational commitments.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "Recovery objectives \u2014 BCDR design guidance", "fit": "direct", "rationale": "Microsoft Azure BCDR's recovery objectives guidance provides explicit frameworks for defining RTO and RPO by system tier and mapping them to failover architecture and availability zone design decisions. The Resilience Risk Appetite and Threshold Setting control implements the governance layer that ensures Azure BCDR recovery objective recommendations are elevated into formally approved organizational thresholds \u2014 backed by BIA, reviewed annually, and reflected in system-level SLO commitments for AI systems.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RG-04", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "The organization must have a formally approved Resilience Risk Appetite Statement with quantitative RTO/RPO thresholds per AI system tier, supported by current BIA documentation completed within 12 months, and all production AI systems must have system-level SLOs demonstrably aligned to their tier-mandated thresholds.", "evidence_required": [ "Approved Resilience Risk Appetite Statement with explicit quantitative RTO/RPO thresholds per system tier (Tier 1/2/3), Resilience Steering Committee approval signature, and review date within the last 12 months", "BIA reports for all Tier 1 and Tier 2 AI systems completed within 12 months, documenting AI-specific dependencies including model inference pipelines and training data stores", "AI system inventory showing tier classification (Tier 1/2/3) for every production AI system with BIA basis and last classification review date", "System-level SLO documentation for each production AI system with explicit mapping to the governing tier threshold from the approved appetite statement", "Resilience Steering Committee meeting record confirming the threshold review was conducted and approved with quorum" ], "machine_tests": [ "Query AI system inventory for production systems missing tier classification or with a classification last reviewed more than 12 months ago \u2192 assert 0 violations", "Query SLO registry and cross-reference each AI system's RTO/RPO commitment against its tier threshold from the approved appetite statement \u2192 assert 0 systems with commitments looser than their governing threshold without a documented exception", "Query the appetite statement governance record for the most recent Resilience Steering Committee approval \u2192 assert approval date is within 12 months and quorum is documented", "Query BIA completion records for all Tier 1 AI systems \u2192 assert all have a BIA completed within 12 months with AI-specific dependency coverage confirmed" ], "human_review": [ "Assess whether the BIA methodology covers AI-specific dependencies (model inference latency, training data restore time, GPU availability) and whether tier classifications reflect current business impact rather than legacy IT categorizations", "Review whether proposed RTO/RPO thresholds were validated against current architecture recovery test results before Resilience Steering Committee approval, and identify any thresholds that engineering teams know are currently unachievable", "Verify that AI systems with SLOs exceeding the tier threshold have documented exceptions with remediation plans approved by the Resilience Steering Committee" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Inheriting general IT RTO/RPO thresholds for AI systems without conducting AI-specific BIA validation that accounts for inference pipeline recovery times and model artifact restore windows", "Setting RTO/RPO thresholds as aspirational targets without validating them against actual architecture recovery capabilities, producing commitments that cannot be met in practice", "Applying a single threshold uniformly to all AI systems regardless of criticality tier, treating a mission-critical autonomous decision system the same as a low-risk internal reporting tool", "Approving thresholds through informal agreement rather than documented Resilience Steering Committee approval with quorum, making them unenforceable as governance commitments", "Cascading thresholds to SLOs without documenting the tier-to-SLO mapping, so auditors cannot verify alignment and engineering teams cannot determine which governance threshold governs their system" ], "update_status": "current", "layer_code": "RG" }, { "id": "RG-05", "layer": "RG", "plane": "control", "name": "Resilience Program Resourcing", "plain": "The AI resilience program must be adequately resourced with dedicated funding, qualified staffing, and appropriate tooling. Resourcing decisions must be documented, approved through formal budget processes, and reviewed at least annually to ensure the program can meet its mandated objectives.", "threat": { "tags": [ "underfunded-program", "staffing-gap", "tooling-debt", "resource-erosion" ], "desc": "An underfunded or understaffed resilience program produces governance artifacts that do not reflect operational reality. Test cadences are deferred for lack of resources, tooling gaps mean failures are undetected until catastrophic, and staff turnover in resilience roles leaves the program without institutional knowledge. Regulators increasingly scrutinize whether resilience programs are substantively resourced or paper exercises." }, "standard": [ { "id": "iso_22301", "section": "\u00a77.1", "title": "Resources for the BCMS" }, { "id": "dora", "section": "Art. 5(2)(g)", "title": "Management body allocates and reviews the digital operational resilience budget" }, { "id": "cobit_dss04", "section": "DSS04.01", "title": "Define the business continuity policy, objectives and scope \u2014 roles and resources" }, { "id": "nist_csf", "section": "GV.SC-07", "title": "Risks posed by suppliers and third parties are understood, recorded, prioritized, assessed, responded to, and monitored" } ], "sources": [ { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RG-05 Resilience Program Resourcing control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RG-05 Resilience Program Resourcing control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Establish a dedicated resilience program budget line with annual planning integrated into enterprise budget cycles. Define staffing requirements by role (resilience engineers, BCM specialists, tooling owners) and maintain a skills inventory. Document tooling requirements covering monitoring, chaos engineering, backup verification, and DR orchestration. Conduct an annual resource adequacy review against program obligations.", "steps": [ "Develop a resilience program resource plan covering personnel, tooling, infrastructure, and testing costs; obtain approval through the enterprise budget process.", "Define minimum staffing levels by role and document current staffing against those minimums; escalate gaps to the Resilience Steering Committee.", "Maintain an approved tooling inventory covering all categories of resilience tooling; conduct annual gap assessment against test and monitoring obligations.", "Conduct an annual resource adequacy review comparing actual resources against program obligations and bring results to the Resilience Steering Committee." ], "site_reliability": { "summary": "Resilience tooling and staffing directly affect SRE capacity to test and validate recovery capabilities. SRE teams should flag resource constraints that prevent required test cadences from being met.", "actions": [ "Maintain a list of resilience test obligations that cannot be met due to resource constraints and escalate through the governance process.", "Include resilience tooling requirements in annual platform engineering budget submissions." ], "failure_signals": [ "Chaos engineering or DR tests deferred more than one quarter due to resource constraints.", "Monitoring coverage gaps for AI system resilience metrics due to tooling budget limits." ] }, "grc_auditor": { "summary": "Resource adequacy is a governance indicator of program seriousness. Auditors should assess whether the program is funded to meet its mandated obligations, not just to maintain governance artifacts.", "actions": [ "Request the resilience program resource plan and budget approval documentation.", "Cross-reference the approved test calendar against actual test execution to identify deferrals attributed to resource constraints.", "Review the staffing inventory against minimum role requirements and identify positions that have been vacant more than 90 days." ], "metrics": [ "Resilience program budget approved and tracked against plan: binary (yes/no), target yes.", "Staffing positions at or above minimum level: target 100%.", "Test deferrals attributable to resource constraints in the last 12 months: target 0." ], "failure_signals": [ "No dedicated resilience budget line; program funded ad hoc from project budgets.", "Key resilience roles vacant for more than 90 days without approved backfill.", "Program obligations that cannot be met with current resources without a documented remediation plan." ] }, "business_continuity": { "summary": "BCM program effectiveness depends on adequate resourcing. The BCM function should lead the annual resource adequacy review and surface gaps to the governance committee before they result in program failures.", "actions": [ "Prepare an annual program resource adequacy assessment for the Resilience Steering Committee covering staffing, tooling, and budget.", "Include AI-specific tooling requirements (inference pipeline recovery tools, model snapshot management) in BCM resource planning." ], "failure_signals": [ "BCM resource assessments that do not cover AI-specific resilience tooling needs.", "Resource gaps identified during incidents rather than through the annual planning process." ] }, "it_operations": { "summary": "Resourcing shortfalls surface first in operations: unfilled on-call rotations, deferred test cycles, and tooling debt are the leading indicators the budget process needs to see.", "actions": [ "Report operational resourcing gaps (on-call coverage, skipped tests, tooling debt) into the annual resourcing review.", "Quantify the operational cost of deferred resilience work for budget justification.", "Track headcount and tooling allocations against the approved resourcing plan." ], "failure_signals": [ "Resilience test cycles are skipped because no one is available to run them.", "On-call rotations for AI recovery are chronically understaffed.", "Budget decisions are made without visibility into operational shortfalls." ] }, "security_architect": { "summary": "Security capacity is part of resilience resourcing: recovery-path hardening, backup security, and provider assessments all fail silently when unfunded. Ensure the resourcing plan covers security work explicitly.", "actions": [ "Itemize security-of-resilience work (backup hardening, failover security parity, provider assessments) in resourcing plans.", "Flag resilience initiatives that arrive without funded security review effort.", "Track security debt accumulating in recovery and continuity infrastructure as a resourcing metric." ], "failure_signals": [ "Failover infrastructure hardening is perpetually deferred for lack of budget.", "Provider security assessments lapse because no one is funded to run them.", "Resilience projects ship with security review unfunded and unperformed." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "AI resilience programs are often funded as a subset of general IT resilience budgets without AI-specific resource planning. Dedicated roles for AI resilience are uncommon outside large technology organizations." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "Chief Risk Officer", "VP Engineering", "BCM Team" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a77.1", "fit": "direct", "rationale": "ISO 22301:2019 \u00a77.1 requires organizations to determine and provide the resources needed for the establishment, implementation, maintenance, and continual improvement of the BCMS. This is the direct normative basis for formal resilience program resourcing.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 5(2)(g)", "fit": "direct", "rationale": "EU DORA Article 5(2)(g) makes the management body responsible for allocating and periodically reviewing the budget needed to fulfil the entity's digital operational resilience needs, including ICT training. For covered entities this makes resilience program resourcing a management-body obligation (Article 5(4) addresses management-body training, not budget).", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.01", "fit": "partial", "rationale": "COBIT 2019 DSS04.01 requires defining the business continuity policy, objectives and scope, including the roles, responsibilities and resources needed to deliver it. Resilience program resourcing decisions are made and sustained through exactly that governance practice (DSS04.07 covers backup arrangements, not resourcing).", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "GV.SC-07", "fit": "adjacent", "rationale": "NIST CSF 2.0 GV.SC-07 requires the risks posed by suppliers and third parties to be understood, recorded, prioritized, assessed, responded to, and monitored over the course of the relationship. Sustained program resourcing is what keeps that supplier-risk monitoring running for AI dependencies; the fit is adjacent because the subcategory addresses supply-chain risk rather than budgeting.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a72.2", "fit": "partial", "rationale": "NIST SP 800-34 \u00a72.2 addresses resource requirements for contingency planning programs. While not prescriptive about budget processes, it establishes that programs require dedicated resources to fulfill their obligations.", "normative_force": "voluntary-standard", "source_version": "Rev. 1", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "Foundations \u2014 organizational prerequisites for reliability", "fit": "partial", "rationale": "The AWS Well-Architected Reliability Pillar's Foundations focus area establishes that organizations must make foundational infrastructure investments and define prerequisite team structures before attempting to achieve workload reliability. The Resilience Program Resourcing control formalizes the governance process for planning, approving, and tracking these foundational investments \u2014 staffing, tooling, and budget \u2014 ensuring AI system resilience programs are resourced to meet their mandated obligations rather than operating on ad hoc allocations.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Toil reduction and on-call practices", "fit": "partial", "rationale": "Google SRE's practices on toil reduction and on-call staffing establish that adequate engineering headcount, appropriate tooling investment, and sustainable on-call workloads are prerequisites for maintaining AI system reliability SLOs over time. The Resilience Program Resourcing control governs the organizational budget and staffing planning process that ensures SRE and BCM teams have the personnel and tooling required to meet resilience obligations \u2014 directly operationalizing the resourcing principles that Google SRE identifies as foundational to program effectiveness.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RG-05", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "The resilience program must have an approved and tracked budget line item, all defined resilience roles filled at or above minimum staffing level with no vacancy exceeding 90 days, a tooling inventory with no documented gaps that have caused a required test cadence deferral in the review period, and an annual resource adequacy review delivered to the Resilience Steering Committee.", "evidence_required": [ "Approved resilience program resource plan with dedicated budget line item, planning approval date within the current fiscal year, and budget vs. actuals tracking record", "Staffing inventory showing each required resilience role, the minimum headcount threshold, current headcount, and vacancy status with days-open for any open positions", "Tooling inventory covering all resilience tooling categories (monitoring, chaos engineering, DR orchestration, backup verification, model snapshot management) with gap assessment results and last review date", "Annual resource adequacy review report presented to and acknowledged by the Resilience Steering Committee, covering staffing, tooling, and budget against program obligations", "Test calendar showing scheduled obligations vs. actual execution, with any deferrals coded by root cause (resource constraint vs. other)" ], "machine_tests": [ "Query staffing inventory for resilience roles with vacancy duration >90 days and no documented interim assignment \u2192 assert 0 violations", "Query test calendar execution records for deferrals with root cause coded as resource-constraint in the last 12 months \u2192 assert 0", "Query the budget system for the resilience program budget line item \u2192 assert a current fiscal year approval record exists with a dedicated line entry (not a cost-pooled allocation)", "Query resource adequacy review records \u2192 assert a Resilience Steering Committee-acknowledged review exists with a completion date within the last 12 months" ], "human_review": [ "Assess whether the tooling inventory covers AI-specific resilience tooling needs including inference pipeline recovery tools, model snapshot management, and GPU capacity reservation for DR scenarios, or whether it reflects only generic IT resilience tooling", "Review whether staff assigned to resilience roles have the required AI-specific skills and experience, not just general IT BCM or SRE backgrounds, to fulfill AI resilience obligations", "Evaluate whether resource gaps identified in prior annual reviews have been remediated within the committed timeline or have been rolled forward without resolution" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Funding the resilience program as an undifferentiated subset of general IT operations budgets without a dedicated budget line, making it invisible to governance and subject to reallocation without accountability", "Accepting key resilience role vacancies beyond 90 days without designating an interim accountable party, creating unacknowledged capability gaps in the program", "Planning resilience tooling investments without separating AI-specific requirements (inference pipeline recovery, model artifact management) from general IT DR tooling, leaving AI-specific tooling gaps undetected", "Using resource constraints as the standing explanation for test deferrals without escalating the constraint to the Resilience Steering Committee for remediation or acceptance", "Conducting annual resource adequacy reviews as paper exercises without cross-referencing the approved test calendar to identify obligations that cannot be met with current resources" ], "update_status": "current", "layer_code": "RG" }, { "id": "RG-06", "layer": "RG", "plane": "control", "name": "Resilience Incident Response Integration", "plain": "AI resilience incidents must be integrated into the enterprise incident management process, with AI-specific severity classifications, escalation paths, recovery decision authorities, and post-incident review requirements defined and published. The integration must ensure that AI system failures trigger resilience governance review where they represent recovery objective breaches.", "threat": { "tags": [ "incident-silo", "missed-escalation", "governance-bypass", "learning-failure" ], "desc": "When AI resilience incidents are handled outside the enterprise incident management process, governance oversight is lost. Recovery decisions are made without appropriate authority, RTO/RPO breaches are not recorded as governance findings, and post-incident reviews fail to identify systemic resilience failures. This creates a feedback loop where incidents do not improve the resilience program, and repeated failures do not trigger governance escalation." }, "standard": [ { "id": "dora", "section": "Art. 17-23", "title": "ICT-related incident management and reporting" }, { "id": "iso_22301", "section": "\u00a78.4.5", "title": "Recovery" }, { "id": "nist_sp800_34", "section": "\u00a74.2", "title": "Activation and notification phase" }, { "id": "nist_csf", "section": "RC.RP", "title": "Incident recovery plan execution" } ], "sources": [ { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RG-06 Resilience Incident Response Integration control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RG-06 Resilience Incident Response Integration control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RG-06 Resilience Incident Response Integration control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RG-06 Resilience Incident Response Integration control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_azure_reliability_2024", "title": "Microsoft Azure Resiliency & BCDR", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/azure/reliability/overview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_azure_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Azure Resiliency & BCDR requirements informing the apeiris://resilience/controls/RG-06 Resilience Incident Response Integration control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Define AI-specific incident severity tiers mapped to resilience impact: Severity 1 for RTO/RPO breach of Tier 1 AI systems; Severity 2 for Tier 2 breaches or Tier 1 near-misses. Publish escalation paths that route Severity 1 and 2 AI resilience incidents to the named accountable executive within defined timeframes. Require post-incident resilience review for all Severity 1 incidents and any incident resulting in governance escalation.", "steps": [ "Define AI resilience incident severity classifications and map them to enterprise incident severity tiers; publish in the incident management runbook.", "Document escalation paths for AI resilience incidents that result in RTO/RPO breaches, including timeframes for notifying the named accountable executive and the Resilience Steering Committee.", "Establish a post-incident resilience review process for Severity 1 incidents: review must assess whether governance controls failed, RTO/RPO breach must be recorded as a finding, and remediation must be tracked in the governance committee.", "Integrate AI resilience incident metrics into the enterprise incident management reporting dashboard and ensure they flow into board-level resilience reporting." ], "site_reliability": { "summary": "SRE incident response must integrate resilience governance obligations: declaring when an incident constitutes an RTO/RPO breach, escalating per the defined path, and ensuring post-incident reviews meet governance requirements.", "actions": [ "Update incident response runbooks to include resilience governance escalation triggers and timeframes.", "Ensure incident tracking systems capture RTO/RPO breach status as a mandatory field for AI system incidents." ], "failure_signals": [ "RTO/RPO breaches not recorded in the incident tracking system as governance findings.", "Post-incident reviews completed without resilience governance input for Severity 1 events." ] }, "grc_auditor": { "summary": "Incident integration is a key indicator of whether governance controls are operationally effective. Auditors should verify that AI resilience incidents are captured, escalated, and reviewed in accordance with the defined process.", "actions": [ "Pull the incident log for the review period and filter for AI system incidents; verify severity classifications align with the defined AI resilience taxonomy.", "Sample all Severity 1 AI resilience incidents from the last 12 months and verify post-incident resilience reviews were completed and findings tracked.", "Verify that RTO/RPO breaches are reflected in board-level resilience reporting." ], "metrics": [ "AI resilience incidents with correct severity classification: target 100%.", "Severity 1 incidents with completed post-incident resilience review: target 100%.", "RTO/RPO breaches reflected in governance reporting within required timeframe: target 100%." ], "failure_signals": [ "AI system incidents classified without reference to resilience impact tiers.", "Post-incident reviews completed without governance input for Severity 1 events.", "RTO/RPO breach incidents not appearing in board-level resilience reports." ] }, "it_operations": { "summary": "Operations teams are typically the first responders for AI resilience incidents. They must understand the governance integration requirements and trigger the correct escalation paths.", "actions": [ "Train operations staff on AI resilience severity classifications and governance escalation triggers.", "Ensure incident tracking system fields capture AI resilience-specific data including RTO/RPO status and recovery decision authority." ], "failure_signals": [ "Operations teams routing AI resilience incidents through general IT processes without triggering resilience governance escalation.", "Incident records missing RTO/RPO breach status for AI system outages." ] }, "business_continuity": { "summary": "AI resilience incidents must flow through the same continuity machinery as other disruptions: shared classification, escalation, and post-incident review \u2014 so AI failures activate business continuity when thresholds are met.", "actions": [ "Integrate AI incident classifications with enterprise continuity activation criteria.", "Ensure continuity plans reference AI incident escalation paths and vice versa.", "Include AI failure scenarios in enterprise-level continuity exercises, not just technical drills." ], "failure_signals": [ "A major AI outage never triggers continuity procedures because criteria don't map.", "AI incidents and continuity events are reviewed in separate forums with no shared lessons.", "Enterprise exercises omit AI dependencies that the BIA rates critical." ] }, "security_architect": { "summary": "Resilience incidents are often security incidents in disguise (and vice versa): integration must guarantee joint triage, shared evidence handling, and coordinated response authority.", "actions": [ "Define joint triage criteria that route ambiguous AI incidents to both security and resilience response.", "Align evidence-handling procedures so resilience recovery does not destroy forensic artifacts.", "Exercise combined scenarios (e.g., ransomware forcing AI system recovery) with both teams." ], "failure_signals": [ "Recovery actions wipe forensic evidence before security completes triage.", "Security and resilience teams run parallel, conflicting responses to the same event.", "No playbook exists for security-caused AI recovery scenarios." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "AI resilience incidents are rarely classified distinctly from general IT incidents in enterprise incident management processes, resulting in governance oversight gaps for AI-specific recovery failures." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "cloud-native", "multi-tenant" ], "implementers": [ "CISO", "IT Operations", "BCM Team" ], "frameworks": [ { "framework": "dora", "requirement_id": "Art. 17-23", "fit": "direct", "rationale": "DORA Articles 17-23 establish comprehensive requirements for ICT-related incident management, classification, reporting, and post-incident review. For covered entities, integrating AI resilience incidents into the enterprise incident management process that meets DORA standards is a legal obligation.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "iso_22301", "requirement_id": "\u00a78.4.5", "fit": "direct", "rationale": "ISO 22301:2019 \u00a78.4.5 (Recovery) requires documented procedures for recovering business activities after an incident. Integrating AI resilience incidents into enterprise incident management ensures those recovery procedures are triggered consistently for AI failures.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a74.2", "fit": "direct", "rationale": "NIST SP 800-34 Rev 1 \u00a74.2 defines the activation and notification phase \u2014 activation criteria, notification procedures, and outage assessment \u2014 that begins contingency plan execution. Integrating AI resilience triggers into enterprise incident response implements exactly this phase.", "normative_force": "voluntary-standard", "source_version": "Rev. 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "RC.RP", "fit": "direct", "rationale": "NIST CSF 2.0 RC.RP (Incident Recovery Plan Execution) requires that recovery plans are executed during and after cybersecurity incidents. The integration of resilience governance into incident response ensures this is done with appropriate oversight.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.05", "fit": "partial", "rationale": "COBIT DSS04.05 addresses exercising, testing, and reviewing continuity plans, which includes the post-incident review requirement. The partial fit reflects that COBIT addresses testing rather than the incident integration governance process specifically.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "aws_reliability", "requirement_id": "Failure management \u2014 recovery procedures", "fit": "partial", "rationale": "The AWS Well-Architected Reliability Pillar's failure management focus area covers incident detection, escalation, recovery execution, and post-incident learning that must be integrated into organizational incident response processes. The Resilience Incident Response Integration control governs how these AWS failure management practices connect to resilience governance, ensuring that RTO/RPO breach incidents trigger governance-level review, recovery decisions are made by the designated authority, and failures drive documented remediation in the resilience program.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Incident management and postmortem practices", "fit": "direct", "rationale": "Google SRE's incident management framework and blameless postmortem practices define the operational process for responding to system failures \u2014 severity classification, escalation, structured recovery, and post-incident analysis \u2014 that this control requires to be integrated with resilience governance. The Resilience Incident Response Integration control mandates that these SRE practices are formally connected to the governance layer so that RTO/RPO breaches are recorded as governance findings, post-incident reviews include governance input, and lessons learned feed back into the resilience program through the Resilience Steering Committee.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_azure_resil", "requirement_id": "Failover configuration and recovery objectives", "fit": "partial", "rationale": "Microsoft Azure BCDR's failover configuration and recovery objective guidance defines the technical recovery procedures and availability commitments that are the operational expression of RTO/RPO targets for cloud-hosted AI systems. The Resilience Incident Response Integration control ensures these technical recovery procedures are connected to the governance layer so that failover decisions are made by the designated authority, failures against recovery objectives are correctly classified as governance incidents, and post-incident reviews assess whether BCDR configurations performed as designed.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RG-06", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "All production AI system incidents must be classified against the published AI resilience severity taxonomy, RTO/RPO breach events must trigger the documented governance escalation path within the required timeframe, and all Severity 1 incidents must have a completed post-incident resilience review within 30 days of resolution that includes governance findings.", "evidence_required": [ "Published AI resilience incident severity taxonomy with explicit mapping of RTO/RPO breach scenarios to severity levels, documented escalation timeframes per severity, and recovery decision authority matrix", "Incident management system records for all AI system incidents in the review period showing severity classification field populated using the AI resilience taxonomy", "Governance escalation records for all Severity 1 and Severity 2 AI resilience incidents, including timestamp of executive notification and Resilience Steering Committee notification against the required timeframe", "Post-incident resilience review reports for all Severity 1 incidents, confirming completion within 30 days of resolution and inclusion of governance control failure assessment", "Board-level resilience reporting artifacts showing RTO/RPO breach incidents are reflected in governance reporting with correct severity and timeline" ], "machine_tests": [ "Query incident management system for all AI system incidents in the review period and check for missing or null severity classification fields \u2192 assert 0 unclassified incidents", "Query for Severity 1 AI resilience incidents without a post-incident review record linked and marked complete within 30 days of resolution date \u2192 assert 0 violations", "Query governance escalation log for Severity 1 incidents and verify executive notification timestamp is within the defined escalation timeframe from incident declaration \u2192 assert 0 late notifications", "Query the board reporting record for RTO/RPO breach incidents and verify each appears in the next governance reporting cycle \u2192 assert 0 breaches missing from reporting" ], "human_review": [ "Review a sample of AI incident records to verify severity classifications accurately reflect the AI resilience taxonomy, not general IT priority levels, and that RTO/RPO breach status is recorded as a mandatory field", "Assess post-incident review reports for Severity 1 incidents to confirm they address governance control failures and resilience program gaps, not only technical root causes and engineering remediation steps", "Verify that recovery decisions documented in incident records identify the authority who made them, confirming the designated decision authority structure was followed rather than improvised during the incident" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Routing AI resilience incidents through the general IT incident process using IT severity levels without applying the AI resilience severity taxonomy, causing RTO/RPO breach events to be misclassified and governance escalation to be skipped", "Completing post-incident reviews as technical-only retrospectives focused on engineering root causes without governance input on whether resilience controls failed and whether the incident constitutes a governance finding", "Failing to record RTO/RPO breach incidents as governance findings in the resilience risk register and board reporting, keeping repeated failures invisible to governance oversight", "Making recovery decisions during incidents without documenting which authority made them, defeating the defined recovery decision authority matrix and creating unverifiable recovery records", "Treating governance escalation notification timeframes as aspirational rather than binding, with escalation happening after-the-fact when the incident is already resolved" ], "update_status": "current", "layer_code": "RG" }, { "id": "RG-07", "layer": "RG", "plane": "control", "name": "Resilience Program Metrics and Board Reporting", "plain": "The resilience program must define, track, and report a standardized set of metrics covering RTO/RPO compliance rates, recovery test pass rates, resilience coverage of the AI system portfolio, and incident frequency by severity. These metrics must be reported to the Resilience Steering Committee quarterly and to the board or audit committee at least annually.", "threat": { "tags": [ "invisible-risk", "metric-gaming", "reporting-gap", "governance-blindspot" ], "desc": "Without standardized metrics and board-level reporting, resilience program performance is invisible to governance bodies. Deteriorating compliance rates are not detected until a major incident exposes them. Without board-level visibility, resilience investment is difficult to justify and program failures are not attributed to governance oversight failures. Metrics that are not standardized enable program teams to report selectively, masking systemic weaknesses." }, "standard": [ { "id": "iso_22301", "section": "\u00a79.1", "title": "Monitoring, measurement, analysis and evaluation" }, { "id": "dora", "section": "Art. 6(5)", "title": "ICT risk management framework review and reporting" }, { "id": "cobit_dss04", "section": "DSS04.04", "title": "Exercise, test and review the BCP and DRP \u2014 results reported to management" }, { "id": "nist_csf", "section": "GV.OV-03", "title": "Organizational cybersecurity risk management performance is evaluated and reviewed" } ], "sources": [ { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RG-07 Resilience Program Metrics and Board Reporting control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RG-07 Resilience Program Metrics and Board Reporting control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RG-07 Resilience Program Metrics and Board Reporting control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Define a resilience metrics framework covering: (1) RTO/RPO compliance rate by tier (target: 100% of tests meet tier threshold), (2) Recovery test pass rate (target: \u226595% per quarter), (3) AI system resilience coverage (percentage of production AI systems with current test results), (4) Mean time to recovery (MTTR) by tier, (5) Severity 1 incident frequency and trend. Produce a quarterly metrics dashboard for the Resilience Steering Committee and an annual board report.", "steps": [ "Define the standard resilience metrics set and document target values and measurement methodology for each metric in a metrics register.", "Implement automated metric collection where possible; document manual collection processes with designated owners for each metric.", "Produce and publish a quarterly resilience metrics dashboard for the Resilience Steering Committee, including trend data and commentary on metrics outside target range.", "Develop an annual board resilience report template covering program status, key metrics, material risks, and investment recommendations; obtain Resilience Steering Committee approval before submission to the board." ], "site_reliability": { "summary": "SRE teams are the primary source of data for resilience metrics including test pass rates and MTTR. They must ensure test results are captured in a form that feeds the governance metrics framework.", "actions": [ "Ensure chaos engineering and DR test results are recorded in a system that feeds the governance metrics dashboard.", "Report MTTR data by AI system tier following each recovery test and production incident." ], "failure_signals": [ "Test results recorded in engineering-only systems that do not feed governance reporting.", "MTTR data not available at tier granularity for governance reporting." ] }, "grc_auditor": { "summary": "Board-level resilience reporting is a key governance control that provides independent visibility into program performance. Auditors should verify that metrics are accurate, complete, and presented with sufficient context for governance decision-making.", "actions": [ "Review the metrics register and verify that target values and measurement methodologies are documented and approved.", "Cross-reference reported metrics against underlying source data for at least two quarters to verify accuracy.", "Review board resilience reports for completeness: do they include material risks, trend data, and management commentary on metrics outside target range?" ], "metrics": [ "Quarterly resilience dashboard delivered to Resilience Steering Committee: target 100% of quarters.", "Annual board resilience report delivered and acknowledged by the board: target 100% compliance.", "Metrics with documented measurement methodology and named owner: target 100%." ], "failure_signals": [ "Board reports that present only favorable metrics and omit metrics outside target range.", "Metrics without documented measurement methodology, enabling inconsistent calculation.", "Governance reporting that lags by more than 30 days from the end of the reporting period." ] }, "business_continuity": { "summary": "BCM owns the program metrics framework and is responsible for ensuring the metrics accurately reflect resilience program performance. BCM should lead the annual board report preparation and present findings to the governance committee.", "actions": [ "Own the resilience metrics register and conduct an annual review of metric definitions and target values.", "Prepare the annual board resilience report and present it to the Resilience Steering Committee before board submission." ], "failure_signals": [ "Metrics framework not reviewed in over 12 months, potentially using stale definitions.", "BCM not involved in board report preparation, resulting in loss of program context." ] }, "it_operations": { "summary": "Operational telemetry feeds the board metrics: MTTR, test pass rates, and objective-compliance numbers must be produced from systems of record, reproducibly, on the reporting calendar.", "actions": [ "Automate metric production (MTTR, RTO compliance, test coverage) from operational systems of record.", "Validate metric definitions with governance so reported numbers match operational meaning.", "Flag data-quality gaps in metric inputs before reports reach the board." ], "failure_signals": [ "Board metrics are hand-assembled and cannot be reproduced.", "Reported MTTR uses a different definition than operations tracks.", "Metric trends shift because of data gaps, not real performance change." ] }, "security_architect": { "summary": "Board reporting should include the security posture of resilience capabilities: recovery-path security status and provider trust risks belong beside availability metrics.", "actions": [ "Add security-of-resilience indicators (failover security parity, backup immutability status) to the metrics set.", "Review the reporting pack for resilience claims that overstate security assurance.", "Report security-driven resilience risks (e.g., unassessed fallback providers) with owners and trends." ], "failure_signals": [ "Board packs report availability green while recovery paths carry known security gaps.", "No metric tracks security parity of standby environments.", "Security risks to resilience surface at board level only after incidents." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Standardized resilience metrics dashboards with board-level reporting are uncommon for AI systems specifically; most board reporting covers general IT availability without AI-specific resilience indicators." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "BCM Team", "Chief Risk Officer", "VP Engineering" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a79.1", "fit": "direct", "rationale": "ISO 22301:2019 \u00a79.1 requires organizations to evaluate the performance of the BCMS and determine what to monitor, when, and how to analyze and evaluate the results. This directly mandates the metrics and reporting framework this control implements.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 6(5)", "fit": "direct", "rationale": "EU DORA Article 6(5) requires the ICT risk management framework to be documented, reviewed at least yearly (and upon major incidents or supervisory instruction) and continuously improved, with a report on the review available to the competent authority on request. Board-level resilience metrics reporting is how the management body exercises that review in practice (Article 6(8) defines the digital operational resilience strategy itself).", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "cobit_dss04", "requirement_id": "DSS04.04", "fit": "direct", "rationale": "COBIT 2019 DSS04.04 requires continuity plans to be exercised and tested with the results reviewed and reported through management. Test pass rates and review outcomes are core inputs to this control's board reporting (DSS04.06 covers training only).", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "GV.OV-03", "fit": "direct", "rationale": "NIST CSF 2.0 GV.OV-03 requires organizational cybersecurity risk management performance to be evaluated and reviewed for adjustments needed. The resilience metrics and board reporting framework produces and reviews exactly that performance evidence.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.1", "fit": "partial", "rationale": "NIST SP 800-34 Rev 1 \u00a73.1 requires the contingency planning policy statement to define roles, responsibilities, and reporting requirements for the program. Program metrics and management reporting flow from those policy-level reporting requirements (there is no \u00a72.4 in the process chapters).", "normative_force": "voluntary-standard", "source_version": "Rev. 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "openai_preparedness", "requirement_id": "\u00a75.1 Internal governance \u2014 Capabilities and Safeguards Reports", "fit": "adjacent", "rationale": "Under the OpenAI Preparedness Framework v2, Capabilities Reports and Safeguards Reports are compiled and escalated to the Safety Advisory Group and leadership \u2014 a structured risk-reporting cadence that binds OpenAI. It is an informative analogue for the board-level resilience reporting this control requires.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "aws_reliability", "requirement_id": "REL 12 \u2014 Test reliability (documented outcomes)", "fit": "partial", "rationale": "AWS Well-Architected Reliability Pillar REL 12 (Test reliability) requires reliability tests to be conducted and their outcomes documented and analyzed \u2014 recovery test pass rates and MTTR trends are the operational metrics this control aggregates for board reporting.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "SLO/SLA management and error budget policies", "fit": "direct", "rationale": "Google SRE's SLO/SLA management and error budget framework defines the quantitative reliability metrics \u2014 availability, MTTR, error budget consumption rate, SLO compliance \u2014 that represent the operational expression of resilience risk appetite and form the core metrics this control requires organizations to track and report. The Resilience Program Metrics and Board Reporting control elevates these SRE operational metrics into a formal governance reporting framework with standardized definitions, approved target values, quarterly steering committee dashboards, and annual board visibility.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RG-07", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "The resilience program must produce a quarterly metrics dashboard delivered to the Resilience Steering Committee within 30 days of each quarter end covering all five required metric categories with documented targets and trend data, and an annual board resilience report acknowledged by the board or audit committee within the required cycle.", "evidence_required": [ "Quarterly resilience metrics dashboards for all four quarters in the review period with delivery timestamps confirming receipt by the Resilience Steering Committee within 30 days of quarter end", "Metrics register with documented target values, measurement methodology, and named metric owner for each of the five required metric categories (RTO/RPO compliance rate, recovery test pass rate, AI system resilience coverage, MTTR by tier, Severity 1 incident frequency)", "Annual board resilience report with Resilience Steering Committee approval date preceding board submission, and board or audit committee acknowledgment record", "Source data records (test execution logs, incident records) cross-referenceable against at least two sampled quarters of reported dashboard values to verify metric accuracy", "Evidence that metrics outside target range in any quarter have associated management commentary and governance discussion record in the Resilience Steering Committee minutes" ], "machine_tests": [ "Query the metrics delivery system for quarterly dashboard publication timestamps in the review period and compare to quarter-end dates \u2192 assert all dashboards delivered within 30 days of quarter end with 0 late deliveries", "Query the metrics register for all required metric categories and check for missing measurement methodology or named owner entries \u2192 assert 0 metrics without both fields populated", "Query the board reporting record for the annual resilience report \u2192 assert a Resilience Steering Committee approval date preceding the board submission date exists within the current annual cycle", "Query MTTR reporting fields in dashboard records and verify tier-granularity breakdown is present (Tier 1/2/3 reported separately) \u2192 assert all dashboards contain per-tier MTTR values rather than a program average" ], "human_review": [ "Cross-reference reported RTO/RPO compliance rates and recovery test pass rates from two sampled quarters against the underlying test execution records and incident logs to verify accuracy and completeness", "Assess the annual board resilience report for inclusion of metrics that were outside target range during the year, and verify management commentary explains root cause and remediation rather than omitting unfavorable results", "Review trend data in quarterly dashboards for metrics showing sustained deterioration and verify that a governance discussion and remediation plan were initiated in the Resilience Steering Committee record" ], "blocking_effect": "advisory", "normative_status": "binding-law", "anti_patterns": [ "Presenting only metrics that are within target range in board reports while omitting or downplaying metrics with unfavorable results, preventing the board from having an accurate picture of resilience program performance", "Defining metrics without documented measurement methodologies, enabling inconsistent calculation across reporting periods that makes trend comparison unreliable", "Delivering quarterly dashboards more than 30 days after quarter end, reducing their value as timely inputs to governance decision-making and potentially violating regulatory reporting timelines", "Reporting MTTR as a program-wide average rather than at tier granularity, masking critical AI system underperformance behind the aggregate of lower-criticality systems", "Using AI system resilience coverage as the primary reported metric while deprioritizing RTO/RPO compliance rate, allowing broad nominal coverage without evidence that covered systems can actually meet their recovery objectives" ], "update_status": "current", "layer_code": "RG" }, { "id": "RG-08", "layer": "RG", "plane": "lifecycle", "name": "ResilienceAttestation Production", "plain": "The organization must produce a ResilienceAttestation artifact that synthesizes evidence from all resilience domain controls (RV, RP, RO, FO, RE, and RG layers) into a signed, machine-readable attestation certifying that resilience controls are implemented, tested, and evidenced to the required standard. This attestation is the authoritative evidence artifact for the resilience domain.", "threat": { "tags": [ "attestation-gap", "unverifiable-claim", "evidence-fragmentation", "compliance-theater" ], "desc": "Without a formal attestation artifact, resilience compliance claims are unverifiable assertions rather than evidence-backed commitments. Governance bodies, regulators, and downstream consumers of AI systems cannot independently verify that resilience controls have been implemented and tested. Evidence scattered across disparate systems cannot be synthesized into a coherent assurance picture, enabling compliance theater where controls are claimed but not evidenced." }, "standard": [ { "id": "iso_22301", "section": "\u00a79.2", "title": "Internal audit and evidence of conformity" }, { "id": "dora", "section": "Art. 6(6)", "title": "Internal audit of the ICT risk management framework" }, { "id": "nist_800_160_v2", "section": "\u00a73.2", "title": "Cyber resiliency analysis \u2014 evidence of resilience properties" }, { "id": "nist_csf", "section": "GV.OV-01", "title": "Cybersecurity risk management strategy outcomes are reviewed to inform and adjust strategy" } ], "sources": [ { "id": "anthropic_rsp", "title": "Anthropic Responsible Scaling Policy v3.3", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "3.3", "published_on": "2026-05-26", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/responsible-scaling-policy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_rsp", "relationship": "informative_reference", "rationale": "Establishes Anthropic Responsible Scaling Policy v3.3 requirements informing the apeiris://resilience/controls/RG-08 ResilienceAttestation Production control.", "reviewed_on": "2026-07-02" }, { "id": "openai_preparedness", "title": "OpenAI Preparedness Framework v2", "authority": "OpenAI, L.L.C.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2", "published_on": "2025-04-15", "retrieved_on": "2026-06-29", "canonical_url": "https://cdn.openai.com/pdf/18a02b5d-6b67-4cec-ab64-68cdfbddebcd/preparedness-framework-v2.pdf", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "openai_preparedness", "relationship": "informative_reference", "rationale": "Establishes OpenAI Preparedness Framework v2 requirements informing the apeiris://resilience/controls/RG-08 ResilienceAttestation Production control.", "reviewed_on": "2026-07-02" }, { "id": "aws_well_arch_reliability_2024", "title": "AWS Well-Architected Reliability Pillar", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/wellarchitected/latest/reliability-pillar/welcome.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_well_arch_reliability_2024", "relationship": "informative_reference", "rationale": "Establishes AWS Well-Architected Reliability Pillar requirements informing the apeiris://resilience/controls/RG-08 ResilienceAttestation Production control.", "reviewed_on": "2026-07-01" }, { "id": "google_sre_workbook_2018", "title": "Site Reliability Engineering (SRE Book, 2016)", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2016", "published_on": "2016-04-01", "retrieved_on": "2026-06-29", "canonical_url": "https://sre.google/sre-book/table-of-contents/", "license": "Creative Commons", "status": "current", "flagship": false, "source_id": "google_sre_workbook_2018", "relationship": "informative_reference", "rationale": "Establishes Site Reliability Engineering (SRE Book, 2016) requirements informing the apeiris://resilience/controls/RG-08 ResilienceAttestation Production control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Define the ResilienceAttestation schema incorporating evidence references from all 6 resilience layers (RV, RP, RO, FO, RE, RG). Implement an evidence aggregation pipeline that collects attestation inputs from each layer's terminal control and assembles them into a signed artifact. The attestation must include: evidence_id, actor, verdict (pass/fail/conditional), confidence, collected_at, valid_until, blocking_effect, integrity.hash (sha256), and integrity.signature (Ed25519). Produce the attestation on a scheduled cadence and on demand for regulatory submissions.", "steps": [ "Define the ResilienceAttestation JSON schema with all required fields from the Apeiris evidence ontology; publish the schema to apeiris-control-core/evidence.schema.json.", "Implement an evidence collection pipeline that pulls terminal control evidence from each resilience layer (RV-08, RP-08, RO-08, FO-08, RE-08, RG-07 metrics) and validates completeness before attestation generation.", "Implement Ed25519 signing of the assembled attestation artifact; store the attestation in an append-only evidence store with sha256 integrity verification.", "Establish a production schedule for ResilienceAttestation generation: automated daily for monitoring continuity, on-demand for regulatory submissions, and after any Severity 1 incident resolution.", "Publish the ResilienceAttestation endpoint at the apeiris://resilience/controls/RG-08 canonical URI and ensure it is accessible to authorized consumers via the integration API." ], "site_reliability": { "summary": "The attestation pipeline is a production system that must itself be resilient. SRE teams are responsible for the availability and integrity of the attestation generation and signing infrastructure.", "actions": [ "Implement monitoring for the attestation pipeline covering evidence collection failures, signing failures, and delivery latency.", "Ensure the attestation infrastructure has its own recovery objectives documented and tested independently of the systems it attests." ], "failure_signals": [ "Attestation pipeline failures that result in gaps in the attestation audit trail.", "Signing key rotation not performed within the required cadence, invalidating historical attestation signatures." ] }, "grc_auditor": { "summary": "The ResilienceAttestation is the primary audit artifact for the resilience domain. Auditors should verify that attestations are complete, signed, and that the underlying evidence accurately reflects control implementation status.", "actions": [ "Request the most recent ResilienceAttestation artifact and verify schema compliance, signature validity, and evidence completeness.", "Cross-reference the verdict in the attestation against the underlying control evidence for a sample of 3 resilience controls.", "Verify that attestations with conditional or fail verdicts have associated remediation tracking in the governance committee record." ], "metrics": [ "ResilienceAttestation produced on schedule with no gaps in the audit trail: target 100%.", "Attestation signature verification pass rate: target 100%.", "Attestations with conditional or fail verdicts that have an associated governance remediation record: target 100%." ], "failure_signals": [ "Attestation gaps in the audit trail exceeding 48 hours for monitored systems.", "Attestation signatures that fail Ed25519 verification, indicating tampering or key management failure.", "Attestations with fail verdicts that do not have associated governance escalation records." ] }, "business_continuity": { "summary": "The ResilienceAttestation is the authoritative evidence artifact that BCM uses to demonstrate program compliance to regulators and the board. BCM should participate in attestation schema design and review attestation content before regulatory submission.", "actions": [ "Review ResilienceAttestation content prior to regulatory submission to ensure verdicts accurately reflect BCM program status.", "Ensure the attestation schema covers all BCM program obligations including those beyond AI systems." ], "failure_signals": [ "Attestations submitted to regulators without BCM review, resulting in inaccurate verdicts.", "Attestation schema gaps that do not reflect material BCM program elements." ] }, "security_architect": { "summary": "The attestation artifact must be designed with integrity and authenticity guarantees. The security architect is responsible for the signing architecture, key management, and integrity verification design.", "actions": [ "Design the Ed25519 signing architecture including key generation, storage in HSM or equivalent, rotation policy, and revocation procedures.", "Define the sha256 integrity hash scope and verify it covers all material attestation fields before the signature is applied." ], "failure_signals": [ "Signing keys stored outside of HSM or equivalent hardware security boundary.", "Integrity hash scope that excludes material attestation fields, enabling partial tampering without detection." ] }, "it_operations": { "summary": "Attestation production consumes operational evidence: control-state data, test results, and incident records must be complete, current, and machine-collectable when the attestation is generated.", "actions": [ "Automate collection of operational evidence (test results, incident metrics, control states) feeding the attestation pipeline.", "Reconcile operational reality against attestation claims before each signing cycle.", "Alert when evidence inputs go stale relative to the attestation cadence." ], "failure_signals": [ "Attestations are generated from evidence that predates major operational changes.", "Operational records contradict signed attestation claims.", "Evidence collection requires manual assembly each cycle." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Machine-readable, signed attestation artifacts for AI resilience domains do not exist in most enterprises. This control defines the target state; current state is typically informal compliance documentation without integrity guarantees." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise", "cloud-native" ], "implementers": [ "Security Architecture", "BCM Team", "Platform Engineering" ], "frameworks": [ { "framework": "iso_22301", "requirement_id": "\u00a79.2", "fit": "direct", "rationale": "ISO 22301:2019 \u00a79.2 requires organizations to conduct internal audits and maintain documented information as evidence of conformity to the BCMS requirements. The ResilienceAttestation is the machine-readable, integrity-protected form of this conformity evidence.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dora", "requirement_id": "Art. 6(6)", "fit": "direct", "rationale": "EU DORA Article 6(6) requires the ICT risk management framework to be subject to regular internal audit by auditors with sufficient ICT risk knowledge and independence. A signed ResilienceAttestation gives internal audit a machine-verifiable statement of AI resilience control posture to audit against.", "normative_force": "binding-law", "source_version": "2022/2554", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_800_160_v2", "requirement_id": "\u00a73.2", "fit": "direct", "rationale": "NIST SP 800-160 Vol 2 Rev 1 \u00a73.2 describes cyber resiliency analysis, which produces evidence about whether a system's resilience properties actually hold. The ResilienceAttestation is the formal, signed packaging of that evidence for AI systems.", "normative_force": "voluntary-standard", "source_version": "Rev 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_csf", "requirement_id": "GV.OV-01", "fit": "direct", "rationale": "NIST CSF 2.0 GV.OV-01 requires cybersecurity risk management strategy outcomes to be reviewed to inform and adjust strategy and direction. The ResilienceAttestation encapsulates measured resilience outcomes in a reviewable, machine-readable artifact.", "normative_force": "voluntary-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_27031", "requirement_id": "Cl. 13", "fit": "partial", "rationale": "ISO/IEC 27031:2025 Clause 13 requires management evaluation of ICT readiness, drawing on documented performance evidence. The ResilienceAttestation is a signed, machine-readable form of that documented evidence.", "normative_force": "voluntary-standard", "source_version": "2025", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_sp800_34", "requirement_id": "\u00a73.5", "fit": "adjacent", "rationale": "NIST SP 800-34 Rev 1 \u00a73.5 requires test, training, and exercise results to be documented. Attestation production consumes those documented results as evidence inputs; the fit is adjacent because 800-34 addresses documentation rather than signed attestation artifacts.", "normative_force": "voluntary-standard", "source_version": "Rev. 1", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "anthropic_rsp", "requirement_id": "Risk Reports and capability determinations \u2014 documented safeguard evidence", "fit": "adjacent", "rationale": "Under the Responsible Scaling Policy (v3.3), Anthropic documents capability determinations and safeguard adequacy in Risk Reports reviewed through its internal governance process. That evidence-artifact discipline binds Anthropic and is the closest analogue to the signed ResilienceAttestation this control requires deployers to produce.", "normative_force": "best-practice", "source_version": "3.3", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "openai_preparedness", "requirement_id": "Capabilities and Safeguards Reports (\u00a75.1 Internal governance)", "fit": "adjacent", "rationale": "The OpenAI Preparedness Framework v2 requires Capabilities Reports and Safeguards Reports \u2014 documented evidence that capability levels were measured and safeguards assessed \u2014 reviewed by the Safety Advisory Group. PF v2 has no 'safety case' construct; these reports are the real evidence artifacts, and they bind OpenAI rather than deployers.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "aws_reliability", "requirement_id": "REL 12 \u2014 Test reliability (documented evidence)", "fit": "adjacent", "rationale": "AWS Well-Architected Reliability Pillar REL 12 (Test reliability) requires documented, retained evidence of reliability testing outcomes. The ResilienceAttestation packages that evidence in signed, machine-readable form.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "google_sre", "requirement_id": "Production readiness review", "fit": "adjacent", "rationale": "Google SRE's Production Readiness Review produces documented readiness sign-offs that confirm a system meets reliability and operational standards before production launch, establishing the principle that formal evidence artifacts gate deployment. The ResilienceAttestation Production control formalizes and extends this concept by producing a machine-readable, Ed25519-signed attestation covering the full resilience control domain, providing a persistent, tamper-evident artifact rather than an informal sign-off that satisfies the evidence requirements of regulatory submissions and governance attestation cycles.", "normative_force": "best-practice", "source_version": "2016", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://resilience/controls/RG-08", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "A valid ResilienceAttestation artifact must be produced on the required daily cadence, contain evidence references from all six resilience layer terminal controls (RV-08, RP-08, RO-08, FO-08, RE-08, RG-07), pass Ed25519 signature verification against the published public key, and have no gaps in the attestation audit trail exceeding 48 hours.", "evidence_required": [ "ResilienceAttestation JSON artifacts conforming to the published schema, with all required fields populated: evidence_id, actor, verdict, confidence, collected_at, valid_until, blocking_effect, integrity.hash (sha256), and integrity.signature (Ed25519)", "Evidence collection pipeline logs showing successful pull from each of the six layer terminal controls (RV-08, RP-08, RO-08, FO-08, RE-08, RG-07) with completion timestamps and evidence reference IDs for each attestation cycle", "Key management records confirming Ed25519 signing key generated and stored in HSM with documented rotation policy and revocation procedure, and key rotation log showing rotations completed within the required cadence", "Attestation pipeline monitoring logs for the review period confirming no unresolved evidence collection or signing failures, with any pipeline errors documented alongside their resolution records", "Attestation audit trail records showing no gaps exceeding 48 hours in the review period, with attestation_id and timestamp for each cycle" ], "machine_tests": [ "Load the most recent ResilienceAttestation artifact, verify Ed25519 signature against the published public key \u2192 assert signature_valid=true with 0 verification failures", "Parse the evidence_references array in the most recent attestation and verify all six layer terminal control IDs are present (RV-08, RP-08, RO-08, FO-08, RE-08, RG-07) \u2192 assert count=6 with no missing layer references", "Query the attestation audit trail for gaps between consecutive attestation timestamps exceeding 48 hours in the review period \u2192 assert 0 gaps", "Recompute the sha256 hash over material attestation fields and compare to the integrity.hash value stored in the artifact \u2192 assert computed_hash equals stored_hash with no mismatch" ], "human_review": [ "Cross-reference attestation verdicts for three sampled resilience controls against their underlying evidence artifacts to confirm verdicts accurately reflect control implementation and test status rather than being synthesized from summary inputs", "Review the Ed25519 key management documentation for HSM storage confirmation, rotation cadence compliance, and documented revocation procedures, and verify the key used for current attestations is within its authorized rotation window", "Assess whether attestations with fail or conditional verdicts in the review period have associated governance escalation records in the Resilience Steering Committee minutes, confirming that non-passing attestations triggered governance action" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Generating attestation artifacts by assembling summary compliance documentation rather than by pulling structured evidence from each layer terminal control, producing attestations that assert compliance without traceable evidence references", "Storing Ed25519 signing keys in software key stores or configuration files rather than HSMs, exposing the signing authority to compromise and invalidating the integrity guarantee of all attestations signed with the exposed key", "Producing attestations manually through periodic document assembly rather than via an automated evidence aggregation pipeline, creating gaps in the attestation audit trail and making on-demand regulatory submission impractical", "Accepting audit trail gaps exceeding 48 hours without declaring a governance incident, allowing periods of unattested control status to pass without detection or remediation", "Omitting failed or conditional controls from the evidence_references array in attestation artifacts to produce a clean pass verdict, creating fraudulent attestations that misrepresent actual control implementation status" ], "update_status": "current", "cross_domain": { "feeds": [ "apeiris://compliance/controls/AU-08" ] }, "layer_code": "RG" } ] } }