{ "dataset": { "meta": { "domain": "data", "domain_slug": "data", "domain_number": 12, "title": "Apeiris Data Control Matrix", "description": "Apeiris Data Control Matrix: 48 machine-readable controls across 6 layers.", "version": "1.1.0", "published": "2026-07-02", "layers": 6, "controls_count": 48, "baseline_controls": [ "DX-01", "DX-08", "DI-01", "DI-08", "DM-01", "DM-08", "DL-01", "DL-08", "DA-01" ], "canonical_prefix": "apeiris://data/controls/", "attestation_artifact": "DataGovernanceAttestation", "attestation_control": "DV-08", "alias_domain": "dataverifier.ai", "frameworks": [ "anthropic_privacy", "aws_lake_formation", "cobit_2019", "dama_dmbok", "data_contract_spec", "databricks_unity", "dcam", "eu_ai_act", "eu_data_act", "eu_data_gov_act", "gdpr", "google_dataplex", "iso_27701", "iso_5259", "iso_8000", "microsoft_purview", "nist_pf", "openlineage", "snowflake_horizon" ], "lenses": [ "data_governance_officer", "legal_counsel", "grc_auditor", "it_operations", "data_engineer" ], "license": "CC BY 4.0", "source": "https://apeiris.ai/domains/data/", "integration_endpoint": "https://apeiris.ai/integration/domains/data-controls-full.json", "source_freshness": { "status": "current", "checked_on": "2026-07-02", "review_cadence": "quarterly" }, "baseline_control_count": 9, "generated_at": "2026-07-02T00:00:00.000Z", "subtitle": "apeiris.ai/domains/data — Apeiris Data", "site": "https://apeiris.ai/domains/data", "corpus_url": "https://apeiris.ai/integration/domains/data-controls-full.json", "schema_version": "1.1.0", "schema_extended_on": "2026-06-29", "extended_schema_fields": [ "validation_objective", "evidence_required", "machine_tests", "human_review", "blocking_effect", "normative_status", "anti_patterns", "update_status" ] }, "controls": [ { "id": "DX-01", "layer": "DX", "plane": "control", "name": "Data Sensitivity Classification Framework for AI", "plain": "All data processed, generated, or consumed by AI systems must be assigned a sensitivity classification from a unified enterprise taxonomy before exchange, storage, or use. The taxonomy covers both personal and non-personal data, AI-derived outputs, model training artifacts, and inference results.", "threat": { "tags": [ "unclassified-data-exposure", "sensitivity-misassignment", "data-leakage", "ai-output-mishandling" ], "desc": "Without a consistent sensitivity taxonomy, AI pipelines routinely ingest or emit data at the wrong protection level. Unclassified training datasets containing PII flow into public-facing models. AI-generated outputs inherit no sensitivity label and are shared without controls. Misclassification silently erodes data governance and creates regulatory exposure under GDPR and the EU AI Act." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 7 Data Security", "title": "Data sensitivity classification requirements" }, { "id": "eu_ai_act", "section": "Art. 10(3)", "title": "Training data quality and relevance" }, { "id": "gdpr", "section": "Art. 5(1)(f)", "title": "Integrity and confidentiality of personal data" }, { "id": "microsoft_purview", "section": "Information Protection — sensitivity labels", "title": "Automated classification and labeling" } ], "sources": [ { "id": "microsoft_purview_data_governance_2024", "title": "Microsoft Purview (Data Governance)", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/purview/purview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_purview_data_governance_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Purview (Data Governance) requirements informing the apeiris://data/controls/DX-01 Data Sensitivity Classification Framework for AI control.", "reviewed_on": "2026-07-01" }, { "id": "iso_5259_1_2024", "title": "ISO/IEC 5259-1:2024 — Data Quality for Analytics and ML — Part 1: Overview, terminology, and examples", "authority": "International Organization for Standardization / International Electrotechnical Commission", "source_type": "standard", "normative_force": "voluntary-standard", "version": "2024", "published_on": "2024-07-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.iso.org/standard/81088.html", "license": "proprietary", "status": "current", "flagship": false, "source_id": "iso_5259_1_2024", "relationship": "implementation_pattern", "rationale": "Establishes ISO/IEC 5259-1:2024 — Data Quality for Analytics and ML — Part 1: Overview, terminology, and examples requirements informing the apeiris://data/controls/DX-01 Data Sensitivity Classification Framework for AI control.", "reviewed_on": "2026-07-02" }, { "source_id": "ms_ifc_agents", "normative_force": "best-practice", "relationship": "informative_reference", "rationale": "Grounds data sensitivity classification for AI: IFC labels each datum with a confidentiality class (public/confidential/read-access list) at ingestion.", "reviewed_on": "2026-07-03", "title": "Information-flow control: Moving toward secure, autonomous agents", "authority": "Microsoft", "canonical_url": "https://commandline.microsoft.com/information-flow-control-moving-toward-secure-autonomous-agents/" } ], "implementation": { "pattern": "Define a five-level sensitivity taxonomy (Public, Internal, Confidential, Restricted, Regulated) with AI-specific sub-types for model weights, training corpora, inference logs, and synthetic data. Enforce classification at data ingestion, API boundary, and storage layer via automated scanning and policy engine.", "steps": [ "Define the enterprise sensitivity taxonomy with at least five levels and explicit AI-specific data type sub-categories covering model weights, training corpora, inference logs, evaluation outputs, and synthetic data.", "Deploy automated classification tooling (e.g., Microsoft Purview, Amazon Macie) to scan all AI data repositories and apply sensitivity labels at scale.", "Integrate classification enforcement into all data pipeline ingestion points so unlabeled data is quarantined pending review before any downstream AI processing.", "Publish the taxonomy as a machine-readable schema accessible to all downstream data consumers, AI system owners, and automated governance tools.", "Establish a quarterly taxonomy review board to address new AI data types and update sub-category definitions as AI system patterns evolve." ], "data_governance_officer": { "summary": "The classification framework is the foundational control for all downstream data exchange governance. Ownership of the taxonomy and its enforcement policy sits with Data Governance.", "actions": [ "Author and version-control the enterprise sensitivity taxonomy document including AI-specific sub-types.", "Chair the quarterly taxonomy review board and document all schema changes with effective dates.", "Verify that AI system data inventories map every dataset to a sensitivity level before system deployment approval." ], "failure_signals": [ "More than 5% of active datasets lack a sensitivity classification.", "AI model training datasets contain unlabeled or misclassified records identified at audit.", "No taxonomy review conducted in the prior six months." ] }, "data_engineer": { "summary": "Classification enforcement is implemented at the pipeline level. Every ingestion, transformation, and exchange step must propagate and honor the sensitivity label.", "actions": [ "Instrument data pipelines to read and enforce sensitivity labels at every stage including transformations that may change the effective classification.", "Implement quarantine logic for unlabeled data with alerting to the data governance queue before any processing proceeds.", "Ensure AI training and inference pipelines emit labeled outputs with inherited or elevated classification when combining data of differing sensitivity." ], "failure_signals": [ "Pipeline processing unlabeled records without triggering a quarantine event.", "Output datasets missing inherited sensitivity labels after transformation.", "Classification tooling scan coverage below 90% of active AI datasets." ] }, "grc_auditor": { "summary": "The taxonomy and its application to AI data assets is a primary audit artifact for GDPR, EU AI Act, and sector-specific compliance reviews.", "actions": [ "Sample 15% of AI datasets and verify sensitivity classification accuracy against the current taxonomy version.", "Confirm automated scan coverage and review false-positive and false-negative rates for classification tooling.", "Check taxonomy versioning records to confirm reviews have occurred at the required cadence." ], "metrics": [ "Dataset classification coverage: target 100% of active AI datasets.", "Classification accuracy on sampled audit: target >= 95%.", "Taxonomy review cadence: at minimum quarterly." ], "failure_signals": [ "Classification coverage below 95% in any AI data domain.", "Two or more material misclassifications identified on the audit sample.", "Taxonomy not reviewed within the past six months." ] }, "legal_counsel": { "summary": "Classification is where legal obligations attach to data. Counsel must confirm the taxonomy's regulated categories map to actual legal definitions — personal data, special categories, trade secrets — so downstream controls inherit the right obligations.", "actions": [ "Review the sensitivity taxonomy annually to confirm regulated categories match current legal definitions under GDPR, the EU AI Act, and applicable sectoral law.", "Approve the legal-hold and regulatory flags each classification level carries so mislabeled data cannot silently shed obligations.", "Advise on classification of novel AI artifacts — model weights, synthetic outputs, inference logs — where legal status is unsettled." ], "failure_signals": [ "Regulated data classes in the taxonomy with no mapped legal basis or obligation.", "AI-derived outputs circulating with no classification and no legal review of their status.", "Classification disputes resolved by engineering convenience rather than documented legal analysis." ] }, "it_operations": { "summary": "Classification only works if labels travel with data through operational systems. Operations enforces labeling at storage, pipeline and platform boundaries and keeps scanning coverage complete.", "actions": [ "Deploy and maintain automated classification scanning across all AI data stores, with coverage tracked as an SLO.", "Enforce label propagation through pipelines and copies so derived data inherits classification.", "Quarantine unlabeled data at ingestion until classification completes." ], "failure_signals": [ "Data stores outside scanning coverage for more than one cycle.", "Labels stripped by pipeline copies or format conversions.", "Quarantine queues bypassed under delivery pressure." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most enterprises apply sensitivity classification to structured data but lack AI-specific sub-types for model artifacts, inference logs, and synthetic outputs." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "eu-high-risk-ai" ], "implementers": [ "Data Governance Office", "Data Engineering", "Platform Engineering" ], "frameworks": [ { "framework": "dama_dmbok", "requirement_id": "Ch. 7", "fit": "direct", "rationale": "DAMA DMBOK 2 Chapter 7 establishes data security classification as a foundational data management capability. It requires a documented classification scheme aligned with business risk and regulatory obligations. The framework explicitly addresses classification of sensitive data across the enterprise data landscape.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "eu_ai_act", "requirement_id": "Art. 10(3)", "fit": "direct", "rationale": "EU AI Act Article 10(3) mandates that training, validation, and testing datasets for high-risk AI systems undergo data governance practices including examination for biases and relevance. Sensitivity classification is a prerequisite for identifying and controlling regulated data types within these datasets.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "gdpr", "requirement_id": "Art. 5(1)(f)", "fit": "direct", "rationale": "GDPR Article 5(1)(f) requires personal data to be processed with appropriate security using technical and organisational measures. Sensitivity classification is the technical mechanism by which appropriate security controls are applied proportionately to data risk. Without classification, proportional protection cannot be demonstrated.", "normative_force": "binding-law", "source_version": "2016/679", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "microsoft_purview", "requirement_id": "Information Protection — sensitivity labels", "fit": "direct", "rationale": "Microsoft Purview Information Protection provides sensitivity labels that can be applied automatically across Microsoft 365 and Azure data estates, extending to AI workloads such as Azure OpenAI datasets and Microsoft Fabric pipelines. It operationalizes the enterprise taxonomy with policy enforcement at the storage and exchange layer.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "nist_pf", "requirement_id": "ID.IM-P1", "fit": "partial", "rationale": "NIST Privacy Framework function IDENTIFY requires organizations to inventory data elements with their sensitivity and processing context. Sensitivity classification directly operationalizes this identification requirement for AI-specific data types including model artifacts and inference outputs.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_5259", "requirement_id": "ISO/IEC 5259-1 §5 (Data quality concepts)", "fit": "partial", "rationale": "ISO/IEC 5259-1:2024 establishes concepts and terminology for data quality in analytics and machine learning, including how quality requirements attach to data life cycle stages. The classification framework aligns its data-category definitions with this shared vocabulary; the measurable data quality dimensions themselves are specified in ISO/IEC 5259-2.", "normative_force": "voluntary-standard", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "ms_ifc_agents", "requirement_id": "Confidentiality label (public/confidential/read-access list) — Label data by sensitivity (Communicating labels)", "fit": "direct", "rationale": "IFC's label-data step assigns a confidentiality dimension (public, confidential, or an explicit read-access list) to every piece of data an agent ingests — a data sensitivity classification applied to agent data flow.", "normative_force": "best-practice", "source_version": "2026", "reviewed_on": "2026-07-03", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://data/controls/DX-01", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "All data assets processed, generated, or consumed by AI systems carry a valid sensitivity classification from the current enterprise taxonomy version before any exchange, storage, or downstream AI use. The taxonomy must cover AI-specific sub-types including model weights, training corpora, inference logs, evaluation outputs, and synthetic data, and all active AI datasets must return a non-null classification label traceable to that taxonomy version.", "evidence_required": [ "sensitivity_taxonomy_document versioned and effective-dated with at minimum five classification levels and explicit AI-specific sub-types for model weights, training corpora, inference logs, evaluation outputs, and synthetic data", "data_inventory_scan_report showing automated classification coverage of active AI datasets with percentage_classified, scan_tool_identity, scan_date, and false_positive_rate", "pipeline_quarantine_log confirming unlabeled data events were captured with quarantine_reason='missing_sensitivity_label', blocked from downstream processing, and resolved before any AI use", "taxonomy_review_record documenting the most recent quarterly review board outcome with effective_date, change_log entries, and quorum confirmation", "classification_accuracy_sample_report from audit sampling >= 15% of AI datasets with pass/fail verdict per record and overall accuracy percentage" ], "machine_tests": [ "Submit an unlabeled dataset record to the AI data ingestion pipeline → assert pipeline emits a quarantine event with quarantine_reason='missing_sensitivity_label' and blocks downstream processing within 30 seconds", "Query the data catalog API for all active AI datasets → assert >= 99% return a non-null sensitivity_classification field matching an entry in the current taxonomy_version", "Ingest a combined dataset containing Confidential and Public records → assert the output dataset carries Confidential classification with an inheritance_reason field populated referencing the highest-sensitivity source", "Invoke automated classification tooling scan against a test repository containing known PII records → assert scan result includes label, confidence_score, and taxonomy_version fields returned within the declared SLA" ], "human_review": [ "Review the enterprise sensitivity taxonomy document for AI-specific sub-type completeness, version currency, and alignment with current AI system data types in production including any novel artifact types introduced in the past quarter", "Assess the quarterly taxonomy review board records for quorum, documented decisions, and effective-date tracking on all schema changes made since the prior review", "Evaluate automated classification tooling false-positive and false-negative rates from the most recent audit sample to determine whether classification accuracy meets the >= 95% threshold required" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Applying a general-purpose data classification scheme without AI-specific sub-types, leaving model weights, training corpora, and synthetic outputs unclassified or defaulting to Internal regardless of their actual sensitivity", "Treating AI-generated outputs as inheriting the lowest-sensitivity classification of any input rather than the highest, allowing Regulated-data derivatives to circulate as Public", "Running automated classification tooling without quarantine logic for unlabeled data, permitting unclassified records to flow downstream through AI pipelines without triggering any governance event", "Maintaining the sensitivity taxonomy as an informal wiki document rather than a versioned, governed artifact, making it impossible to reconstruct which classification rules applied at a historical point in time for regulatory inquiry", "Excluding model weights and fine-tuning checkpoints from classification scope because they are not traditional records, leaving high-value IP and any embedded PII without proportional protection" ], "update_status": "current", "layer_code": "DX" }, { "id": "DX-02", "layer": "DX", "plane": "control", "name": "Inter-System Data Transfer Authorization", "plain": "All data transfers between AI systems, data stores, and consuming applications must be explicitly authorized by an access control policy that specifies permitted source-destination pairs, authorized data types, and approved transfer mechanisms. Unauthorized or undeclared transfers must be detected and blocked at the network or middleware layer.", "threat": { "tags": [ "unauthorized-data-transfer", "lateral-data-movement", "data-exfiltration", "privilege-escalation" ], "desc": "Without explicit transfer authorization controls, AI systems may pull data from systems outside their intended scope through misconfiguration or through prompt injection attacks that instruct agents to relay sensitive data to unintended destinations. Lateral data movement between AI agents can aggregate information in ways that breach purpose-limitation obligations and create composite privacy violations not visible at the individual transfer level." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 7 Data Security", "title": "Data access control and transfer authorization" }, { "id": "dcam", "section": "Data Control Environment (Component 7)", "title": "Operational data controls across the data lifecycle" }, { "id": "cobit_2019", "section": "DSS05.04", "title": "Manage user and system access authorization" }, { "id": "gdpr", "section": "Art. 5(1)(b)", "title": "Purpose limitation for data processing" } ], "sources": [ { "id": "dama_dmbok", "title": "DAMA-DMBOK 2nd Ed", "authority": "DAMA International", "source_type": "standard", "normative_force": "best-practice", "version": "2 (2nd Edition, Revised 2024)", "published_on": "2017-07-05", "retrieved_on": "2026-07-02", "canonical_url": "https://www.dama.org/cpages/body-of-knowledge", "license": "proprietary", "status": "current", "flagship": false, "source_id": "dama_dmbok", "relationship": "informative_reference", "rationale": "Establishes DAMA-DMBOK 2nd Ed Ch. 7 requirements informing the apeiris://data/controls/DX-02 Inter-System Data Transfer Authorization control.", "reviewed_on": "2026-07-02" }, { "id": "dcam", "title": "DCAM v2.2", "authority": "EDM Council", "source_type": "standard", "normative_force": "industry-framework", "version": "2.2", "published_on": "2022-01-01", "retrieved_on": "2026-07-02", "canonical_url": "https://edmcouncil.org/frameworks/dcam/", "license": "proprietary", "status": "current", "flagship": false, "source_id": "dcam", "relationship": "informative_reference", "rationale": "Establishes DCAM v2.2 Data Control Environment (Component 7) requirements informing the apeiris://data/controls/DX-02 Inter-System Data Transfer Authorization control.", "reviewed_on": "2026-07-02" }, { "id": "gdpr", "title": "GDPR", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2016/679", "published_on": "2016-04-27", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj", "license": "open-access", "status": "current", "flagship": false, "source_id": "gdpr", "relationship": "normative_requirement", "rationale": "Establishes GDPR Art. 5(1)(b) requirements informing the apeiris://data/controls/DX-02 Inter-System Data Transfer Authorization control.", "reviewed_on": "2026-07-02" }, { "id": "cobit_2019", "title": "COBIT 2019", "authority": "ISACA", "source_type": "standard", "normative_force": "industry-framework", "version": "2019", "published_on": "2018-11-12", "retrieved_on": "2026-07-02", "canonical_url": "https://www.isaca.org/resources/cobit", "license": "proprietary", "status": "current", "flagship": false, "source_id": "cobit_2019", "relationship": "informative_reference", "rationale": "Establishes COBIT 2019 DSS05.04 requirements informing the apeiris://data/controls/DX-02 Inter-System Data Transfer Authorization control.", "reviewed_on": "2026-07-02" }, { "id": "nist_pf", "title": "NIST Privacy Framework 1.0", "authority": "National Institute of Standards and Technology", "source_type": "standard", "normative_force": "voluntary-standard", "version": "1.0", "published_on": "2020-01-16", "retrieved_on": "2026-07-02", "canonical_url": "https://www.nist.gov/privacy-framework", "license": "open-access", "status": "current", "flagship": false, "source_id": "nist_pf", "relationship": "implementation_pattern", "rationale": "Establishes NIST Privacy Framework 1.0 CT.PO-P2 requirements informing the apeiris://data/controls/DX-02 Inter-System Data Transfer Authorization control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Implement a data transfer authorization matrix specifying permitted source-destination pairs per data classification level. Enforce via network policy, API gateway, and data pipeline middleware. All transfer attempts emit an authorization event to the audit log with policy reference and outcome.", "steps": [ "Compile a complete inventory of all inter-system data flows involving AI components, sourced from network diagrams, cloud flow logs, and pipeline metadata.", "Assign each flow an authorization status (approved, provisional, blocked) and map it to a specific policy entry with data type and classification scope.", "Implement technical enforcement at API gateway and service mesh layers to block flows absent from the authorization matrix and log the rejection.", "Deploy a change-management process requiring formal approval before any new data flow is provisioned for an AI system.", "Conduct semi-annual data flow mapping reviews using network telemetry to identify new or undeclared transfers relative to the authorization matrix." ], "data_governance_officer": { "summary": "The authorization matrix is owned by Data Governance and represents the policy layer for inter-system data movement. It must align with data classification, purpose-limitation obligations, and third-party agreements.", "actions": [ "Maintain and version-control the inter-system transfer authorization matrix as a governed artifact.", "Approve all new data flows through a formal change process before provisioning.", "Trigger immediate review for any AI agent that begins transferring data types not in its approved manifest." ], "failure_signals": [ "Active data flows identified in network telemetry but not mapped in the authorization matrix.", "New AI system deployed without a transfer authorization review.", "Authorization matrix not reviewed in the prior six months." ] }, "data_engineer": { "summary": "Transfer authorization is enforced at the infrastructure layer. All data pipelines must reference the authorization matrix and emit transfer events to the audit log.", "actions": [ "Integrate API gateway enforcement rules derived from the authorization matrix for all AI data exchange endpoints.", "Instrument pipeline connectors to emit transfer authorization events including policy ID, data type, and classification level.", "Implement circuit-breaker logic to halt pipelines that attempt transfers outside approved scope and alert operations within defined SLA." ], "failure_signals": [ "Transfer events missing policy reference IDs in the audit log.", "Pipeline bypass events detected outside approved maintenance windows.", "New pipeline connector deployed without a corresponding authorization matrix entry." ] }, "grc_auditor": { "summary": "The authorization matrix and its enforcement posture are key audit artifacts for data governance, privacy, and AI governance reviews.", "actions": [ "Request the authorization matrix and cross-reference against observed network flow logs for the audit period.", "Identify flows in network telemetry not represented in the matrix as unauthorized transfers.", "Verify change-management records for all flows provisioned during the audit period." ], "metrics": [ "Authorization matrix coverage of active flows: target 100%.", "Unauthorized transfer incidents per quarter: target zero.", "Time-to-detection for undeclared flows: target <= 24 hours." ], "failure_signals": [ "Undeclared flows identified in network telemetry.", "Transfer authorization matrix not updated to reflect current system topology.", "Change-management records missing for flows provisioned in the audit period." ] }, "legal_counsel": { "summary": "Every inter-system transfer of personal data needs a purpose-compatibility judgment. Counsel defines what purposes are compatible under GDPR Art. 5(1)(b) and reviews transfer authorizations that stretch original collection purposes.", "actions": [ "Define the purpose-compatibility test transfer approvers must apply before authorizing personal data movement into AI systems.", "Review transfer authorizations that repurpose data collected for other ends, documenting the compatibility analysis.", "Escalate transfers that would require a new lawful basis or data-subject notice before they execute." ], "failure_signals": [ "Transfers approved with no recorded purpose-compatibility analysis.", "Personal data collected for operations appearing in AI training stores without legal review.", "Purpose-limitation objections raised by the DPO or regulators after the fact." ] }, "it_operations": { "summary": "Transfer authorization is enforced in infrastructure — network policy, service identity, and gateway rules. Operations makes unauthorized inter-system movement technically hard, not just procedurally forbidden.", "actions": [ "Implement network and service-identity controls so only authorized system pairs can move governed data.", "Instrument transfer paths so every inter-system movement emits an auditable event.", "Alert on transfer attempts that lack a matching authorization record." ], "failure_signals": [ "Data movement paths that bypass gateway instrumentation.", "Transfers observed in telemetry with no authorization record.", "Shadow integrations discovered between systems with no approved transfer route." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Organizations typically have network-level firewall rules but lack an explicit data-type-aware transfer authorization matrix for AI system interactions." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "federated-enterprise", "high-risk-sector" ], "implementers": [ "Data Governance Office", "Platform Engineering", "Network Security" ], "frameworks": [ { "framework": "dama_dmbok", "requirement_id": "Ch. 7", "fit": "direct", "rationale": "DAMA DMBOK 2 Chapter 7 establishes data access control as a core data security discipline requiring explicit authorization of which systems may access data and under what conditions. The inter-system authorization matrix operationalizes this requirement for AI data flows.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "dcam", "requirement_id": "Data Control Environment (Component 7)", "fit": "direct", "rationale": "DCAM v2.2 Component 7 (Data Control Environment) covers the operational controls applied to data across its lifecycle, including controls over data movement between systems. Transfer authorization is such an operational data control; capability 4.2 concerns the technology stack, not entitlement.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "gdpr", "requirement_id": "Art. 5(1)(b)", "fit": "direct", "rationale": "GDPR Article 5(1)(b) limits processing to specified, explicit and legitimate purposes. Inter-system transfer authorization is the control point where purpose compatibility is checked before personal data moves between systems: a transfer into a system with an incompatible purpose is precisely what purpose limitation prohibits.", "normative_force": "binding-law", "source_version": "2016/679", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "cobit_2019", "requirement_id": "DSS05.04", "fit": "partial", "rationale": "COBIT 2019 DSS05.04 requires management of access rights using the principle of least privilege. The inter-system authorization matrix extends this to data flows between AI systems as non-human access subjects requiring equivalent access governance.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_pf", "requirement_id": "CT.PO-P2", "fit": "partial", "rationale": "NIST Privacy Framework CT.PO-P2 requires established policies, processes and procedures for enabling data review, transfer, sharing or disclosure, alteration, and deletion. Inter-system transfer authorization is the transfer/disclosure policy control for AI data flows.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "owasp_ai_exchange", "requirement_id": "segregatedata", "fit": "supporting", "rationale": "DX-02 blocks any inter-system data flow not present in the transfer authorization matrix, segregating data flows to limit exposure between AI systems and stores.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://data/controls/DX-02", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every active data flow between AI systems, data stores, and consuming applications appears as an approved entry in the inter-system transfer authorization matrix with documented data_type, classification_level, and policy_reference. Any transfer attempt not represented in the authorization matrix is detected and blocked at the network or middleware layer before data reaches the destination, with a rejection event emitted to the audit log.", "evidence_required": [ "inter_system_transfer_authorization_matrix listing approved source-destination pairs with data_type, classification_level, authorization_policy_id, approval_date, and approver_identity for each active flow", "api_gateway_rejection_log showing blocked transfer attempts with source, destination, data_type, policy_id, and outcome=DENIED for all flows not represented in the matrix", "network_flow_telemetry_comparison_report cross-referencing observed active flows against the authorization matrix entries with coverage_gap_count and gap_detail", "change_management_records for all flows provisioned in the review period confirming formal approval and approver identity preceded technical provisioning" ], "machine_tests": [ "Send a data transfer request from an AI system to a destination not present in that system's authorization matrix → assert the API gateway returns 403 with error_code=unauthorized_destination and emits a policy_violation event with source and destination fields", "Issue a transfer request with data_type=Restricted where the authorization matrix entry for that source-destination pair permits only data_type=Internal → assert the request is blocked and a classification_scope_violation event is logged", "Query the authorization matrix API for a flow that was explicitly revoked last quarter → assert the entry returns status=revoked and the API gateway enforces the revocation in real time with no data returned" ], "human_review": [ "Review the authorization matrix for completeness against network flow telemetry, identifying any observed flows not represented as approved entries and classifying each gap as authorized-pending or unauthorized", "Assess change management records for all flows provisioned in the review period to verify that formal approval preceded technical provisioning rather than being added retroactively after operations discovered the flow", "Evaluate whether data type scope entries in the authorization matrix reflect current least-privilege requirements or have drifted toward overly broad categories that neutralize the control" ], "blocking_effect": "blocks-runtime-action", "normative_status": "binding-law", "anti_patterns": [ "Maintaining only network-level firewall rules without a data-type-aware authorization matrix, allowing any flow between two permitted IP ranges regardless of the sensitivity of data being transferred", "Using wildcard data_type scopes such as all_data in the authorization matrix for AI system flows because granular scoping is considered too complex, eliminating the security value of the control", "Provisioning new AI data flows without a formal change-management record and adding them to the authorization matrix retroactively after operations discovers the flow in telemetry", "Reviewing the authorization matrix only at annual audit time rather than semi-annually, allowing unauthorized flows to persist for months before detection", "Treating AI agent-to-agent lateral data transfers as internal traffic exempt from the authorization matrix, creating an unmonitored lateral movement path within the AI pipeline layer" ], "update_status": "current", "layer_code": "DX" }, { "id": "DX-03", "layer": "DX", "plane": "control", "name": "API Contract Governance for AI Data Exchange", "plain": "All APIs used to exchange data to or from AI systems must be governed by versioned contracts specifying schema, field semantics, authentication requirements, rate limits, and breaking-change policy. Changes to these APIs must follow a managed lifecycle preventing silent schema drift that could corrupt AI inputs or outputs.", "threat": { "tags": [ "schema-drift", "breaking-change-exposure", "data-contract-violation", "api-misuse" ], "desc": "AI systems consuming unversioned or poorly governed APIs silently degrade when upstream schemas change. A field rename or type change in a training data feed can corrupt model inputs without triggering any alert. Uncontrolled breaking changes create data quality failures that propagate into model behavior, producing incorrect outputs that may not surface until downstream consequences occur in production." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 8", "title": "Data Integration and Interoperability — interface and contract management" }, { "id": "dcam", "section": "Capability 3.3", "title": "Identify the Data" }, { "id": "databricks_unity", "section": "Delta Sharing (open protocol)", "title": "Open data sharing protocol with schema metadata" } ], "sources": [ { "id": "google_dataplex_bigquery_2024", "title": "Google Cloud Dataplex & BigQuery", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://cloud.google.com/dataplex/docs/introduction", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "google_dataplex_bigquery_2024", "relationship": "informative_reference", "rationale": "Establishes Google Cloud Dataplex & BigQuery requirements informing the apeiris://data/controls/DX-03 API Contract Governance for AI Data Exchange control.", "reviewed_on": "2026-07-01" }, { "id": "snowflake_horizon_data_governance_2024", "title": "Snowflake Horizon (Data Governance)", "authority": "Snowflake Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.snowflake.com/en/data-cloud/workloads/data-governance/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "snowflake_horizon", "relationship": "informative_reference", "rationale": "Establishes Snowflake Horizon (Data Governance) requirements informing the apeiris://data/controls/DX-03 API Contract Governance for AI Data Exchange control.", "reviewed_on": "2026-07-01" }, { "id": "databricks_unity_catalog_2024", "title": "Databricks Unity Catalog", "authority": "Databricks, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.databricks.com/en/data-governance/unity-catalog/index.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "databricks_unity_catalog_2024", "relationship": "informative_reference", "rationale": "Establishes Databricks Unity Catalog requirements informing the apeiris://data/controls/DX-03 API Contract Governance for AI Data Exchange control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Register all AI data exchange APIs in a central API catalog with machine-readable OpenAPI or AsyncAPI contracts. Enforce schema validation at API gateway ingress. Implement semantic versioning with a defined breaking-change policy requiring consumer notification and migration windows before deprecation.", "steps": [ "Register all AI data exchange APIs in a central catalog with published OpenAPI 3.x or AsyncAPI contracts including field-level semantic descriptions.", "Enforce schema validation at API gateway so requests and responses non-conformant with the registered contract are rejected with a structured error referencing the contract version.", "Adopt semantic versioning for all AI data APIs and define a breaking-change policy specifying minimum deprecation notice (target 90 days) and migration support requirements.", "Implement automated contract compatibility testing in CI/CD so breaking changes are flagged before deployment and consumers are notified before the change reaches production.", "Maintain a contract registry audit log capturing schema version history, consumer subscriptions, deprecation events, and migration completion status." ], "data_governance_officer": { "summary": "API contract governance ensures that data exchange interfaces are stable, documented, and change-managed. Data Governance sets the schema versioning policy and approves breaking changes.", "actions": [ "Define and publish the enterprise API versioning and breaking-change policy as a governed document.", "Maintain the API contract registry as an authoritative data catalog asset with current version status.", "Approve breaking-change notices and confirm consumer notification records before deprecation proceeds." ], "failure_signals": [ "AI data APIs in production with no registered contract in the catalog.", "Breaking changes deployed without documented consumer notification.", "API catalog coverage below 90% of active AI data exchange endpoints." ] }, "data_engineer": { "summary": "Schema validation and contract testing are implemented in the pipeline and CI/CD layer. Engineers are responsible for contract-first API development and automated compatibility checks.", "actions": [ "Implement schema validation middleware at API ingress for all AI data exchange endpoints rejecting non-conformant payloads.", "Add contract compatibility tests (e.g., Pact, schemathesis) to CI/CD pipelines for all AI data APIs.", "Emit structured schema-validation rejection events to the audit log with contract version, field path, and error details." ], "failure_signals": [ "Schema validation disabled or bypassed on any production AI data API.", "Contract compatibility tests absent from CI/CD for AI data endpoints.", "Schema-validation rejection rate spike without a corresponding incident raised." ] }, "it_operations": { "summary": "API gateway configuration enforces the registered contracts at runtime. Operations manages gateway schema policy updates and monitors rejection rates.", "actions": [ "Configure API gateway schema enforcement policies derived from the registered OpenAPI contracts for all AI data endpoints.", "Monitor schema-validation rejection rate dashboards and alert on anomalous spikes indicating consumer drift.", "Enforce contract version routing so consumers can migrate at their own pace within the declared deprecation window." ], "failure_signals": [ "API gateway schema enforcement policies out of sync with registered contracts.", "No monitoring on schema-validation rejection rates for AI data endpoints.", "Deprecated API versions still receiving production traffic after the declared end-of-life date." ] }, "grc_auditor": { "summary": "API contract governance supports data quality and lineage obligations. Auditors verify contract completeness, versioning compliance, and change-management adherence.", "actions": [ "Verify that all AI data exchange APIs have registered contracts with a current version and effective date.", "Review change-management records for breaking API changes deployed in the audit period.", "Confirm consumer notification records for all deprecations executed in the audit period." ], "metrics": [ "API contract registry coverage: target >= 95% of active AI data exchange endpoints.", "Breaking changes with documented consumer notification: target 100%.", "Average notice period from breaking-change announcement to deprecation: target >= 90 days." ], "failure_signals": [ "AI data APIs in production with unregistered or outdated contracts.", "Breaking changes deployed without documented consumer notification.", "Consumer migration support not provided within the declared deprecation window." ] }, "legal_counsel": { "summary": "Data exchanged through APIs is still governed data. Counsel ensures API contract terms — permitted use, retention, onward sharing — are legally enforceable and consistent with the data's classification and consent basis.", "actions": [ "Review API contract templates so permitted-use and retention clauses match the sensitivity of the data exposed.", "Confirm external API consumers are bound by executed agreements before production access is granted.", "Advise on liability and intellectual-property allocation for AI outputs delivered through APIs." ], "failure_signals": [ "Production APIs serving regulated data to consumers with no executed terms.", "API terms silent on AI training use of the exchanged data.", "Contract versions diverging from deployed schema versions with no review." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most teams have informal versioning practices but lack machine-enforced contract registries for AI data exchange interfaces." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "federated-enterprise", "multi-tenant" ], "implementers": [ "Data Engineering", "Platform Engineering", "API Governance" ], "frameworks": [ { "framework": "dama_dmbok", "requirement_id": "Ch. 8", "fit": "direct", "rationale": "DAMA DMBOK 2 Chapter 8 establishes reference and master data management disciplines that directly encompass data interface and contract management. Versioned API contracts are the primary mechanism for ensuring data consumers receive consistent, well-defined data structures across the enterprise.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "dcam", "requirement_id": "Capability 3.3", "fit": "direct", "rationale": "DCAM v2.2 capability 3.3 (Identify the Data) requires data to be identified and defined with agreed metadata. API contract governance builds on that identification: a contract can only bind schema and semantics that have been identified and defined.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "databricks_unity", "requirement_id": "Delta Sharing (open protocol)", "fit": "direct", "rationale": "Delta Sharing is an open data sharing protocol — stewarded by Databricks and integrated with Unity Catalog, but usable standalone — that exchanges data with schema metadata attached. Its share/recipient model gives cross-boundary exchange a governed contract surface.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "google_dataplex", "requirement_id": "Dataplex Universal Catalog (Data Catalog successor)", "fit": "partial", "rationale": "Google's standalone Data Catalog API was discontinued in the first half of 2026 in favor of Dataplex Universal Catalog (since rebranded Knowledge Catalog) entries and aspects. API-contract metadata on Google Cloud should be registered there rather than against the retired API.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://data/controls/DX-03", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "All APIs used to exchange data to or from AI systems have a registered, versioned contract in the central API catalog with schema validation enforced at the API gateway. No breaking change is deployed without advance consumer notification of at least 90 days, and the contract registry reflects the currently deployed version for every active AI data exchange endpoint.", "evidence_required": [ "api_contract_registry_export listing all active AI data exchange endpoints with registered contract_version, effective_date, consumer_list, and current_status for each endpoint", "api_gateway_schema_validation_rejection_log confirming non-conformant payloads are rejected at ingress with structured error bodies containing contract_version and schema_field_path of the violation", "breaking_change_notice_records for all API version changes in the review period including notification_date, affected_consumer_list, planned_deprecation_date, and migration_support_status", "ci_cd_contract_compatibility_test_results confirming compatibility checks ran and passed before each deployment touching an AI data exchange API" ], "machine_tests": [ "Submit a request payload to an AI data exchange API with a required field missing per the registered OpenAPI contract → assert the gateway returns 400 with an error body containing contract_version and the schema_path of the missing field", "Attempt to call an API version past its declared deprecation end-of-life date → assert the gateway returns 410 Gone with a structured error referencing the superseding version and migration_guide_url", "Introduce a simulated breaking schema change (rename a required field) in a CI/CD test deployment without updating the registered contract → assert the contract compatibility test fails and the deployment is blocked before reaching staging" ], "human_review": [ "Review the API contract registry for completeness by cross-referencing against the AI system inventory to identify any active AI data exchange endpoints missing a registered and current contract", "Assess breaking-change notice records for the review period to verify that consumers received notification with at least 90 days of lead time and that migration support was provided before deprecation executed", "Evaluate the contract versioning policy document for clarity of breaking versus non-breaking change definitions and whether teams are applying these definitions consistently across AI data API releases" ], "blocking_effect": "blocks-deployment", "normative_status": "industry-framework", "anti_patterns": [ "Publishing OpenAPI specifications as static documentation files rather than machine-enforced gateway contracts, so schema validation runs only in tests and not at production runtime where AI consumers may send non-conformant payloads", "Classifying field type changes (string to integer) or required field renames as non-breaking changes and deploying them without consumer notification, silently corrupting AI data consumers", "Maintaining a single latest API version label without semantic versioning, making it impossible for AI data consumers to pin to a stable version or detect when a breaking change has been deployed", "Requiring API contract registration only for external-facing APIs while exempting internal AI system-to-system data exchange APIs, leaving the highest-throughput interfaces ungoverned", "Bypassing contract compatibility tests in CI/CD pipelines for hotfix deployments, creating a path through which breaking schema changes reach production without consumer notification" ], "update_status": "current", "layer_code": "DX" }, { "id": "DX-04", "layer": "DX", "plane": "control", "name": "Cross-Border Data Transfer Controls", "plain": "AI data transferred across national borders must have a documented lawful basis, appropriate safeguards, and technical controls that restrict data routing to approved geographic zones. High-risk AI systems processing personal data must additionally document Transfer Impact Assessments prior to deployment.", "threat": { "tags": [ "illegal-cross-border-transfer", "data-sovereignty-violation", "regulatory-non-compliance", "inadequate-safeguard" ], "desc": "Cloud-native AI pipelines frequently route data through multiple geographic regions to exploit available compute, creating cross-border transfers that trigger GDPR Chapter V, national data localization requirements, and sector-specific restrictions. Without explicit transfer controls, AI systems may inadvertently transfer personal data to jurisdictions without adequate protection, exposing the organization to enforcement action and requiring costly remediation of deployed systems." }, "standard": [ { "id": "gdpr", "section": "Art. 5 + Ch. V (Arts. 44-49)", "title": "Cross-border transfer lawful basis and safeguards" }, { "id": "iso_27701", "section": "§7.5.3", "title": "Cross-border personal information transfers" }, { "id": "nist_pf", "section": "GV.PO-P5", "title": "Legal, regulatory and contractual requirements regarding privacy" } ], "sources": [ { "id": "microsoft_purview_data_governance_2024", "title": "Microsoft Purview (Data Governance)", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/purview/purview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_purview_data_governance_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Purview (Data Governance) requirements informing the apeiris://data/controls/DX-04 Cross-Border Data Transfer Controls control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Maintain a transfer impact register documenting all cross-border AI data flows with lawful basis (adequacy decision, SCCs, BCRs, or exemption), data types, destination countries, and safeguards in place. Enforce geographic routing restrictions at the cloud infrastructure layer using region constraints and data residency policies.", "steps": [ "Map all AI data flows that cross national borders using cloud provider region metadata and network egress logs.", "For each cross-border flow, document the lawful basis and applicable safeguards in a transfer impact register maintained by Data Governance.", "Execute Transfer Impact Assessments (TIAs) for all flows to non-adequate third countries processing personal data, with legal sign-off before system deployment.", "Implement cloud-provider data residency controls (region lock, VPC constraints) to enforce approved geographic routing for AI workloads.", "Establish a legal review trigger for any new AI system deployment that would process personal data in or from EU-regulated jurisdictions or jurisdictions with data localization requirements." ], "legal_counsel": { "summary": "Cross-border transfer legality is a legal function. Counsel must maintain current awareness of adequacy decisions, SCCs, BCRs, and national data localization requirements affecting AI deployments.", "actions": [ "Maintain a jurisdictional transfer matrix mapping each destination country to its transfer mechanism and current adequacy status.", "Review and approve Transfer Impact Assessments for all AI data flows to non-adequate third countries.", "Monitor regulatory changes to adequacy decisions and trigger emergency reviews when adequacy status changes for a destination country." ], "failure_signals": [ "Cross-border flows to non-adequate countries without executed SCCs or BCRs on file.", "Transfer impact register not updated after adequacy status change for a destination country.", "New AI system processing EU personal data deployed without legal sign-off on the transfer mechanism." ] }, "data_governance_officer": { "summary": "Data Governance maintains the transfer impact register and coordinates with Legal and Engineering to operationalize geographic routing restrictions.", "actions": [ "Own and maintain the cross-border transfer impact register as a living governance document with current entry for each active flow.", "Ensure all AI data flow owners register cross-border transfers in the register before system deployment.", "Trigger bi-annual reviews of the transfer register to capture new or changed flows." ], "failure_signals": [ "Cross-border flows discovered in cloud egress telemetry not registered in the transfer impact register.", "Transfer impact register entries missing lawful basis documentation.", "Bi-annual register review overdue." ] }, "it_operations": { "summary": "Technical enforcement of geographic routing is an operations responsibility. Cloud-layer region constraints and data residency policies must be configured and monitored.", "actions": [ "Configure cloud region constraints and data residency policies for all AI workloads processing regulated personal data.", "Monitor cloud egress telemetry for data leaving approved geographic regions and alert on anomalies.", "Alert on any AI compute workload scheduled in a region outside the approved transfer list within one business day." ], "failure_signals": [ "AI workloads running in unapproved geographic regions.", "Cloud egress telemetry gaps preventing cross-border flow detection.", "Data residency policies not enforced on new AI infrastructure deployments." ] }, "grc_auditor": { "summary": "Cross-border transfer compliance is a high-priority audit area under GDPR and national AI regulations. Auditors verify completeness of the transfer register and adequacy of safeguards.", "actions": [ "Cross-reference the transfer impact register against cloud egress logs to identify undeclared cross-border flows.", "Verify executed SCCs, BCRs, or adequacy reliance documentation for all flows to non-adequate countries.", "Review TIA quality and legal sign-off for high-risk AI systems transferring personal data to third countries." ], "metrics": [ "Transfer register completeness: target 100% of detected cross-border flows.", "Flows with documented lawful basis: target 100%.", "TIAs completed for non-adequate country flows: target 100%." ], "failure_signals": [ "Undeclared cross-border flows identified in cloud egress telemetry.", "Any flow to a non-adequate country without documented and signed SCCs or BCRs.", "TIA not completed for high-risk AI systems within six months of deployment." ] }, "data_engineer": { "summary": "Cross-border restrictions are enforced in pipeline configuration. Engineers make every data flow jurisdiction-aware — destination regions declared, pinned and validated before data moves.", "actions": [ "Declare destination jurisdiction as required metadata on every AI data flow and validate it against the approved transfer matrix at deploy time.", "Pin storage and compute regions in pipeline and infrastructure-as-code configuration rather than relying on provider defaults.", "Fail transfers to destinations not present in the approved transfer list and surface the failure to the flow owner." ], "failure_signals": [ "Pipelines writing to regions outside the approved transfer matrix.", "Data flows with no declared destination jurisdiction metadata.", "Hardcoded endpoints bypassing region-pinning configuration." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Cross-border AI data flows are rarely tracked at the granularity required by GDPR Chapter V. Most organizations rely on contractual boilerplate without technical enforcement of geographic routing." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "eu-high-risk-ai", "high-risk-sector", "federated-enterprise" ], "implementers": [ "Legal / Privacy Counsel", "Data Governance Office", "Cloud Infrastructure" ], "frameworks": [ { "framework": "gdpr", "requirement_id": "Art. 44-49", "fit": "direct", "rationale": "GDPR Chapter V Articles 44-49 establish the mandatory legal framework for all transfers of personal data to third countries. These provisions directly require lawful basis documentation and safeguards that this control operationalizes for AI data pipelines.", "normative_force": "binding-law", "source_version": "2016/679", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "iso_27701", "requirement_id": "§7.5.3", "fit": "direct", "rationale": "ISO/IEC 27701 Section 7.5.3 requires organizations to apply controls for cross-border personal information transfers consistent with applicable legal requirements. The transfer impact register and geographic routing controls directly satisfy this requirement.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_pf", "requirement_id": "GV.PO-P5", "fit": "partial", "rationale": "NIST Privacy Framework GV.PO-P5 requires legal, regulatory, and contractual requirements regarding privacy to be understood and managed. Cross-border transfer controls operationalize exactly this requirement class — adequacy decisions, SCCs and localization mandates — for AI data flows. GV.PO-P3 concerns workforce roles and responsibilities.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "microsoft_purview", "requirement_id": "Microsoft 365 Advanced Data Residency", "fit": "partial", "rationale": "Data residency commitments on Microsoft platforms are delivered by Microsoft 365 Advanced Data Residency and Azure region controls rather than by Purview itself. Teams implementing DX-04 on Microsoft estates combine those residency features with Purview visibility over where regulated data lives.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "owasp_ai_exchange", "requirement_id": "alloweddata", "fit": "supporting", "rationale": "DX-04 requires a documented lawful transfer basis and TIA before any cross-border AI data flow, restricting AI data movement to legally permitted data.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://data/controls/DX-04", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every cross-border AI data flow is documented in the transfer impact register with a documented lawful transfer basis, appropriate safeguards, and — for flows to non-adequate countries processing personal data — a completed and legally signed Transfer Impact Assessment executed before system deployment. Technical geographic routing controls enforced at the cloud infrastructure layer prevent AI workloads from routing data through regions not in the approved transfer list.", "evidence_required": [ "transfer_impact_register listing all cross-border AI data flows with destination_country, lawful_basis_type, safeguard_document_reference, legal_signoff_date, and adequacy_status for each entry", "transfer_impact_assessment_documents for all flows to non-adequate third countries with legal_counsel_signature and sign-off date confirming pre-deployment approval was obtained", "cloud_region_policy_configuration_export confirming data residency constraints and region locks are enforced at the infrastructure layer for all AI workloads processing regulated personal data", "cloud_egress_telemetry_comparison_report cross-referencing observed transfer destinations against the register entries to identify undeclared cross-border flows and coverage gap count" ], "machine_tests": [ "Attempt to provision an AI training workload to a cloud region not on the approved geographic routing list via the infrastructure API → assert the provisioning request is rejected with error referencing the data_residency_policy and the approved_region_list", "Query cloud egress telemetry for the prior 30 days and cross-reference against the transfer impact register → assert zero flows are detected to countries classified as non-adequate without a completed TIA entry in the register", "Update a destination country's adequacy status to inadequate in the compliance system → assert all AI flows to that country are flagged for immediate legal review and an alert is delivered to the compliance team within 4 hours" ], "human_review": [ "Review the transfer impact register for completeness by cross-referencing against cloud egress telemetry and pipeline destination configurations to identify any undeclared cross-border flows not captured in the register", "Assess the quality of TIAs for high-risk AI systems by reviewing the adequacy analysis, threat assessment, and supplementary measures documentation for legal sufficiency and currency relative to the current regulatory environment", "Verify that adequacy decision monitoring is active and that the register has been updated following any regulatory changes to adequacy status for destination countries used by AI workloads" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Relying on executed SCCs without completing a Transfer Impact Assessment for flows to high-surveillance jurisdictions, treating contract execution as a substitute for assessing whether the legal environment of the destination provides adequate protection", "Enforcing geographic routing constraints at the cloud account level rather than the AI workload level, allowing a single permissive account-level policy to authorize cross-border flows for workloads that should be regionally locked", "Treating inference inputs to commercial LLM API providers as outside cross-border transfer scope because the data 'only passes through' provider infrastructure, when personal data in prompts may be processed and retained in non-adequate jurisdictions", "Maintaining the transfer impact register as a point-in-time document rather than a living artifact, so that new AI data flows or changes in adequacy status are not captured until the next scheduled review", "Completing TIAs only at initial system deployment without reassessing when AI system data sources, destination endpoints, or personal data processing volumes materially change" ], "update_status": "current", "layer_code": "DX" }, { "id": "DX-05", "layer": "DX", "plane": "control", "name": "Third-Party Data Sharing Agreements and Controls", "plain": "All sharing of AI data with third parties — including model vendors, data brokers, annotation services, research partners, and sub-processors — must be governed by executed data sharing agreements or Data Processing Agreements specifying permitted use, retention limits, onward transfer restrictions, and security requirements. Technical controls must enforce sharing scope at the data layer.", "threat": { "tags": [ "unauthorized-third-party-sharing", "vendor-data-leakage", "sub-processor-non-compliance", "purpose-creep" ], "desc": "AI systems routinely share training data, inference inputs, and evaluation outputs with model vendors, annotation services, and evaluation platforms. Without executed agreements and technical scope enforcement, third parties may retain, re-use, or onward-transfer data in ways inconsistent with the original collection purpose. Vendor data leakage from AI sub-processors is a significant and underestimated risk vector for both personal data and proprietary business information." }, "standard": [ { "id": "gdpr", "section": "Art. 28", "title": "Processor obligations and DPA requirements" }, { "id": "iso_27701", "section": "§7.5", "title": "Third-party data sharing and sub-processor controls" }, { "id": "nist_pf", "section": "CT.PO-P2", "title": "Data processing agreements and third-party sharing" }, { "id": "anthropic_privacy", "section": "Privacy Policy", "title": "Vendor privacy disclosures and data processing terms" } ], "sources": [ { "id": "anthropic_privacy_policy_2024", "title": "Anthropic Data Handling & Privacy Policy", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-09-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/privacy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_privacy_policy_2024", "relationship": "informative_reference", "rationale": "Establishes Anthropic Data Handling & Privacy Policy requirements informing the apeiris://data/controls/DX-05 Third-Party Data Sharing Agreements and Controls control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Maintain a third-party data sharing register mapping each sharing relationship to its executed agreement, permitted data types, purpose, and retention limits. Implement technical controls — field-level redaction, differential privacy, tokenization — to restrict shared data to agreed scope. Conduct annual due diligence reviews of high-risk third-party processors.", "steps": [ "Compile a third-party data sharing register listing all entities that receive AI data, including model API providers, annotation vendors, evaluation platforms, and research partners.", "Ensure executed agreements (DPAs for personal data, NDAs or DSAs for proprietary data) are in place for every sharing relationship before any data is shared.", "Implement field-level access controls, tokenization, or data masking to enforce the agreed scope of each sharing relationship at the technical layer.", "Conduct annual due diligence reviews of high-risk third-party processors including security questionnaires, sub-processor disclosure reviews, and audit right exercises.", "Establish a vendor offboarding procedure ensuring data return or verified deletion certificates upon relationship termination." ], "legal_counsel": { "summary": "Third-party sharing agreements are legal instruments requiring counsel review and execution. Legal is responsible for DPA adequacy, permitted-use clauses, and onward-transfer restrictions.", "actions": [ "Review and negotiate DPAs with all third parties receiving personal data as part of AI workloads.", "Maintain a signed agreement inventory cross-referenced against the sharing register.", "Review third-party sub-processor disclosures and update agreements when sub-processors change in a material way." ], "failure_signals": [ "Third parties receiving personal data without an executed DPA on file.", "Agreements missing permitted-use scope or onward-transfer restriction clauses.", "Sub-processor change notifications not reviewed within 30 days of receipt." ] }, "data_governance_officer": { "summary": "Data Governance maintains the sharing register and ensures technical scope controls align with agreement terms. All new sharing relationships must be registered before data sharing begins.", "actions": [ "Maintain the third-party data sharing register with current agreement status, data type scope, and retention limits.", "Validate that technical access controls enforce the scope committed in each executed agreement.", "Initiate due diligence reviews for high-risk processors and track findings to closure." ], "failure_signals": [ "Active sharing relationships not reflected in the register.", "Technical controls permitting data access beyond the agreed scope.", "Due diligence reviews overdue for high-risk processors." ] }, "grc_auditor": { "summary": "Third-party data sharing is a key audit scope for GDPR, AI Act, and sector compliance. Auditors verify agreement completeness, scope enforcement, and due diligence cadence.", "actions": [ "Cross-reference the sharing register against API call logs and data egress records to identify undocumented sharing relationships.", "Sample executed agreements to verify they contain required DPA clauses, purpose limitations, and data deletion obligations.", "Confirm annual due diligence reviews were completed for all high-risk processors in the audit period." ], "metrics": [ "Third-party sharing relationships with executed agreements: target 100%.", "High-risk processors with completed annual due diligence: target 100%.", "Sharing relationships registered as a fraction of detected active relationships: target >= 98%." ], "failure_signals": [ "Undocumented sharing relationships identified in egress logs.", "Any high-risk processor without a completed annual due diligence review.", "DPA terms missing key clauses (purpose limitation, data deletion, sub-processor disclosure)." ] }, "it_operations": { "summary": "Third-party sharing agreements bind operations to concrete mechanics: approved channels, egress controls and offboarding. Operations ensures data leaves only via contracted routes and stops when contracts end.", "actions": [ "Restrict external data egress to approved, contract-mapped channels with per-partner identities.", "Automate partner offboarding so access and feeds terminate on contract end dates.", "Monitor egress volumes per partner and alert on flows outside agreed scope." ], "failure_signals": [ "Active data feeds to partners whose agreements have lapsed.", "Egress to third parties over unapproved channels.", "Partner access credentials outliving the relationship." ] }, "data_engineer": { "summary": "Third-party sharing runs through engineered channels. Engineers build partner feeds on approved connectors with contract scope encoded in configuration, so what is shared is exactly what was agreed.", "actions": [ "Implement partner data feeds only via approved sharing mechanisms with per-partner identities and scoped datasets.", "Encode the contractual field-level scope in feed configuration so out-of-scope columns cannot be shared silently.", "Tag shared datasets with agreement identifiers so every external flow traces to an executed contract." ], "failure_signals": [ "Ad-hoc exports to partners outside the engineered sharing channels.", "Feed configurations exposing columns beyond the contracted scope.", "Shared datasets with no agreement identifier attached." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Organizations frequently focus DPA compliance on traditional data processors and miss AI-specific vendors such as model API providers, fine-tuning services, and evaluation platforms." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "cloud-native" ], "implementers": [ "Legal / Privacy Counsel", "Data Governance Office", "Vendor Management" ], "frameworks": [ { "framework": "gdpr", "requirement_id": "Art. 28", "fit": "direct", "rationale": "GDPR Article 28 mandates that controllers only use processors providing sufficient guarantees via binding contracts (DPAs). This requirement directly applies to all AI model vendors, annotation services, and evaluation platforms receiving personal data, making DPA execution a binding legal obligation.", "normative_force": "binding-law", "source_version": "2016/679", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "iso_27701", "requirement_id": "§7.5", "fit": "direct", "rationale": "ISO/IEC 27701 Section 7.5 provides detailed requirements for managing third-party relationships involving personal information processing including contractual obligations, security requirements, and sub-processor management. This maps directly to the AI data sharing agreement obligations.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_pf", "requirement_id": "CT.PO-P2", "fit": "direct", "rationale": "NIST Privacy Framework CT.PO-P2 requires that data processing agreements are established with third parties consistent with organizational privacy policies. The third-party data sharing register and agreement enforcement operationalize this subcategory for AI workloads.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "anthropic_privacy", "requirement_id": "Privacy Policy", "fit": "direct", "rationale": "Anthropic's Privacy Policy discloses how personal data submitted to its services is used, retained and shared, and its commercial terms offer a Data Processing Addendum (including a zero-data-retention option) for enterprise customers. It is a working example of the vendor disclosures and contract terms DX-05 requires organizations to obtain from AI providers.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "eu_ai_act", "requirement_id": "Art. 23-24", "fit": "partial", "rationale": "EU AI Act Articles 23 and 24 place obligations on importers and distributors of high-risk AI systems to verify conformity documentation before making systems available. (The draft-text Articles 28-29 were renumbered in the adopted Regulation 2024/1689.) Third-party data sharing agreements are the contractual mechanism through which data-specific documentation and compliance duties flow along that supply chain.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "owasp_ai_exchange", "requirement_id": "segregatedata", "fit": "supporting", "rationale": "DX-05 technically enforces per-third-party scope so no external recipient can access data types or volumes beyond their executed agreement, limiting data exposure.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://data/controls/DX-05", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every third-party entity receiving AI data is listed in the third-party data sharing register with an executed agreement specifying permitted data types, retention limits, and onward transfer restrictions. Technical controls at the data layer enforce the agreed scope so that no third party can access data types or volumes beyond those specified in their executed agreement.", "evidence_required": [ "third_party_data_sharing_register listing all entities receiving AI data with agreement_type, agreement_execution_date, permitted_data_types, retention_limit, and sub_processor_disclosure_status for each relationship", "executed_dpa_inventory confirming a signed Data Processing Agreement with purpose_limitation, data_deletion_obligation, sub_processor_disclosure, audit_right, and breach_notification_timeline clauses exists for every personal-data sharing relationship", "technical_scope_enforcement_audit confirming field-level access controls, tokenization, or masking rules restrict each third party to their agreed data type and volume scope", "annual_due_diligence_review_records for all high-risk processors including security_questionnaire_results, sub_processor_change_notification_review dates, and open_findings_remediation_status", "vendor_offboarding_records for terminated relationships with data_return_certificate or verified_deletion_certificate and date of confirmed deletion" ], "machine_tests": [ "Make an authenticated API call to the third-party data sharing endpoint using Vendor A credentials requesting a field outside Vendor A's permitted data type scope → assert the response is 403 Forbidden with error_code=scope_exceeded and no restricted fields are returned in the payload", "Query the sharing register API for all active sharing relationships and cross-reference against API call logs from the prior 30 days → assert every entity making data calls appears in the register with an executed and current agreement", "Submit a bulk data request from a vendor that exceeds the volume ceiling specified in their agreement → assert the request is throttled with error referencing agreement_id and the permitted_volume_ceiling" ], "human_review": [ "Review executed DPAs for all third parties receiving personal data through AI workloads to verify each agreement contains required clauses: purpose limitation, data deletion obligation, sub-processor disclosure, audit right, and breach notification timeline", "Assess the technical scope enforcement configuration for each high-risk sharing relationship to verify field-level controls match the agreed data type scope and have not drifted since the agreement was executed", "Evaluate due diligence review records for all high-risk processors to confirm security questionnaire responses were reviewed against minimum standards and any open findings have a tracked remediation plan" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Treating commercial LLM API providers as software vendors rather than data processors and omitting DPA execution, when inference inputs containing personal data make them GDPR Article 28 processors requiring binding agreements", "Executing a single broad master DPA with a vendor covering all current and future use cases rather than data-type-specific agreements, obscuring whether specific AI use cases fall within the permitted purpose scope", "Limiting technical scope enforcement to network-level IP allowlisting rather than field-level access control, allowing a credentialed vendor to request data types beyond their agreement scope", "Failing to review sub-processor change disclosures when notified by a vendor, permitting AI data to flow to new sub-processors without assessment of their compliance posture or security controls", "Treating the sharing register as an initial intake form rather than a living control document, so that relationship changes such as a vendor expanding data use for model training are not captured until audit discovery" ], "update_status": "current", "layer_code": "DX" }, { "id": "DX-06", "layer": "DX", "plane": "data", "name": "Data Exchange Audit Trail", "plain": "Every data movement event involving AI systems must be logged with source, destination, data type, classification level, volume, timestamp, initiating actor, and legal basis. Audit logs must be tamper-evident, retained for the required regulatory period, and available for real-time query by authorized governance functions.", "threat": { "tags": [ "unlogged-data-movement", "audit-gap", "data-lineage-loss", "tamper-after-incident" ], "desc": "Without comprehensive data movement logging, organizations cannot reconstruct what data flowed where after an incident. AI systems that process personal data without audit trails cannot demonstrate GDPR lawfulness of processing on request. Gaps in the audit trail also prevent detection of unauthorized lateral data movement, making data exfiltration by malicious insiders or compromised AI agents invisible until downstream consequences surface." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 7 Data Security", "title": "Data activity monitoring and audit logging" }, { "id": "cobit_2019", "section": "MEA03", "title": "Compliance monitoring and reporting" }, { "id": "gdpr", "section": "Art. 5(2) + Art. 30", "title": "Accountability and records of processing activities" }, { "id": "iso_27701", "section": "§8.2.6", "title": "Records related to processing of PII" } ], "sources": [ { "id": "microsoft_purview_data_governance_2024", "title": "Microsoft Purview (Data Governance)", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/purview/purview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_purview_data_governance_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Purview (Data Governance) requirements informing the apeiris://data/controls/DX-06 Data Exchange Audit Trail control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Centralize all data exchange events in an append-only audit log with WORM-configured storage and structured event schema capturing source, destination, data type, classification, volume, actor, and legal basis. Index the log for real-time query and alert on anomalous patterns using SIEM rules.", "steps": [ "Define a standard data exchange event schema covering: event_id, timestamp, source_system, destination_system, data_type, classification_level, record_count, bytes_transferred, initiating_actor, authorization_policy_id, and legal_basis.", "Configure all AI data pipeline connectors, API gateways, and ETL tools to emit conformant exchange events to the central log before and after each transfer.", "Implement tamper-evident log storage using WORM-configured object storage or an immutable cloud logging service with access controls preventing deletion.", "Configure SIEM alerting rules on anomalous exchange patterns including off-hours bulk transfers, transfers to new destinations, and transfers exceeding volume baselines.", "Establish a log retention schedule aligned with the longest applicable regulatory retention requirement for each data type, targeting at minimum six years for EU-regulated personal data." ], "data_governance_officer": { "summary": "The data exchange audit trail is the operational record for data governance accountability. Data Governance defines the event schema and retention policy.", "actions": [ "Define and publish the standard exchange event schema as a governed artifact with version history.", "Set and document the retention schedule per data type and regulatory requirement.", "Use the audit trail as the primary evidence artifact for responding to regulatory inquiries and data subject access requests." ], "failure_signals": [ "Exchange events missing required fields such as legal basis or classification level.", "Retention periods shorter than the applicable regulatory requirement for any data type.", "Audit trail unavailable or incomplete when responding to a regulatory inquiry." ] }, "data_engineer": { "summary": "Log emission and schema conformance is an engineering responsibility. All data movement code must emit events to the central log before completing a transfer.", "actions": [ "Instrument all data pipeline connectors and AI data APIs to emit conformant exchange events including pre-transfer events for attempted transfers.", "Add log coverage validation to CI/CD so new pipeline components must include exchange event instrumentation before deployment.", "Validate schema conformance of emitted events in a staging environment before promoting to production." ], "failure_signals": [ "Pipeline components emitting non-conformant or partial exchange events.", "Exchange events absent for known data movements identified in network telemetry.", "Pre-transfer logging missing from high-volume AI pipeline connectors." ] }, "it_operations": { "summary": "Operations manages the central log infrastructure and SIEM integration. Log availability, integrity, and alerting effectiveness are operational responsibilities.", "actions": [ "Configure WORM or immutable storage for the central exchange log with access controls preventing log deletion.", "Integrate the exchange event stream into SIEM with anomaly detection rules active and tested quarterly.", "Monitor log pipeline health and alert on ingestion gaps or lag exceeding defined thresholds." ], "failure_signals": [ "Log storage not configured with WORM or immutability controls.", "SIEM anomaly detection rules inactive or not tested in the prior quarter.", "Log pipeline lag exceeding 15 minutes for real-time monitoring purposes." ] }, "grc_auditor": { "summary": "The audit trail is the primary artifact for demonstrating GDPR accountability, AI Act compliance, and data governance effectiveness. Auditors verify completeness, tamper-evidence, and retention.", "actions": [ "Verify audit trail completeness by cross-referencing exchange events against known data flows from network telemetry and pipeline configuration.", "Confirm log storage is tamper-evident and retention periods meet the longest applicable regulatory requirement.", "Test query capability by requesting a sample of exchange events for a specific time window and dataset as a completeness spot check." ], "metrics": [ "Exchange event completeness: target >= 99% of known data movement events logged.", "Log storage with tamper-evident controls verified: target 100%.", "Log retention compliance with longest applicable requirement: target 100%." ], "failure_signals": [ "Known data movements not represented in the exchange audit log.", "Log storage without tamper-evident controls.", "Log retention shorter than the applicable regulatory minimum for any data type." ] }, "legal_counsel": { "summary": "Exchange audit trails are the accountability record GDPR Art. 5(2) presumes. Counsel defines what the trail must capture to survive regulatory inquiry and litigation discovery, and how long it must be retained.", "actions": [ "Specify the minimum audit-trail fields needed to demonstrate lawful transfer — parties, purpose, legal basis, timestamp.", "Set audit-log retention periods that satisfy accountability duties without violating storage-limitation constraints.", "Use the trail when responding to data-subject access requests and regulator inquiries about AI data flows." ], "failure_signals": [ "Regulator or DSAR responses assembled manually because the trail cannot answer who received what data.", "Audit logs deleted before applicable limitation periods expire.", "Exchange events involving personal data with no recorded legal basis." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Most organizations have application-level logging but lack structured, schema-conformant data exchange event logs with legal basis fields required for GDPR Article 30 accountability." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "Data Engineering", "IT Operations / Security", "Data Governance Office" ], "frameworks": [ { "framework": "gdpr", "requirement_id": "Art. 5(2) + Art. 30", "fit": "direct", "rationale": "GDPR Article 5(2) establishes the accountability principle requiring controllers to demonstrate compliance with the data protection principles. Article 30 requires records of processing activities. A structured, tamper-evident data exchange audit trail is the primary technical mechanism for satisfying both requirements for AI data processing.", "normative_force": "binding-law", "source_version": "2016/679", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 7", "fit": "direct", "rationale": "DAMA DMBOK 2 Chapter 7 identifies data activity monitoring and audit logging as core data security disciplines. The data exchange audit trail is the applied implementation of these disciplines for AI data movement events with AI-specific fields for legal basis and classification level.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "iso_27701", "requirement_id": "§8.2.6", "fit": "direct", "rationale": "ISO/IEC 27701:2019 §8.2.6 requires PII processors to determine and maintain the records necessary to demonstrate compliance with their obligations — for data exchange, a complete trail of what personal data was transferred, when, to whom, and under what instruction. §8.2.3 concerns marketing and advertising use, not records.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_2019", "requirement_id": "MEA03", "fit": "partial", "rationale": "COBIT 2019 MEA03 establishes monitoring and evaluation of compliance with external requirements. A comprehensive data exchange audit trail provides the evidence base for demonstrating regulatory compliance in data governance monitoring processes and responding to regulator requests.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "microsoft_purview", "requirement_id": "Purview Audit (unified audit log)", "fit": "partial", "rationale": "Microsoft Purview Audit — a distinct Purview solution from the data governance catalog — captures a unified audit log across Microsoft 365 services. It records data-access and sharing events a data exchange audit trail can draw on for Microsoft-hosted AI data flows.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://data/controls/DX-06", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every data movement event involving an AI system is captured in an append-only, tamper-evident audit log with a conformant event schema including event_id, timestamp, source_system, destination_system, data_type, classification_level, record_count, initiating_actor, authorization_policy_id, and legal_basis fields all populated. Logs are retained for at minimum the longest applicable regulatory period per data type and are queryable by authorized governance functions within the defined query SLA.", "evidence_required": [ "data_exchange_event_log sample showing conformant event records with event_id, timestamp, source_system, destination_system, data_type, classification_level, record_count, bytes_transferred, initiating_actor, authorization_policy_id, and legal_basis all populated for data movement events in the review period", "worm_storage_configuration_certificate or immutable_log_attestation confirming the log storage backend has tamper-evident controls and deletion prevention configured", "log_completeness_comparison_report cross-referencing exchange events against network flow telemetry for the review period with completeness_rate and gap_detail", "log_retention_schedule_document specifying retention_period_by_data_type aligned to the longest applicable regulatory requirement with evidence that live logs meet the schedule", "siem_alert_rule_configuration_export showing active anomaly detection rules and the most recent quarterly rule test results with pass/fail status" ], "machine_tests": [ "Initiate a known data transfer through the AI data pipeline and query the audit log within 60 seconds → assert a conformant exchange event exists with all required schema fields populated including authorization_policy_id and legal_basis", "Attempt to delete an exchange log entry via the storage management API → assert the deletion is rejected with an immutability error and a deletion_attempt_alert event is fired to the SIEM", "Trigger a simulated off-hours bulk transfer exceeding the established volume baseline → assert a SIEM anomaly detection alert fires referencing the event_ids of the anomalous transfers within the defined alert SLA", "Query the audit log for exchange events from a 90-day-old time window → assert events are retrievable within the defined query SLA and schema conformance is maintained on all historical records returned" ], "human_review": [ "Review a random sample of exchange log entries across multiple data types to verify that legal_basis and classification_level fields are populated accurately and consistently with the governance policy applicable to each flow", "Assess the SIEM anomaly detection rule set for coverage of key risk scenarios including off-hours bulk transfers, transfers to new or undeclared destinations, and classification-level mismatches between the policy and the actual data transferred", "Verify that the log retention schedule is current and that the storage system's configured retention period matches the documented schedule for each data type classification, including any recent additions from new AI system deployments" ], "blocking_effect": "advisory", "normative_status": "binding-law", "anti_patterns": [ "Logging data exchange events only at the application layer without correlation to network-layer telemetry, creating an audit trail that misses transfers initiated outside the application such as by AI agents with direct database access", "Emitting exchange events only after a transfer completes rather than capturing a pre-transfer initiation event, so that transfers interrupted by an attacker leave no evidence of the attempted movement in the log", "Omitting the legal_basis field from exchange events on the grounds it is implicit from the system context, making it impossible to demonstrate GDPR Article 30 accountability for each individual processing event on regulatory request", "Storing audit logs in the same storage systems as operational data with the same access controls, allowing an administrator who can delete operational data to also delete the audit trail of that deletion", "Configuring log retention periods based on operational storage cost rather than regulatory requirement, silently falling below the minimum retention period for EU personal data without triggering any compliance alert" ], "update_status": "current", "layer_code": "DX" }, { "id": "DX-07", "layer": "DX", "plane": "control", "name": "Data Exchange Security Controls", "plain": "All data exchanged by AI systems must be encrypted in transit using current cryptographic standards, authenticated at both ends of the exchange, verified for integrity on receipt, and protected against replay. High-sensitivity exchanges must additionally support non-repudiation through digital signatures using HSM-backed keys.", "threat": { "tags": [ "data-interception", "man-in-the-middle", "integrity-tampering", "replay-attack" ], "desc": "AI systems processing unencrypted or weakly authenticated data exchanges are vulnerable to interception, injection, and tampering attacks. A man-in-the-middle attack on a model training data feed can silently poison the training dataset, degrading model safety properties in ways that may not surface until after deployment. Without integrity verification, an attacker who compromises a data relay can modify AI inputs or outputs in transit without triggering any alert." }, "standard": [ { "id": "nist_pf", "section": "PR.DS-P2", "title": "Data-in-transit are protected" }, { "id": "iso_27701", "section": "§7.4.9", "title": "PII transmission controls" }, { "id": "dama_dmbok", "section": "Ch. 7 Data Security", "title": "Cryptographic controls for data protection" }, { "id": "microsoft_purview", "section": "Platform encryption in transit (TLS)", "title": "TLS enforcement across Microsoft 365 and Azure services" } ], "sources": [ { "id": "microsoft_purview_data_governance_2024", "title": "Microsoft Purview (Data Governance)", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/purview/purview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_purview_data_governance_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Purview (Data Governance) requirements informing the apeiris://data/controls/DX-07 Data Exchange Security Controls control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Enforce TLS 1.3 (minimum TLS 1.2) for all data exchange connections. Require mutual TLS for AI system-to-system communication. Implement HMAC or digital signature verification for all AI data payloads classified as Confidential or higher. Deploy anti-replay controls using nonce and timestamp validation at the API gateway.", "steps": [ "Audit all AI data exchange endpoints and enforce a minimum TLS 1.2 policy with TLS 1.3 preferred; reject connections using deprecated protocols and log the attempt.", "Implement mutual TLS for all system-to-system AI data exchanges with certificate rotation managed through an automated PKI or ACME-compatible CA.", "Add HMAC payload integrity verification for all exchanges involving data classified as Confidential or higher, validating the HMAC on receipt before processing.", "Deploy nonce and timestamp validation at API gateway to prevent replay attacks on AI data feeds and model inference inputs.", "For non-repudiation requirements in financial, regulatory, or high-stakes AI decisions, implement Ed25519 digital signatures on exchange payloads with signing keys held in HSM custody." ], "data_engineer": { "summary": "Encryption, authentication, and integrity controls are implemented in the pipeline and API layer. Engineers are responsible for enforcing cryptographic requirements at every exchange boundary.", "actions": [ "Enforce TLS 1.3 on all new AI data exchange endpoints and deprecate any existing endpoints using TLS 1.1 or below on a tracked remediation schedule.", "Implement mTLS for all internal AI system-to-system data exchanges as a pipeline build-time requirement.", "Add HMAC integrity verification to pipeline connectors for Confidential and higher classified data with validation before downstream processing." ], "failure_signals": [ "AI data endpoints accepting TLS 1.1 or non-TLS connections.", "System-to-system exchanges without mutual certificate authentication.", "HMAC or signature verification absent from Confidential-classified data pipelines." ] }, "it_operations": { "summary": "Certificate management, PKI infrastructure, and cryptographic policy enforcement are operations responsibilities. Certificate expiry and protocol downgrade events are high-priority alerts.", "actions": [ "Manage automated certificate lifecycle for all AI data exchange endpoints via enterprise PKI with automated renewal at 30 days before expiry.", "Monitor TLS configuration compliance across AI endpoints and alert on protocol downgrade or certificate expiry within one business day.", "Configure HSM-backed key management for non-repudiation signing keys used in regulated AI data exchanges." ], "failure_signals": [ "Certificate expiry events for AI data exchange endpoints.", "Protocol downgrade alerts not actioned within the defined SLA.", "HSM not used for non-repudiation signing keys in regulated environments." ] }, "grc_auditor": { "summary": "Cryptographic controls for data in transit are a standard audit domain. Auditors verify protocol compliance, certificate management effectiveness, and integrity verification coverage.", "actions": [ "Run TLS configuration scans against all public-facing and internal AI data exchange endpoints and document any non-compliant findings.", "Verify certificate inventory completeness and check for expired or near-expiry certificates in the AI data exchange infrastructure.", "Confirm HMAC or signature verification is implemented for all Confidential and higher classified exchanges." ], "metrics": [ "AI endpoints with TLS 1.2+ enforced: target 100%.", "System-to-system AI exchanges with mTLS: target 100%.", "Confidential+ exchanges with payload integrity verification: target 100%.", "Certificate expiry incidents per quarter: target zero." ], "failure_signals": [ "AI data endpoints identified using deprecated protocols.", "Certificate expiry incidents not prevented by automated rotation.", "Confidential-classified exchanges without HMAC or signature verification." ] }, "legal_counsel": { "summary": "Encryption in transit is a legally expected 'appropriate technical measure' under GDPR Art. 32. Counsel translates that open norm into the concrete baseline the organization commits to, and assesses breach-notification exposure when it fails.", "actions": [ "Approve the minimum transport-security baseline referenced in DPAs and vendor contracts.", "Assess breach-notification obligations when unencrypted or misrouted transfers are discovered.", "Confirm cross-border transfers use transport protections consistent with transfer impact assessments." ], "failure_signals": [ "DPAs promising encryption standards that operations does not actually enforce.", "Interception or misrouting incidents assessed without legal input on notification duties.", "Security exceptions for legacy endpoints renewed indefinitely without documented risk acceptance." ] }, "data_governance_officer": { "summary": "Data Governance owns the register of approved exchange channels and their security baselines, and tracks exceptions. Encryption standards are a governance commitment, not just an engineering setting.", "actions": [ "Maintain the register of approved exchange channels with their required transport-security baseline and review it on a defined cadence.", "Track and time-bound exceptions for legacy endpoints that cannot meet the baseline, with named owners.", "Coordinate legal, security and engineering when the baseline changes so DPAs, configurations and monitoring move together." ], "failure_signals": [ "Exchange channels in production that are absent from the governance register.", "Open-ended exceptions with no owner or expiry.", "Baseline changes that never propagate to contracts or monitoring." ] } }, "maturity": { "current": "defined", "target": "managed", "notes": "TLS enforcement is broadly implemented but mutual TLS and payload-level integrity verification for AI data pipelines lag behind general application security practices." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "eu-high-risk-ai" ], "implementers": [ "Data Engineering", "IT Operations / Security", "Platform Engineering" ], "frameworks": [ { "framework": "iso_27701", "requirement_id": "§7.4.9", "fit": "direct", "rationale": "ISO/IEC 27701:2019 §7.4.9 requires PII transmitted over a data-transmission network to be subject to appropriate controls designed to ensure the data reaches its intended destination. Encryption in transit, endpoint authentication and integrity protection for AI data exchange implement this control. §7.4.3 concerns accuracy and quality, not transmission.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_pf", "requirement_id": "PR.DS-P2", "fit": "direct", "rationale": "NIST Privacy Framework PR.DS-P2 requires data-in-transit to be protected. TLS enforcement, mutual authentication and key management for AI data exchange are the direct implementation. PR.DS-P1 covers data-at-rest.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 7", "fit": "direct", "rationale": "DAMA DMBOK 2 Chapter 7 identifies cryptographic controls as a core data security practice for protecting data in transit. The encryption, authentication, and integrity controls specified here implement these practices for AI data exchange with explicit coverage of model training feeds and inference pathways.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_purview", "requirement_id": "Microsoft platform encryption in transit (TLS)", "fit": "partial", "rationale": "Encryption in transit on Microsoft platforms is provided by service-level TLS enforcement across Microsoft 365 and Azure rather than by Purview; DX-07 implementations on those estates rely on platform TLS plus Purview DLP visibility over exchange endpoints.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "gdpr", "requirement_id": "Art. 5(1)(f)", "fit": "partial", "rationale": "GDPR Article 5(1)(f) requires personal data to be processed with appropriate security including protection against unauthorized disclosure and alteration. Encryption in transit and integrity verification are the minimum technical measures needed to satisfy this requirement for data exchanges involving personal data in AI systems.", "normative_force": "binding-law", "source_version": "2016/679", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" } ], "canonical_id": "apeiris://data/controls/DX-07", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "All data exchanged by AI systems is encrypted in transit using TLS 1.2 minimum (TLS 1.3 preferred), with mutual TLS enforced for all system-to-system exchanges, HMAC or digital signature payload integrity verification active for all Confidential-or-higher classified transfers, and anti-replay controls enforcing nonce and timestamp validation at the API gateway. No AI data exchange endpoint accepts connections using deprecated protocols, and no Confidential-or-higher payload is processed without integrity verification.", "evidence_required": [ "tls_configuration_scan_report covering all AI data exchange endpoints with protocol_version, cipher_suite, certificate_expiry, and compliance_status for each endpoint confirming no endpoint accepts connections below TLS 1.2", "mtls_configuration_audit listing all system-to-system AI exchange pathways with mutual_auth=true, certificate_authority_reference, and certificate_rotation_date for each pathway", "hmac_or_signature_verification_log showing integrity check results including algorithm, key_id, and pass/fail verdict for each Confidential-or-higher classified transfer in the review period", "replay_protection_configuration_evidence showing nonce_validation and timestamp_tolerance_seconds settings active at the API gateway for all AI data endpoints", "certificate_lifecycle_management_report confirming automated renewal triggered at >= 30 days before expiry for all AI data exchange certificates with zero expiry incidents in the review period" ], "machine_tests": [ "Attempt to establish a TLS 1.1 connection to an AI data exchange endpoint → assert the connection is rejected with a TLS handshake failure and a protocol_downgrade_attempt event is logged to the security audit trail", "Submit a Confidential-classified data payload to an AI data exchange API with a missing or cryptographically invalid HMAC header → assert the API returns 400 with error_code=integrity_verification_failed and the payload is not processed", "Replay a captured valid exchange request using the same nonce and a timestamp outside the configured tolerance window → assert the API gateway returns 401 with error_code=replay_detected and no data is processed or logged as a successful transfer", "Submit a mTLS connection request from an AI system using an expired client certificate → assert the TLS handshake fails with a certificate_expired error and a connection_rejected event is emitted to the security log" ], "human_review": [ "Review the TLS configuration scan results for all AI data exchange endpoints to identify any endpoints with non-compliant protocol versions, weak cipher suites, or certificates approaching expiry that automated tooling has not triggered renewal for", "Assess the HMAC and digital signature key management approach for Confidential-or-higher exchanges to verify HSM custody is in place for non-repudiation signing keys used in regulated AI data contexts", "Evaluate the certificate lifecycle management process to determine whether automated renewal has been consistently triggered at the 30-day pre-expiry threshold across the full AI data exchange endpoint inventory" ], "blocking_effect": "blocks-deployment", "normative_status": "certification-standard", "anti_patterns": [ "Applying TLS enforcement only to external-facing AI data exchange endpoints and exempting internal system-to-system connections as trusted network traffic, leaving internal AI data flows vulnerable to lateral movement and insider interception", "Using a shared HMAC secret key across all AI data pipelines rather than per-pipeline keys, so a compromise of one pipeline's key material enables tampering across all pipelines using the same shared secret", "Relying on timestamp-based replay protection alone without nonce validation, allowing an attacker who captures a valid request to replay it later within the timestamp tolerance window without detection", "Treating certificate expiry as an incident to remediate reactively rather than a preventable event, resulting in expiry-induced outages that disrupt AI data pipelines and force emergency certificate rollout under pressure", "Storing non-repudiation signing keys for high-sensitivity AI data exchanges in application-accessible software key stores rather than HSMs, enabling key extraction by an attacker who compromises the application layer" ], "update_status": "current", "layer_code": "DX" }, { "id": "DX-08", "layer": "DX", "plane": "lifecycle", "name": "Data Exchange Evidence Package", "plain": "A structured evidence package must be compiled demonstrating that all DX-layer controls — sensitivity classification, transfer authorization, API contract governance, cross-border compliance, third-party agreements, audit trail, and security controls — are implemented, operating effectively, and producing evidence sufficient for regulatory review and attestation.", "threat": { "tags": [ "evidence-gap", "compliance-assertion-failure", "audit-trail-incompleteness", "attestation-deficiency" ], "desc": "Without a compiled evidence package, organizations face fragmented and inconsistent responses to regulatory inquiries, audit requests, and AI governance reviews. Evidence gaps become visible only at the point of scrutiny — too late for remediation before enforcement action. An uncompiled evidence package also masks control gaps that individually appear minor but collectively represent systemic data exchange governance failure." }, "standard": [ { "id": "eu_ai_act", "section": "Art. 11 + Annex IV", "title": "Technical documentation and compliance evidence" }, { "id": "gdpr", "section": "Art. 5(2) Accountability", "title": "Demonstrating compliance with data protection principles" }, { "id": "cobit_2019", "section": "MEA02", "title": "Monitoring and evaluation of internal controls" }, { "id": "dama_dmbok", "section": "Ch. 7 Data Security", "title": "Data governance evidence and compliance reporting" } ], "sources": [ { "id": "eu_ai_act", "title": "EU AI Act", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2024/1689", "published_on": "2024-07-12", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2024/1689/oj/eng", "license": "open-access", "status": "current", "flagship": false, "source_id": "eu_ai_act", "relationship": "normative_requirement", "rationale": "Establishes EU AI Act Art. 11 + Annex IV requirements informing the apeiris://data/controls/DX-08 Data Exchange Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "gdpr", "title": "GDPR", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2016/679", "published_on": "2016-04-27", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj", "license": "open-access", "status": "current", "flagship": false, "source_id": "gdpr", "relationship": "normative_requirement", "rationale": "Establishes GDPR Art. 5(2) requirements informing the apeiris://data/controls/DX-08 Data Exchange Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "cobit_2019", "title": "COBIT 2019", "authority": "ISACA", "source_type": "standard", "normative_force": "industry-framework", "version": "2019", "published_on": "2018-11-12", "retrieved_on": "2026-07-02", "canonical_url": "https://www.isaca.org/resources/cobit", "license": "proprietary", "status": "current", "flagship": false, "source_id": "cobit_2019", "relationship": "informative_reference", "rationale": "Establishes COBIT 2019 MEA02 requirements informing the apeiris://data/controls/DX-08 Data Exchange Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "dama_dmbok", "title": "DAMA-DMBOK 2nd Ed", "authority": "DAMA International", "source_type": "standard", "normative_force": "best-practice", "version": "2 (2nd Edition, Revised 2024)", "published_on": "2017-07-05", "retrieved_on": "2026-07-02", "canonical_url": "https://www.dama.org/cpages/body-of-knowledge", "license": "proprietary", "status": "current", "flagship": false, "source_id": "dama_dmbok", "relationship": "informative_reference", "rationale": "Establishes DAMA-DMBOK 2nd Ed Ch. 7 requirements informing the apeiris://data/controls/DX-08 Data Exchange Evidence Package control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Compile the DX Evidence Package as a structured artifact referencing evidence outputs from DX-01 through DX-07. The package must include: taxonomy coverage metrics (DX-01), authorization matrix flow coverage (DX-02), API contract registry completeness (DX-03), transfer impact register with lawful basis documentation (DX-04), third-party agreement inventory (DX-05), audit trail completeness and retention verification (DX-06), and cryptographic control scan results (DX-07).", "steps": [ "Define the DX Evidence Package template specifying required evidence artifacts, responsible owners, minimum acceptable evidence state for each control, and the governance sign-off chain.", "Implement automated evidence collection jobs that pull metrics, coverage percentages, and artifact references from each DX control's operational systems on a scheduled basis.", "Compile the package on a quarterly cadence and additionally before any regulatory audit, material system change, or AI deployment to high-risk-sector environments.", "Route the compiled package through a governance review gate requiring Data Governance Officer sign-off for operational reviews and Legal Counsel sign-off for regulatory submission.", "Retain each compiled evidence package for the applicable regulatory retention period in tamper-evident storage with version history preserved." ], "data_governance_officer": { "summary": "The DX Evidence Package is the primary deliverable of the DX governance program. Data Governance owns the package template, compilation cadence, and sign-off process.", "actions": [ "Maintain the DX Evidence Package template and update it when DX control definitions or regulatory requirements change.", "Chair the quarterly evidence package review and sign off on the compiled package before distribution.", "Use the package as the primary response artifact for regulatory data governance inquiries and AI audits." ], "failure_signals": [ "DX Evidence Package not compiled within the prior quarter.", "Package missing evidence artifacts for one or more DX-layer controls.", "Package not signed off by the Data Governance Officer before regulatory submission." ] }, "legal_counsel": { "summary": "Legal reviews the DX Evidence Package prior to regulatory submission, verifying that lawful basis documentation, agreement inventories, and cross-border transfer records are complete and legally defensible.", "actions": [ "Review the transfer impact register section (DX-04 evidence) and third-party agreement inventory (DX-05 evidence) before package sign-off for regulatory submissions.", "Confirm that lawful basis documentation in the package aligns with current regulatory requirements and has not been superseded by adequacy decision changes.", "Advise on evidence package retention requirements for each applicable regulatory framework and update the retention schedule when requirements change." ], "failure_signals": [ "Package submitted for regulatory review without legal sign-off.", "Lawful basis documentation in the package outdated or referencing superseded adequacy decisions.", "Retention schedule for compiled packages not aligned with the longest applicable requirement." ] }, "grc_auditor": { "summary": "The DX Evidence Package is the primary audit input for data exchange governance reviews. Auditors validate package completeness, evidence currency, and consistency with operational system state.", "actions": [ "Request the current and prior DX Evidence Package and compare for regression in control coverage metrics across reporting periods.", "Cross-validate a sample of package evidence artifacts against the operational systems they reference to confirm accuracy.", "Assess the package against EU AI Act Annex IV and GDPR Article 30 completeness requirements." ], "metrics": [ "DX Evidence Package compiled on schedule: target 4 per year plus ad hoc for major deployments.", "Evidence artifacts present for all seven DX-layer controls: target 100%.", "Governance sign-off (DGO and Legal) recorded for each package: target 100%.", "Time from evidence compilation to package sign-off: target <= 10 business days." ], "failure_signals": [ "DX Evidence Package not compiled for a quarter or for a major AI deployment in a high-risk sector.", "Missing evidence artifacts for any DX-layer control in the compiled package.", "Package evidence artifacts inconsistent with operational system state on audit cross-check." ] }, "it_operations": { "summary": "Operations supports evidence collection automation and tamper-evident storage for compiled packages. Automated collection jobs must run reliably and alert on collection failures within one business day.", "actions": [ "Deploy and maintain automated evidence collection jobs for each of the seven DX-layer controls.", "Configure tamper-evident storage with WORM settings for all compiled DX Evidence Packages.", "Alert the Data Governance Officer on collection job failures within one business day to preserve compilation schedule." ], "failure_signals": [ "Evidence collection jobs failing silently without alert.", "Compiled packages stored without tamper-evident controls.", "Collection job coverage gaps leaving any DX control without automated evidence retrieval." ] }, "data_engineer": { "summary": "The DX evidence package is generated, not compiled by hand. Engineers instrument exchange infrastructure so audit trails, authorization records and security-control evidence export automatically with hashes.", "actions": [ "Automate evidence exports from exchange gateways, transfer logs and authorization systems into the package pipeline.", "Attach content hashes and timestamps to every exported artifact for integrity verification.", "Test package assembly regularly so a complete, verifiable package is producible on demand." ], "failure_signals": [ "Evidence assembled manually from screenshots and ad-hoc queries.", "Exported artifacts without hashes or timestamps.", "Package assembly failing when a regulator or auditor actually asks." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Evidence compilation for data exchange governance is almost universally manual and ad hoc. Automated evidence collection and structured package compilation represents a significant maturity uplift for most organizations." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "Data Governance Office", "Legal / Privacy Counsel", "GRC / Internal Audit" ], "frameworks": [ { "framework": "eu_ai_act", "requirement_id": "Art. 11 + Annex IV", "fit": "direct", "rationale": "EU AI Act Article 11 and Annex IV require high-risk AI systems to maintain technical documentation demonstrating compliance including data governance practices. The DX Evidence Package is the mechanism for compiling and maintaining this documentation for the data exchange layer as a structured, reviewable artifact.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "gdpr", "requirement_id": "Art. 5(2)", "fit": "direct", "rationale": "GDPR Article 5(2) accountability principle requires controllers to be able to demonstrate compliance with the data protection principles. The DX Evidence Package is the structured mechanism for maintaining demonstrable compliance evidence for data exchange activities involving personal data.", "normative_force": "binding-law", "source_version": "2016/679", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "cobit_2019", "requirement_id": "MEA02", "fit": "direct", "rationale": "COBIT 2019 MEA02 requires systematic monitoring and evaluation of the internal control system. The DX Evidence Package provides the compiled evidence base for evaluating whether DX-layer controls are operating effectively and producing governance outcomes on a recurring schedule.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 7", "fit": "partial", "rationale": "DAMA DMBOK 2 Chapter 7 identifies compliance reporting and audit support as core data security activities. The evidence package operationalizes these activities by providing a structured, pre-compiled compliance artifact for data exchange governance.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://data/controls/DX-08", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "The DX Evidence Package must be compiled as a structured artifact within the prior quarter referencing verifiable evidence outputs from each of DX-01 through DX-07, with governance sign-off recorded from both the Data Governance Officer and Legal Counsel before any regulatory submission or high-risk AI deployment.", "evidence_required": [ "dx_evidence_package artifact with section references and evidence artifact links for all seven DX-layer controls (DX-01 through DX-07), timestamped and version-identified", "governance_sign_off_record showing Data Governance Officer and Legal Counsel approvals with signatures and dates for each compiled package", "evidence_collection_job_log confirming automated or manual collection runs completed for each DX control within the compilation window", "package_retention_manifest showing each compiled DX Evidence Package stored in tamper-evident, WORM-compliant storage with retention schedule annotation", "regulatory_submission_record (where applicable) showing the compiled package version submitted and the receiving regulatory body or auditor" ], "machine_tests": [ "Query the evidence package registry for the most recent DX Evidence Package → assert compiled_date is within 90 days of today and status is 'signed-off'", "Enumerate evidence artifact references in the compiled package → assert all seven section IDs (DX-01 through DX-07) are present with non-null artifact_ref values", "Check governance_sign_off_record for most recent package → assert both 'data_governance_officer' and 'legal_counsel' approver fields are populated with timestamps", "Query storage backend for package object → assert immutable=true and retention_policy is set to the applicable regulatory retention period" ], "human_review": [ "Review the compiled DX Evidence Package for completeness against each DX-layer control and assess whether evidence artifact quality meets regulatory reviewability standards", "Verify that the governance sign-off chain reflects the current responsible Data Governance Officer and Legal Counsel, and that sign-off dates precede any regulatory submission date", "Assess evidence package retention schedule against the longest applicable regulatory requirement across all active jurisdictions and confirm alignment" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Assembling the evidence package only when a regulator issues a specific inquiry rather than maintaining it on a defined quarterly cadence", "Storing compiled evidence packages in mutable storage systems that allow post-compilation modification without audit trail, undermining the tamper-evident requirement", "Treating the DX Evidence Package as a point-in-time report rather than a versioned artifact with a chain of custody from collection through sign-off", "Allowing deployment of high-risk AI systems in regulated sectors without a signed-off DX Evidence Package compiled after the most recent material system change", "Compiling the package with placeholder evidence references for one or more DX controls rather than actual artifact links from operational systems" ], "update_status": "current", "layer_code": "DX" }, { "id": "DI-01", "layer": "DI", "plane": "data", "name": "Data Integrity Baseline and Checksum Monitoring", "plain": "Every critical AI dataset used for training, fine-tuning, or evaluation must have a cryptographic fingerprint established at ingest and monitored continuously for unauthorized changes throughout its lifecycle.", "threat": { "tags": [ "data-tampering", "supply-chain-poisoning", "silent-dataset-corruption" ], "desc": "Without baseline checksums, adversaries can silently alter training datasets to introduce poisoned samples or bias without triggering alerts. Supply-chain attacks that modify data at rest go undetected until downstream model behavior diverges in production. Corrupted datasets used in regulated AI contexts create compliance liability that cannot be retrospectively remediated because the integrity breach cannot be scoped." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 13", "title": "Data Quality — integrity dimension" }, { "id": "iso_8000", "section": "ISO 8000-61", "title": "Data quality management — process reference model" }, { "id": "eu_ai_act", "section": "Art. 10(2)(e)", "title": "Assessment of availability, quantity and suitability of data sets" }, { "id": "cobit_2019", "section": "DSS06.05", "title": "Ensure traceability and accountability for information events" } ], "sources": [ { "id": "microsoft_purview_data_governance_2024", "title": "Microsoft Purview (Data Governance)", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/purview/purview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_purview_data_governance_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Purview (Data Governance) requirements informing the apeiris://data/controls/DI-01 Data Integrity Baseline and Checksum Monitoring control.", "reviewed_on": "2026-07-01" }, { "id": "databricks_unity_catalog_2024", "title": "Databricks Unity Catalog", "authority": "Databricks, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.databricks.com/en/data-governance/unity-catalog/index.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "databricks_unity_catalog_2024", "relationship": "informative_reference", "rationale": "Establishes Databricks Unity Catalog requirements informing the apeiris://data/controls/DI-01 Data Integrity Baseline and Checksum Monitoring control.", "reviewed_on": "2026-07-02" }, { "id": "eu_data_act_2023_2854", "title": "EU Data Act — Regulation (EU) 2023/2854", "authority": "European Parliament and Council of the European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2023/2854", "published_on": "2023-12-13", "retrieved_on": "2026-06-29", "canonical_url": "https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A32023R2854", "license": "open-access", "status": "current", "flagship": false, "source_id": "eu_data_act_2023_2854", "relationship": "normative_requirement", "rationale": "Establishes EU Data Act — Regulation (EU) 2023/2854 requirements informing the apeiris://data/controls/DI-01 Data Integrity Baseline and Checksum Monitoring control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Compute SHA-256 checksums for all critical datasets at ingest; store hashes in an append-only integrity ledger; run scheduled and event-driven integrity scans against stored hashes; alert on any deviation with full provenance context.", "steps": [ "Establish an integrity ledger (append-only store, e.g., immutable S3 object store or distributed ledger) to record dataset fingerprints at ingest time.", "Instrument data ingest pipelines to compute SHA-256 (or SHA-3) checksums at the partition, file, and record-batch levels and register them in the ledger before any downstream use.", "Deploy scheduled integrity scans that recompute checksums against the ledger baseline and emit alerts with dataset ID, scope, and delta on any mismatch.", "Integrate alerts into SIEM and data governance workflows so that any integrity deviation triggers quarantine of affected datasets pending investigation." ], "data_governance_officer": { "summary": "This control establishes the foundational integrity assurance that data governance policy requires for AI datasets under management.", "actions": [ "Define the critical dataset inventory that must be enrolled in baseline checksum monitoring.", "Establish SLAs for integrity scan frequency aligned to dataset sensitivity classification.", "Review integrity audit logs quarterly and sign off as part of data governance committee records." ], "failure_signals": [ "Datasets enrolled in checksum monitoring falls below 95% of critical inventory.", "Integrity scan frequency misses SLA for high-sensitivity datasets.", "Integrity deviations not resolved within the defined remediation window." ] }, "data_engineer": { "summary": "Instrumentation of ingest pipelines is the primary implementation responsibility; checksums must be computed at the pipeline, not retrofitted afterward.", "actions": [ "Add checksum computation to all ETL and ELT pipeline output stages using SHA-256 at the partition or file level.", "Publish checksums to the integrity ledger as part of each pipeline run's metadata output.", "Build alerting hooks that block downstream pipeline stages if ledger registration fails." ], "failure_signals": [ "Pipeline runs that complete without checksum registration in the ledger.", "Checksum scan job failures not escalated within 30 minutes.", "Integrity alerts older than the SLA that remain in open status." ] }, "grc_auditor": { "summary": "The integrity ledger and deviation logs are the primary audit artifacts demonstrating this control is operating effectively.", "actions": [ "Pull the integrity ledger export covering the audit period and cross-reference against the critical dataset inventory.", "Sample 15% of datasets and manually verify that checksum records in the ledger match current dataset state.", "Review alert resolution logs to confirm deviations were triaged and closed within the defined SLA." ], "metrics": [ "Critical dataset checksum enrollment rate: target 100%.", "Integrity scan coverage: target 100% of enrolled datasets within scan frequency window.", "Mean time to resolve integrity deviations: target ≤ 4 hours for high-sensitivity datasets." ], "failure_signals": [ "Enrollment rate below 95% for two consecutive reporting periods.", "Unresolved integrity deviations older than SLA present in the audit sample.", "Ledger entries that cannot be verified against source dataset records." ] }, "it_operations": { "summary": "Operations teams are responsible for maintaining the integrity ledger infrastructure and ensuring scan jobs execute reliably on schedule.", "actions": [ "Deploy and maintain the append-only integrity ledger with appropriate availability and durability targets.", "Monitor integrity scan job health and alert on job failures within 30 minutes.", "Maintain storage-level immutability controls (e.g., S3 Object Lock) on the ledger to prevent tampering." ], "failure_signals": [ "Integrity ledger unavailability events that prevent scan completion.", "Scheduled scan jobs failing silently without alert.", "Ledger storage mutability detected by configuration drift scan." ] }, "legal_counsel": { "summary": "Integrity evidence is what makes data trustworthy in a legal sense — accurate under GDPR Art. 5(1)(d) and reliable as evidence. Counsel relies on checksum baselines when defending the provenance and accuracy of AI training data.", "actions": [ "Confirm integrity-monitoring coverage includes datasets subject to accuracy obligations and litigation holds.", "Use integrity baselines to substantiate accuracy and provenance claims in regulatory filings.", "Advise on preservation duties when integrity incidents affect data under legal hold." ], "failure_signals": [ "Accuracy representations made to regulators without integrity evidence behind them.", "Litigation-hold data modified without detection.", "Integrity incidents on regulated datasets closed without a legal notification assessment." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most enterprises have ad hoc checksum practices for file transfers but lack systematic coverage of AI training datasets with append-only ledger tracking." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "eu-high-risk-ai" ], "implementers": [ "Data Engineering", "Platform Engineering", "Data Governance Office" ], "frameworks": [ { "framework": "dama_dmbok", "requirement_id": "Ch. 13", "fit": "direct", "rationale": "DAMA-DMBOK2 Chapter 13 (Data Quality) includes integrity among the data quality dimensions a DQ program measures and monitors. Checksum baselines make the integrity dimension continuously measurable for AI datasets.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "iso_8000", "requirement_id": "ISO 8000-61", "fit": "direct", "rationale": "ISO 8000-61 specifies the process reference model for data quality management, with data quality monitoring and control as defined processes. Checksum baselines and continuous integrity monitoring are the technical implementation of those processes for AI datasets.", "normative_force": "voluntary-standard", "source_version": "2022", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "eu_ai_act", "requirement_id": "Art. 10(2)(e)", "fit": "partial", "rationale": "EU AI Act Article 10(2)(e) requires an assessment of the availability, quantity and suitability of the data sets needed for the high-risk AI system; bias examination sits in Article 10(2)(f)-(g), not (e). Integrity baselines and checksum monitoring provide verifiable evidence that datasets remain the datasets that were assessed, supporting suitability claims over time.", "normative_force": "binding-law", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "cobit_2019", "requirement_id": "DSS06.05", "fit": "partial", "rationale": "COBIT 2019 practice DSS06.05 — ensure traceability and accountability for information events — requires that information events can be traced to accountable parties. Checksum baselines give every dataset event a verifiable integrity anchor, making information events traceable to a known-good state.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "microsoft_purview", "requirement_id": "Data Catalog — Data Lineage", "fit": "partial", "rationale": "Microsoft Purview's data catalog lineage tracking can record the provenance chain for AI datasets, complementing checksum baselines by providing the context needed to investigate integrity deviations. The platform's integration with Azure Data Factory and Synapse pipelines enables automated checksum registration as a lineage event. This provides a cloud-native implementation path for the integrity ledger component of this control.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "databricks_unity", "requirement_id": "Unity Catalog — Data Lineage and Auditing", "fit": "partial", "rationale": "Databricks Unity Catalog provides column-level lineage and audit logging for datasets managed within the Lakehouse architecture, enabling integrity tracking as datasets flow through transformation stages. The Delta log format provides built-in transaction history that can serve as an integrity audit trail for AI pipeline runs. Enterprises building AI pipelines on Databricks can leverage Unity Catalog's audit features to complement this control's ledger requirements.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "eu_data_act", "requirement_id": "Art. 3 (access by design)", "fit": "adjacent", "rationale": "EU Data Act Article 3 obliges connected-product and related-service providers to make product data accessible by design; it does not regulate integrity monitoring. The adjacency is that data holders must make data available with the same quality they enjoy themselves (Arts. 4-5), and checksum baselines are how equivalent quality at handoff can be evidenced.", "normative_force": "binding-law", "source_version": "2023/2854", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "owasp_ai_exchange", "requirement_id": "dataqualitycontrol", "fit": "supporting", "rationale": "DI-01 registers SHA-256 baselines for training datasets and scans for deviation, detecting tampering/poisoning of training data integrity.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "mitre", "requirement_id": "AML.M0007", "fit": "supporting", "direction": "control-supports-requirement", "rationale": "Control \"Every critical AI dataset used for training, fine-tuning, or evaluation must have a…\" enacts ATLAS mitigation AML.M0007 Sanitize Training Data; OpenCRE crosswalks this control’s OWASP AI Exchange concept (dataqualitycontrol) to this mitigation.", "source_version": "v2026.06", "reviewed_on": "2026-07-08", "source_status": "authoritative", "mapping_confidence": "medium", "legal_status": "not-applicable", "control_readiness": "approved", "implementation_maturity": "developing", "assurance_result": "not-assessed", "evidence_status": "no-evidence", "normative_force": "industry-framework", "basis": "asserted", "relation": "equivalent_to", "correction": "ap38-opencre-enhancement 2026-07-08" } ], "canonical_id": "apeiris://data/controls/DI-01", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every critical AI dataset used for training, fine-tuning, or evaluation must have a SHA-256 cryptographic checksum registered in an append-only integrity ledger at ingest, with scheduled integrity scans confirming no deviation from the baseline fingerprint throughout the dataset lifecycle.", "evidence_required": [ "integrity_ledger_export showing dataset_id, checksum_algorithm, checksum_value, and registered_at timestamp for each enrolled critical dataset", "integrity_scan_log confirming scheduled scans completed for all enrolled datasets within the defined SLA window, including scan_timestamp, dataset_id, and result (pass/deviation)", "deviation_alert_record for any checksum mismatches detected, including dataset_id, detection_timestamp, scope_of_deviation, and remediation_status", "critical_dataset_inventory showing all datasets requiring enrollment, with enrollment_status field indicating coverage percentage" ], "machine_tests": [ "Query integrity ledger for all datasets in the critical_dataset_inventory → assert enrollment coverage >= 95% of critical dataset count", "Retrieve latest scan run timestamp per dataset → assert all high-sensitivity datasets were scanned within the SLA frequency window (e.g., within 24 hours for daily SLA)", "Inject a single byte modification into a test dataset copy and run the integrity scan → assert deviation_alert fires within one scan cycle and includes dataset_id and affected partition", "Attempt to append a record to the integrity ledger with a backdated registered_at timestamp → assert the ledger rejects or flags out-of-order writes" ], "human_review": [ "Review the critical dataset inventory for completeness — verify that newly ingested datasets are enrolled in checksum monitoring within the required onboarding window", "Assess integrity deviation records from the prior quarter for patterns suggesting systematic supply-chain or insider tampering, not just individual data errors", "Confirm that the integrity ledger storage backend is configured as append-only or WORM and that no administrative path exists for silent record deletion" ], "blocking_effect": "advisory", "normative_status": "binding-law", "anti_patterns": [ "Computing checksums only at the whole-file level rather than at partition or record-batch granularity, which allows adversaries to tamper with individual partitions while the file-level hash remains unchanged", "Storing integrity checksums in the same mutable storage system as the dataset itself, enabling an attacker who can modify the data to also modify the reference hash", "Running integrity scans on an ad hoc or manual basis rather than as a scheduled, automated job, creating windows of undetected tampering between scans", "Using non-cryptographic checksums (CRC32, Adler-32) for security-sensitive datasets where collision resistance is required", "Enrolling datasets in checksum monitoring after initial ingest processing has already occurred, leaving the pre-registration window as a blind spot for tampering" ], "update_status": "current", "layer_code": "DI" }, { "id": "DI-02", "layer": "DI", "plane": "control", "name": "Tamper Detection for AI Inference Inputs", "plain": "Data fed to AI models during inference must be validated against known-good schemas and structural signatures to detect unauthorized modification, injection, or manipulation before model execution.", "threat": { "tags": [ "inference-input-tampering", "prompt-injection", "adversarial-input-manipulation" ], "desc": "Adversaries who can modify inference inputs can manipulate AI outputs without attacking the model itself, bypassing model-level controls entirely. Man-in-the-middle attacks on API calls, compromised upstream data pipelines, and prompt injection vectors all exploit the gap between data validation at rest and data validation at inference time. Without runtime input integrity checks, attackers can cause AI systems to produce fraudulent outputs that appear authoritative and propagate into downstream decisions." }, "standard": [ { "id": "eu_ai_act", "section": "Art. 15", "title": "Accuracy, robustness and cybersecurity — resilience to input tampering" }, { "id": "dama_dmbok", "section": "Ch. 13", "title": "Data Quality — in-pipeline quality inspection" }, { "id": "nist_pf", "section": "PR.DS-P6", "title": "Integrity checking mechanisms verify information integrity" }, { "id": "cobit_2019", "section": "APO12.06", "title": "Respond to risk — data integrity controls" } ], "sources": [ { "id": "aws_lake_formation_macie_2024", "title": "AWS data governance services (Lake Formation, Macie, Glue, S3)", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/lake-formation/latest/dg/what-is-lake-formation.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_lake_formation_macie_2024", "relationship": "informative_reference", "rationale": "Establishes AWS data governance services (Lake Formation, Macie, Glue, S3) requirements informing the apeiris://data/controls/DI-02 Tamper Detection for AI Inference Inputs control.", "reviewed_on": "2026-07-02" }, { "id": "google_dataplex_bigquery_2024", "title": "Google Cloud Dataplex & BigQuery", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://cloud.google.com/dataplex/docs/introduction", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "google_dataplex_bigquery_2024", "relationship": "informative_reference", "rationale": "Establishes Google Cloud Dataplex & BigQuery requirements informing the apeiris://data/controls/DI-02 Tamper Detection for AI Inference Inputs control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Establish schema contracts and structural signatures for all inference input payloads; validate inputs against contracts at the inference gateway before forwarding to models; log validation outcomes and alert on structural anomalies or schema violations.", "steps": [ "Define and version schema contracts (JSON Schema, Avro, or Protobuf) for all AI inference input formats and publish them to a schema registry.", "Deploy an inference input validation middleware at the API gateway layer that enforces schema compliance, value-range checks, and structural signature verification before requests reach model endpoints.", "Implement a canary-baseline anomaly detector that flags inference inputs deviating statistically from the distribution established during model acceptance testing.", "Log all validation outcomes including pass, reject, and anomaly-flagged requests to an immutable audit log accessible to the SIEM." ], "data_governance_officer": { "summary": "This control closes the gap between dataset-level integrity assurance and runtime inference integrity, ensuring that data governance policies extend through the full inference path.", "actions": [ "Define the data classification tier for inference inputs and map required validation rigor to each tier.", "Establish policy requiring schema contract registration before any new AI inference endpoint goes live.", "Review inference input anomaly trends quarterly and escalate persistent patterns to model risk management." ], "failure_signals": [ "Inference endpoints without registered schema contracts.", "Anomaly detection coverage falls below 90% of production inference traffic.", "Unresolved schema violation alerts older than the defined remediation SLA." ] }, "data_engineer": { "summary": "Schema contracts must be built into pipeline output stages and the validation middleware must be maintained as a first-class infrastructure component.", "actions": [ "Author and register schema contracts for all inference input payload types in the schema registry.", "Build the inference gateway validation layer with schema enforcement and anomaly scoring using statistical distribution baselines.", "Publish validation rejection metrics to the observability platform and configure paging alerts for sustained high rejection rates." ], "failure_signals": [ "Schema registry entries that are stale relative to production payload formats.", "Validation middleware bypassed by direct model endpoint access.", "Anomaly baseline models not refreshed after model retraining events." ] }, "grc_auditor": { "summary": "The schema registry, validation logs, and anomaly alert records are the primary audit artifacts demonstrating runtime inference integrity controls are operating.", "actions": [ "Pull the schema registry export and verify all active inference endpoints have a current registered schema contract.", "Sample validation logs for the audit period and confirm that rejections were escalated and resolved within SLA.", "Verify that the anomaly baseline was updated following the most recent model retraining cycle." ], "metrics": [ "Schema contract coverage of production inference endpoints: target 100%.", "Inference input validation rejection escalation rate: target 100% of rejected batches triaged within SLA.", "Anomaly baseline refresh lag after model retraining: target ≤ 5 business days." ], "failure_signals": [ "Inference endpoints without schema contracts found in audit sample.", "Validation rejection events with no corresponding triage ticket.", "Anomaly baseline older than two model retraining cycles." ] }, "it_operations": { "summary": "The inference gateway validation layer is a critical-path infrastructure component requiring high availability and operational monitoring.", "actions": [ "Monitor inference gateway validation service availability and alert on degraded states within 5 minutes.", "Ensure that validation middleware failures do not silently fail open, allowing unvalidated inputs to reach model endpoints.", "Maintain schema registry replication and failover to prevent validation service unavailability." ], "failure_signals": [ "Validation middleware failures that result in pass-through of unvalidated inference inputs.", "Schema registry unavailability events lasting more than 5 minutes.", "Validation latency exceeding the defined SLA threshold for production inference paths." ] }, "legal_counsel": { "summary": "Tampered inference inputs can turn an AI system into a source of legally significant errors. Counsel assesses liability exposure from decisions made on manipulated inputs and the notification duties tampering may trigger.", "actions": [ "Define which tamper events require legal escalation — regulated decisions, personal data, safety-relevant systems.", "Assess EU AI Act Art. 73 and GDPR Art. 33 notification exposure for confirmed input-tampering incidents.", "Preserve tamper evidence to litigation standards when incidents may lead to claims." ], "failure_signals": [ "Tamper alerts on consequential-decision systems triaged without legal involvement.", "Notification decisions made after statutory clocks have already run.", "Tamper evidence overwritten before preservation." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Most AI deployments validate inference inputs at the application layer but lack schema registry enforcement and statistical anomaly detection at the inference gateway." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "multi-tenant", "eu-high-risk-ai" ], "implementers": [ "Data Engineering", "MLOps", "Platform Engineering", "Security Engineering" ], "frameworks": [ { "framework": "eu_ai_act", "requirement_id": "Art. 15", "fit": "direct", "rationale": "EU AI Act Article 15 requires high-risk AI systems to achieve appropriate accuracy, robustness and cybersecurity, and Article 15(5) explicitly requires resilience against attempts to alter inputs or performance — including data poisoning and adversarial manipulation. Tamper detection for inference inputs is a direct technical measure toward this requirement; Article 10(3) concerns training dataset quality criteria, not runtime input integrity.", "normative_force": "binding-law", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 13", "fit": "direct", "rationale": "DAMA-DMBOK2 Chapter 13's data quality operations include inspecting data as it moves through pipelines; tamper detection extends those in-pipeline inspection points to adversarial modification of inference inputs.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "nist_pf", "requirement_id": "PR.DS-P6", "fit": "partial", "rationale": "NIST Privacy Framework PR.DS-P6 requires integrity checking mechanisms to be used to verify software, firmware, and information integrity. Tamper detection on inference inputs applies such integrity checking to operational AI data.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_2019", "requirement_id": "APO12.06", "fit": "partial", "rationale": "COBIT 2019 APO12.06 requires organizations to respond to identified risks including data integrity risks that affect operational processing. The inference input validation gateway implements the risk response controls that APO12 risk treatment requires. Monitoring validation rejection rates and anomaly trends satisfies COBIT's requirement for risk indicator tracking and escalation.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "google_dataplex", "requirement_id": "Dataplex Auto Data Quality (scheduled/batch scans)", "fit": "partial", "rationale": "Dataplex Auto Data Quality executes rule-based scans on a schedule or on demand — batch validation of tables rather than inline runtime interception. It can detect tampering after the fact; true inline tamper detection requires pipeline-level checks in front of inference endpoints.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "aws_lake_formation", "requirement_id": "AWS Glue Schema Registry", "fit": "partial", "rationale": "AWS Glue Schema Registry — a Glue capability, distinct from Lake Formation — provides a managed schema registry that can enforce schema contracts on Kafka, Kinesis, and batch data streams feeding AI inference endpoints on AWS. Its schema-evolution controls ensure inference input format changes are explicitly versioned and validated before consumption.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "owasp_ai_exchange", "requirement_id": "runtimemodeliointegrity", "fit": "supporting", "rationale": "DI-02 validates every inference payload against a registered schema and signed reference before model execution, protecting model input integrity at runtime.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://data/controls/DI-02", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every data payload submitted to an AI model at inference time must be validated against a registered structural schema and, where applicable, a signed reference signature before model execution begins, with any validation failure resulting in rejection of the inference request and generation of a tamper-detection alert.", "evidence_required": [ "inference_input_validation_log showing request_id, schema_id used, validation_result (pass/reject), and rejection_reason for each inference request processed", "schema_registry_export listing all registered inference input schemas with schema_id, version, dataset_association, and last_updated timestamp", "rejection_event_record for any rejected inference inputs, including payload_hash, rejection_reason, and alert_dispatched_at timestamp", "signature_verification_log (where signed inference inputs are required) showing request_id, expected_signature, verification_result, and failure_action" ], "machine_tests": [ "Submit an inference request with a payload containing a structurally invalid field (wrong type, extra key not in schema) → assert the request is rejected with status 400 and error_code=schema_validation_failure before model execution", "Submit an inference request with a payload containing a prompt injection string pattern (e.g., 'Ignore previous instructions') → assert the tamper-detection filter flags the request and generates an alert with payload_hash", "Submit a well-formed, schema-compliant inference payload → assert the request passes validation and proceeds to model execution with validation_result=pass recorded in the log", "Modify a signed inference payload and submit → assert signature verification fails and the request is rejected with error_code=signature_mismatch" ], "human_review": [ "Review the schema registry to confirm all active inference input schemas are current, versioned, and reflect the latest authorized input structure for each deployed model", "Assess rejection logs from the prior period for patterns suggesting systematic adversarial probing or injection attempts, and evaluate whether detection rules are calibrated to current threat patterns", "Verify that the validation layer is positioned before model execution in the request processing chain and cannot be bypassed by internal API consumers or privileged callers" ], "blocking_effect": "blocks-runtime-action", "normative_status": "binding-law", "anti_patterns": [ "Performing schema validation as a logging-only step without actually blocking non-conformant requests from reaching the model, making tamper detection advisory instead of preventive", "Using a single permissive schema that accepts arbitrary additional properties, allowing injection payloads to pass validation alongside legitimate fields", "Validating inference inputs only at the API gateway boundary and not at the model-serving layer, creating a bypass path for internal service calls", "Relying on application-layer sanitization (string escaping, keyword blocking) as the primary tamper detection mechanism rather than structural schema enforcement", "Not versioning inference input schemas, making it impossible to detect when production deployments drift from the schema that was approved during model evaluation" ], "update_status": "current", "layer_code": "DI" }, { "id": "DI-03", "layer": "DI", "plane": "data", "name": "Cross-System Data Consistency Validation", "plain": "Data that flows across multiple systems in the AI data lifecycle must be reconciled at transfer boundaries to detect corruption, truncation, encoding errors, or synchronization failures before the data is used in AI training or inference.", "threat": { "tags": [ "cross-system-corruption", "data-truncation", "synchronization-drift" ], "desc": "Multi-system AI data pipelines introduce consistency gaps at every transfer boundary where data can be silently corrupted, truncated, or desynchronized. Encoding mismatches, partial writes, and replication lag accumulate into systematic training data errors that degrade model quality in ways that are difficult to attribute. Consistency failures in federated enterprise data architectures are especially dangerous because no single system has visibility across all the authoritative sources the AI is consuming." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 8", "title": "Data Integration and Interoperability — consistency across systems" }, { "id": "dcam", "section": "Capability 5.3", "title": "Data quality issues are identified and remediated" }, { "id": "iso_8000", "section": "ISO 8000-110", "title": "Master data exchange — syntax and semantic encoding" }, { "id": "cobit_2019", "section": "DSS06.05", "title": "Ensure traceability and accountability for information events" } ], "sources": [ { "id": "snowflake_horizon_data_governance_2024", "title": "Snowflake Horizon (Data Governance)", "authority": "Snowflake Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.snowflake.com/en/data-cloud/workloads/data-governance/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "snowflake_horizon", "relationship": "informative_reference", "rationale": "Establishes Snowflake Horizon (Data Governance) requirements informing the apeiris://data/controls/DI-03 Cross-System Data Consistency Validation control.", "reviewed_on": "2026-07-01" }, { "id": "databricks_unity_catalog_2024", "title": "Databricks Unity Catalog", "authority": "Databricks, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.databricks.com/en/data-governance/unity-catalog/index.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "databricks_unity_catalog_2024", "relationship": "informative_reference", "rationale": "Establishes Databricks Unity Catalog requirements informing the apeiris://data/controls/DI-03 Cross-System Data Consistency Validation control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Deploy reconciliation jobs at every inter-system data transfer boundary; compare record counts, hash totals, and key statistical profiles between source and destination; log discrepancies and block downstream AI pipeline stages on unresolved consistency failures.", "steps": [ "Map all inter-system data transfer boundaries in the AI data pipeline topology and document the expected consistency invariants (record count, hash total, statistical profile) for each boundary.", "Deploy automated reconciliation jobs at each boundary that run post-transfer and compare source-side and destination-side record counts, column-level null rates, and aggregate hash totals.", "Implement a consistency gate that blocks downstream AI pipeline stages (feature engineering, training, serving) from consuming data that failed reconciliation until the discrepancy is resolved or explicitly overridden with documented justification.", "Publish reconciliation results to a centralized consistency dashboard with trend analysis to detect systematic drift patterns across pipeline runs." ], "data_governance_officer": { "summary": "Cross-system consistency validation gives the data governance function visibility into data quality failures that span organizational and technical boundaries within the AI data supply chain.", "actions": [ "Define the consistency threshold policy specifying acceptable tolerance levels for reconciliation discrepancies by data classification tier.", "Require that all new cross-system data integrations entering the AI data pipeline are enrolled in reconciliation monitoring before go-live.", "Review consistency dashboard trends quarterly and escalate persistent high-discrepancy transfers to the data steward responsible for each boundary." ], "failure_signals": [ "Transfer boundaries without reconciliation coverage found during governance review.", "Consistency threshold breaches that were not escalated within the defined SLA.", "Downstream AI pipeline stages that consumed data with unresolved reconciliation failures." ] }, "data_engineer": { "summary": "Reconciliation jobs must be architected as first-class pipeline components, not afterthoughts, to ensure consistency checks run reliably and results are actionable.", "actions": [ "Build reconciliation logic into the data pipeline framework as a reusable stage that can be configured per boundary with appropriate consistency invariants.", "Implement hash-based total reconciliation in addition to record-count checks to catch substitution and truncation errors that preserve record counts.", "Create automated remediation paths for common consistency failure patterns (e.g., retry on network-induced partial writes) to reduce mean time to resolution." ], "failure_signals": [ "Reconciliation jobs that fail silently without producing a result record.", "Consistency gate bypasses without a logged override ticket.", "Reconciliation coverage gaps for new transfer boundaries added to the pipeline topology." ] }, "grc_auditor": { "summary": "Reconciliation run logs and consistency dashboard records are the primary evidence that cross-system data integrity is being maintained throughout the AI data supply chain.", "actions": [ "Pull the reconciliation run history for the audit period and verify that all registered transfer boundaries ran reconciliation jobs consistently.", "Sample discrepancy records and confirm that each was resolved or overridden with documented justification within the defined SLA.", "Verify that consistency gate bypass events were approved by an authorized data steward and are traceable to a documented exception." ], "metrics": [ "Transfer boundary reconciliation coverage: target 100% of registered boundaries.", "Mean time to resolve reconciliation discrepancies: target ≤ 8 hours for high-sensitivity data.", "Consistency gate bypass rate: target ≤ 1% of pipeline runs per quarter." ], "failure_signals": [ "Transfer boundaries without reconciliation runs in the audit period.", "Discrepancy records with no resolution ticket or override justification.", "Consistency gate bypass rate exceeding 1% threshold without escalation." ] }, "it_operations": { "summary": "Reconciliation infrastructure must be monitored for reliability because reconciliation job failures silently remove the consistency safety net from the AI pipeline.", "actions": [ "Monitor reconciliation job execution health and alert within 15 minutes of job failures.", "Maintain the consistency dashboard availability and ensure reconciliation result data is retained for the full audit retention period.", "Coordinate with data engineering on infrastructure changes that affect transfer boundaries to ensure reconciliation coverage is maintained." ], "failure_signals": [ "Reconciliation job infrastructure failures not detected within the SLA.", "Consistency dashboard data gaps caused by result store unavailability.", "Infrastructure changes that orphan reconciliation jobs from their transfer boundaries." ] }, "legal_counsel": { "summary": "When systems disagree about the same fact, the organization may be making representations it cannot support. Counsel cares that consequential and reported data reconciles, and that known divergences are disclosed or corrected where required.", "actions": [ "Identify datasets underpinning regulatory reports or contractual representations that must reconcile across systems.", "Review material unresolved divergences for disclosure or correction duties.", "Set documentation standards for the authoritative-source decisions reconciliation relies on." ], "failure_signals": [ "Regulatory filings derived from a replica that disagrees with the system of record.", "Known divergences on consequential data left unresolved past policy deadlines.", "No documented authority when two systems disagree about the same fact." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most enterprises perform record count reconciliation for ETL pipelines but lack hash-total and statistical profile reconciliation at AI-specific pipeline boundaries." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "federated-enterprise", "high-risk-sector", "multi-tenant" ], "implementers": [ "Data Engineering", "Data Governance Office", "Platform Engineering" ], "frameworks": [ { "framework": "dama_dmbok", "requirement_id": "Ch. 8", "fit": "direct", "rationale": "DAMA-DMBOK2 Chapter 8 (Data Integration and Interoperability) addresses keeping data consistent as it replicates and transforms across systems — the discipline DI-03 makes testable with reconciliation and drift detection.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "dcam", "requirement_id": "Capability 5.3", "fit": "direct", "rationale": "DCAM v2.2 capability 5.3 requires data quality issues to be identified and remediated through a defined process. Cross-system consistency validation feeds that process: detected divergences are DQ issues requiring root-cause analysis and remediation.", "normative_force": "certification-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_8000", "requirement_id": "ISO 8000-110", "fit": "direct", "rationale": "ISO 8000-110 specifies requirements for the exchange of master data among systems, including syntax and semantic-encoding requirements. Cross-system consistency validation verifies that master and characteristic data remain semantically equivalent as they replicate across systems.", "normative_force": "voluntary-standard", "source_version": "2022", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_2019", "requirement_id": "DSS06.05", "fit": "partial", "rationale": "COBIT 2019 practice DSS06.05 (ensure traceability and accountability for information events) supplies the audit trail cross-system consistency validation depends on: when replicas diverge, the information-event trail identifies where and when divergence was introduced. DSS06.03 concerns roles and access privileges.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "snowflake_horizon", "requirement_id": "Data Quality Monitoring (data metric functions)", "fit": "partial", "rationale": "Snowflake data metric functions (DMFs) attach quality metrics to tables and record results to an event table, supporting scheduled consistency checks between replicated datasets within Snowflake accounts. Snowflake does not market this as a cross-cloud monitoring capability.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "databricks_unity", "requirement_id": "Lakeflow Declarative Pipelines (DLT) expectations", "fit": "partial", "rationale": "Expectations in Lakeflow Declarative Pipelines (formerly Delta Live Tables) enforce row-level constraints as data replicates between layers, catching cross-system divergence inside Databricks pipelines. The capability belongs to the pipeline engine, not to Unity Catalog.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "owasp_ai_exchange", "requirement_id": "dataqualitycontrol", "fit": "supporting", "rationale": "DI-03 reconciles record counts, row hashes, and schema conformance at each transfer point and quarantines inconsistent data before downstream AI use.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "mitre", "requirement_id": "AML.M0007", "fit": "supporting", "direction": "control-supports-requirement", "rationale": "Control \"Data transferred across AI system boundaries must be reconciled at each transfer point…\" enacts ATLAS mitigation AML.M0007 Sanitize Training Data; OpenCRE crosswalks this control’s OWASP AI Exchange concept (dataqualitycontrol) to this mitigation.", "source_version": "v2026.06", "reviewed_on": "2026-07-08", "source_status": "authoritative", "mapping_confidence": "medium", "legal_status": "not-applicable", "control_readiness": "approved", "implementation_maturity": "developing", "assurance_result": "not-assessed", "evidence_status": "no-evidence", "normative_force": "industry-framework", "basis": "asserted", "relation": "equivalent_to", "correction": "ap38-opencre-enhancement 2026-07-08" } ], "canonical_id": "apeiris://data/controls/DI-03", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Data transferred across AI system boundaries must be reconciled at each transfer point using record counts, row hashes, and schema conformance checks, with any consistency failure triggering quarantine of the transferred dataset and blocking its use in downstream AI processing until the discrepancy is investigated and resolved.", "evidence_required": [ "reconciliation_report for each inter-system data transfer showing source_system, target_system, transfer_id, source_record_count, target_record_count, schema_match_result, and row_hash_comparison_result", "quarantine_event_record for transfers that failed reconciliation, including transfer_id, failure_type (count_mismatch/schema_violation/hash_deviation), quarantine_timestamp, and resolution_status", "transfer_boundary_map document identifying all data transfer points in the AI data pipeline where reconciliation checks are applied", "consistency_sla_record showing actual reconciliation pass rates per transfer boundary versus target SLA thresholds for the reporting period" ], "machine_tests": [ "Run a controlled data transfer between source and target systems with a 1-row deletion injected at the transfer boundary → assert reconciliation detects count mismatch and quarantines the target dataset", "Transfer a dataset with a modified column schema (one field renamed) → assert schema conformance check fails with error detail identifying the non-conformant field", "Execute a full pipeline run with no injected faults → assert all transfer boundaries produce reconciliation_report records with status=pass and zero quarantine events", "Attempt to use a quarantined dataset as input to a downstream pipeline stage → assert the pipeline stage rejects the input with error_code=dataset_quarantined and records the blocked_attempt" ], "human_review": [ "Review the transfer boundary map for completeness — verify all system integration points in the AI data lifecycle are identified and subject to reconciliation checks, with no unmonitored transfer paths", "Assess quarantine event records from the prior period to identify recurring discrepancy patterns that may indicate systematic encoding, synchronization, or transformation errors requiring architectural remediation", "Evaluate reconciliation SLA thresholds against actual data criticality and downstream impact — confirm that high-risk AI datasets have stricter tolerance thresholds than lower-sensitivity data" ], "blocking_effect": "requires-review", "normative_status": "certification-standard", "anti_patterns": [ "Performing reconciliation only on record counts without row-level hash comparison, which allows data corruption within individual records to pass count-based checks undetected", "Running reconciliation checks asynchronously after downstream pipeline stages have already begun processing, eliminating the quarantine window's protective effect", "Treating schema evolution (column additions, type changes) as automatically reconciliation-exempt without explicit change-management approval, creating a blind spot for schema injection attacks", "Storing reconciliation reports in the same mutable pipeline metadata store as pipeline run logs, making post-hoc alteration of reconciliation outcomes possible", "Applying reconciliation only to the final output of multi-hop pipelines rather than at each transfer boundary, missing corruption introduced at intermediate hops" ], "update_status": "current", "layer_code": "DI" }, { "id": "DI-04", "layer": "DI", "plane": "control", "name": "AI Output Data Integrity Verification", "plain": "AI-generated data must be evaluated for accuracy, internal consistency, and structural correctness before it is consumed by downstream systems, written to authoritative records, or used to drive automated decisions.", "threat": { "tags": [ "ai-output-hallucination", "downstream-contamination", "automated-decision-error" ], "desc": "AI models can produce outputs that are structurally valid but factually incorrect, internally inconsistent, or subtly malformed in ways that downstream systems accept and propagate as authoritative data. Without integrity verification at the output boundary, AI-generated errors compound through the data supply chain and become embedded in authoritative records. In agentic AI contexts, unverified outputs can trigger automated downstream actions with material business or legal consequences before any human reviews the data." }, "standard": [ { "id": "eu_ai_act", "section": "Art. 15", "title": "Accuracy, robustness and cybersecurity" }, { "id": "dama_dmbok", "section": "Ch. 13", "title": "Data Quality — quality of derived and output data" }, { "id": "gdpr", "section": "Art. 5(1)(d)", "title": "Accuracy principle for processed data" }, { "id": "cobit_2019", "section": "DSS06.04", "title": "Process errors and exceptions in output" } ], "sources": [ { "id": "anthropic_privacy_policy_2024", "title": "Anthropic Data Handling & Privacy Policy", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-09-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/privacy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_privacy_policy_2024", "relationship": "informative_reference", "rationale": "Establishes Anthropic Data Handling & Privacy Policy requirements informing the apeiris://data/controls/DI-04 AI Output Data Integrity Verification control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Define output quality contracts for each AI model specifying expected schema, value ranges, internal consistency rules, and confidence thresholds; validate all AI outputs against these contracts at the output boundary before downstream consumption; route low-confidence or contract-failing outputs to a human review queue.", "steps": [ "Define output quality contracts per model specifying: required output schema, acceptable value ranges, internal consistency rules (e.g., sum constraints, referential integrity), and minimum confidence scores.", "Deploy output validation middleware at the model serving layer that evaluates each inference response against the registered output contract and tags it with a validation verdict (pass, conditional, fail).", "Route conditional-verdicted outputs to a human-in-the-loop review queue and block failed outputs from downstream consumption; log all verdicts to the integrity audit log.", "Implement statistical drift detection on output distributions to detect gradual degradation in AI output quality that individual contract checks may not catch." ], "data_governance_officer": { "summary": "AI output integrity verification extends data governance accountability to AI-generated data, ensuring that AI outputs are held to the same accuracy standards as other enterprise data assets.", "actions": [ "Define the output quality contract policy specifying required elements and minimum confidence thresholds by model risk tier.", "Establish a human-in-the-loop review process for conditional-verdict outputs with defined SLAs based on the downstream decision stakes.", "Include AI output accuracy metrics in the data quality reporting provided to the data governance committee." ], "failure_signals": [ "AI models deployed to production without registered output quality contracts.", "Human review queue backlogs exceeding the defined SLA.", "Conditional-verdict outputs that bypassed the review queue and reached downstream systems." ] }, "data_engineer": { "summary": "Output contract registration and the validation middleware must be built as part of the model deployment process, not added retrospectively.", "actions": [ "Build the output validation middleware as a sidecar or gateway component in the model serving infrastructure.", "Maintain the output quality contract registry and create tooling for model teams to author and test contracts during the model development phase.", "Implement output distribution monitoring using statistical process control charts to detect drift in output quality over time." ], "failure_signals": [ "Models deployed to production without output contracts registered in the contract registry.", "Output validation middleware bypassed by direct model endpoint access.", "Distribution drift alerts that are not escalated to the model team within the defined SLA." ] }, "legal_counsel": { "summary": "AI output integrity verification is a critical control for legal defensibility when AI-generated data is used in decisions that affect individuals or carries regulatory obligations.", "actions": [ "Advise on which AI output categories require human review before downstream use under applicable law (GDPR Art. 22, EU AI Act, sector regulations).", "Ensure that output quality contract failure records are retained for the duration required to defend automated decision claims.", "Review the human-in-the-loop override process to confirm it satisfies meaningful human review requirements under GDPR Article 22 where applicable." ], "failure_signals": [ "AI-generated data used in automated decisions without evidence of output integrity verification.", "Output quality records not retained for the legally required period.", "Human review queue records that do not demonstrate genuine review rather than rubber-stamping." ] }, "grc_auditor": { "summary": "Output contract registry, validation verdict logs, and human review queue records are the primary evidence that AI output integrity is governed.", "actions": [ "Pull the output contract registry and verify all production models have registered contracts covering the required quality dimensions.", "Sample output validation logs and confirm that fail-verdict outputs were blocked from downstream consumption.", "Review human review queue records to confirm conditional-verdict outputs were reviewed within SLA and that reviewer decisions are documented." ], "metrics": [ "Output quality contract coverage of production models: target 100%.", "Fail-verdict output downstream consumption rate: target 0%.", "Human review queue SLA adherence rate: target ≥ 98%." ], "failure_signals": [ "Production models without output quality contracts.", "Evidence of fail-verdict outputs reaching downstream systems.", "Human review records that lack substantive review documentation." ] }, "it_operations": { "summary": "Output integrity verification runs in the serving path, so operations owns its availability and latency budget. A skipped verification step is a silent integrity gap.", "actions": [ "Operate output-verification services with the same availability targets as the AI systems they guard.", "Monitor verification coverage so every consequential output path runs its checks.", "Fail closed — or route to human review — when verification infrastructure is degraded." ], "failure_signals": [ "Output paths silently skipping verification during incidents.", "Verification service outages that default to pass-through.", "Coverage metrics absent for newly added output channels." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Most AI deployments lack formalized output quality contracts; validation is typically ad hoc and applied inconsistently across model types and deployment contexts." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "cloud-native", "eu-high-risk-ai" ], "implementers": [ "MLOps", "Data Engineering", "Legal Counsel", "Data Governance Office" ], "frameworks": [ { "framework": "eu_ai_act", "requirement_id": "Art. 15", "fit": "direct", "rationale": "EU AI Act Article 15 requires appropriate accuracy, robustness and cybersecurity for high-risk AI systems, with consistent performance against declared metrics. Verifying the integrity of AI outputs before downstream consumption is a direct technical measure supporting Art. 15 accuracy and robustness claims; Article 13 concerns transparency to deployers, not output integrity.", "normative_force": "binding-law", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "gdpr", "requirement_id": "Art. 5(1)(d)", "fit": "direct", "rationale": "GDPR Article 5(1)(d) requires that personal data be accurate and kept up to date, with all reasonable steps taken to ensure inaccurate data is erased or rectified without delay. When AI systems generate or modify data about individuals, the accuracy principle applies to those AI outputs, requiring that integrity verification controls prevent inaccurate AI-generated data from entering authoritative records. Demonstrating compliance with the accuracy principle for AI-generated personal data requires documented output validation evidence.", "normative_force": "binding-law", "source_version": "2018", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 13", "fit": "direct", "rationale": "DAMA-DMBOK2 Chapter 13 applies quality management to derived data as much as to source data; AI outputs are derived data whose integrity must be verified before downstream use.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "cobit_2019", "requirement_id": "DSS06.04", "fit": "partial", "rationale": "COBIT 2019 DSS06.04 requires that processing errors and output exceptions are managed with appropriate detective and corrective controls. The output validation middleware with verdict logging and human review escalation implements the exception management framework DSS06.04 prescribes. The statistical drift monitoring component implements the ongoing monitoring dimension that COBIT's data integrity management objective requires.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_pf", "requirement_id": "PR.DS-P6", "fit": "partial", "rationale": "NIST Privacy Framework PR.DS-P6 requires integrity checking mechanisms to verify information integrity. Output integrity verification extends integrity checking to AI-generated data before downstream consumption.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "anthropic_privacy", "requirement_id": "Usage Policy", "fit": "adjacent", "rationale": "Anthropic's Usage Policy sets conditions on how model outputs may be used, including human-oversight requirements in consequential domains. It illustrates vendor-side expectations that deployers verify and validate AI outputs rather than consume them unexamined.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "owasp_ai_exchange", "requirement_id": "runtimemodeliointegrity", "fit": "supporting", "rationale": "DI-04 verifies structural and factual integrity of AI output before it is written to authoritative systems, protecting model output integrity at runtime.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://data/controls/DI-04", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every AI-generated data artifact must pass structural correctness, factual consistency, and confidence threshold checks before it is written to an authoritative record system or consumed by a downstream automated process, with below-threshold outputs routed to human review rather than passed directly to downstream consumers.", "evidence_required": [ "output_validation_log showing output_id, model_id, validation_checks_applied, confidence_score, threshold_applied, validation_result (pass/review_required/reject), and disposition_action for each AI output evaluated", "human_review_queue_record for outputs routed to human review, including output_id, routing_reason, reviewer_id, review_outcome, and reviewed_at timestamp", "downstream_consumption_gate_log showing that no output with validation_result=reject was written to authoritative record systems or consumed by automated downstream pipelines", "accuracy_evaluation_report covering the period, showing the distribution of confidence scores and the proportion of outputs requiring human review versus automated pass" ], "machine_tests": [ "Submit a known-hallucinated AI output (factually inconsistent with reference data) to the validation pipeline → assert consistency check fails and the output is routed to human review queue rather than passed to downstream consumers", "Submit an AI output with confidence_score below the defined threshold → assert the output is blocked from direct downstream consumption and a human_review_queue record is created", "Submit a structurally malformed AI output (missing required fields, wrong data types) → assert structural validation fails with error detail and the output is rejected before reaching any authoritative record system", "Simulate a validation service outage and attempt to write an AI output to an authoritative record → assert the write is blocked with error_code=validation_service_unavailable rather than defaulting to pass" ], "human_review": [ "Review the human_review_queue for patterns in routing reasons — assess whether specific model configurations or input types are consistently producing outputs that require human intervention, indicating a calibration or training issue", "Verify that the confidence threshold configuration is reviewed and updated when model versions change, and that thresholds are set per output type based on downstream impact severity rather than as a single global value", "Assess the downstream_consumption_gate_log for any instances where outputs bypassed validation — investigate root cause for each bypass event and confirm remediation" ], "blocking_effect": "blocks-runtime-action", "normative_status": "binding-law", "anti_patterns": [ "Using AI model confidence scores as the sole validation mechanism without structural or consistency checks, treating high model confidence as a proxy for factual correctness", "Routing below-threshold outputs to a human review queue but allowing downstream automated pipelines to proceed in parallel on the same output before review is complete", "Applying output validation only to externally visible AI responses and not to AI outputs that feed internal automated decision pipelines, where downstream contamination risk is highest", "Setting a single universal confidence threshold across all output types regardless of the downstream consequence of error — high-impact outputs (medical, financial, legal) require stricter thresholds", "Allowing the validation service to fail open so that if the validation pipeline is unavailable, AI outputs are passed to downstream consumers rather than blocked" ], "update_status": "current", "layer_code": "DI" }, { "id": "DI-05", "layer": "DI", "plane": "data", "name": "Database and Storage Integrity Monitoring", "plain": "Storage systems holding AI training data, model weights, and inference caches must be monitored for storage-level corruption, bit rot, unauthorized writes, and structural degradation that could silently compromise AI system behavior.", "threat": { "tags": [ "storage-corruption", "bit-rot", "model-weight-tampering" ], "desc": "Storage-level corruption in AI training repositories, model weight stores, and inference caches can silently degrade AI system behavior in ways that are indistinguishable from model drift or performance degradation. Model weight files are particularly high-value targets for persistent adversaries because even small bit-flips can alter model behavior in predictable ways while bypassing application-level integrity checks. Inference cache corruption can cause an AI system to serve corrupted cached responses at scale before detection." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 6", "title": "Data Storage and Operations — storage integrity" }, { "id": "cobit_2019", "section": "DSS04.07", "title": "Manage storage integrity and backup verification" }, { "id": "iso_8000", "section": "ISO 8000-61", "title": "Data quality management — process reference model" }, { "id": "eu_ai_act", "section": "Art. 9(2)(a)", "title": "Risk management — identification and analysis of foreseeable risks" } ], "sources": [ { "id": "aws_lake_formation_macie_2024", "title": "AWS data governance services (Lake Formation, Macie, Glue, S3)", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/lake-formation/latest/dg/what-is-lake-formation.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_lake_formation_macie_2024", "relationship": "informative_reference", "rationale": "Establishes AWS data governance services (Lake Formation, Macie, Glue, S3) requirements informing the apeiris://data/controls/DI-05 Database and Storage Integrity Monitoring control.", "reviewed_on": "2026-07-02" }, { "id": "google_dataplex_bigquery_2024", "title": "Google Cloud Dataplex & BigQuery", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://cloud.google.com/dataplex/docs/introduction", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "google_dataplex_bigquery_2024", "relationship": "informative_reference", "rationale": "Establishes Google Cloud Dataplex & BigQuery requirements informing the apeiris://data/controls/DI-05 Database and Storage Integrity Monitoring control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Deploy block-level and file-level integrity monitoring on all storage systems holding AI assets; compute and store checksums for model weight files, training dataset partitions, and inference cache segments at write time; run scheduled verification scans; alert on detected corruption before affected assets are used.", "steps": [ "Enumerate all storage systems (object stores, block storage, distributed file systems) holding AI training data, model weights, feature stores, and inference caches and register them in the storage integrity inventory.", "Implement write-time checksum computation and storage for all registered AI asset types using SHA-256; store checksums in a separate integrity store with write-once access controls.", "Deploy scheduled storage integrity scans that recompute checksums and compare against the stored baseline; configure scans to run at a frequency appropriate to the change rate and value of each storage tier.", "Integrate storage integrity alerts into the model serving pipeline so that model loading is gated on a clean integrity scan result, preventing corrupt model weights from being loaded into production serving." ], "data_governance_officer": { "summary": "Storage integrity monitoring extends data governance accountability to the physical persistence layer where AI assets are held, ensuring that governance policies cover the full custody chain.", "actions": [ "Require that all new AI asset storage systems are enrolled in integrity monitoring before being approved for production use.", "Define the integrity scan frequency policy by storage tier and asset type based on change rate and risk classification.", "Include storage integrity scan results and anomaly trends in the data governance reporting dashboard." ], "failure_signals": [ "AI asset storage systems not enrolled in integrity monitoring discovered during governance review.", "Integrity scan frequency below policy requirements for high-risk storage tiers.", "Model weight files loaded into production serving without a current clean integrity scan result." ] }, "data_engineer": { "summary": "Storage integrity monitoring must be integrated into the model training and serving infrastructure to ensure that integrity scans are part of the asset lifecycle rather than an external check.", "actions": [ "Instrument the model training pipeline to register model weight file checksums in the integrity store at the end of each training run.", "Implement a model loading gate in the serving infrastructure that verifies model weight checksums against the integrity store before loading.", "Configure inference cache invalidation logic to clear cache segments that fail integrity verification rather than serving corrupt cached responses." ], "failure_signals": [ "Model training runs that complete without checksum registration in the integrity store.", "Model serving infrastructure that loads model weights without integrity verification.", "Inference cache integrity scan failures not triggering cache invalidation." ] }, "grc_auditor": { "summary": "The storage integrity inventory, scan results, and model loading gate logs are the primary evidence that storage-level integrity is governed for AI assets.", "actions": [ "Verify the storage integrity inventory is current and covers all production AI asset storage systems.", "Pull integrity scan results for the audit period and confirm that scans ran at the required frequency with no unexplained gaps.", "Sample model loading events and verify that each was preceded by a clean integrity scan result within the defined freshness window." ], "metrics": [ "AI asset storage enrollment rate: target 100% of registered storage systems.", "Integrity scan frequency compliance: target 100% of scans completed within the policy-defined window.", "Model loading gate enforcement rate: target 100% of production model loads preceded by clean integrity scan." ], "failure_signals": [ "Storage systems not enrolled in the integrity inventory.", "Scan frequency gaps for high-risk storage tiers not explained by documented maintenance windows.", "Model loading events without a corresponding integrity scan record in the freshness window." ] }, "it_operations": { "summary": "Operations teams are responsible for maintaining the storage infrastructure, integrity store, and scan job health that this control depends on.", "actions": [ "Monitor integrity scan job execution and alert within 15 minutes of job failures.", "Maintain the integrity store with write-once access controls and redundant storage to prevent the integrity baseline itself from being corrupted.", "Coordinate with the data engineering team on storage system provisioning to ensure integrity monitoring enrollment is part of the provisioning checklist." ], "failure_signals": [ "Integrity store availability incidents that prevent scan jobs from running.", "Storage systems provisioned without integrity monitoring enrollment.", "Write-once access controls on the integrity store found to be misconfigured during access review." ] }, "legal_counsel": { "summary": "Silent storage corruption undermines records-retention and evidence obligations. Counsel needs assurance that data retained to meet legal duties remains intact and provably unaltered for its full retention period.", "actions": [ "Confirm storage-integrity monitoring covers records retained under legal and regulatory retention duties.", "Require integrity verification before retained records are produced in audits or litigation.", "Assess legal exposure when corruption is found in records the organization was obliged to preserve." ], "failure_signals": [ "Legally retained records failing integrity checks at production time.", "Backup-verification gaps on systems holding compliance records.", "Corruption incidents on retained data closed without assessing preservation-duty breach." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most enterprises rely on storage platform-level checksums (e.g., S3 ETag) but lack application-level integrity monitoring specifically scoped to AI asset storage with model loading gates." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "federated-enterprise" ], "implementers": [ "MLOps", "IT Operations", "Platform Engineering", "Data Engineering" ], "frameworks": [ { "framework": "dama_dmbok", "requirement_id": "Ch. 6", "fit": "direct", "rationale": "DAMA-DMBOK2 Chapter 6 (Data Storage and Operations) covers database operations including backup verification and storage health — the operational foundation DI-05 monitors continuously.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "cobit_2019", "requirement_id": "DSS04.07", "fit": "direct", "rationale": "COBIT 2019 DSS04.07 requires that backup and storage integrity are verified through regular testing and that verification results are recorded and actioned. The scheduled integrity scan and gate mechanism directly implements DSS04.07's verification requirement for AI asset storage. The model loading gate that enforces integrity scan freshness satisfies COBIT's requirement that integrity failures prevent affected assets from entering operational use.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "iso_8000", "requirement_id": "ISO 8000-61", "fit": "partial", "rationale": "ISO 8000-61's process reference model includes data quality assurance and control processes that presuppose reliable storage of the data under management; storage integrity monitoring provides that operational foundation.", "normative_force": "voluntary-standard", "source_version": "2022", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "eu_ai_act", "requirement_id": "Art. 9(2)(a)", "fit": "partial", "rationale": "EU AI Act Article 9(2)(a) requires identification and analysis of the known and reasonably foreseeable risks the high-risk AI system can pose. Storage-integrity failures — silent corruption of training corpora, model weights or evaluation sets — are a foreseeable risk class this analysis must cover; continuous storage integrity monitoring is the corresponding mitigation.", "normative_force": "binding-law", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "aws_lake_formation", "requirement_id": "Amazon S3 — Object Integrity and Checksums", "fit": "partial", "rationale": "Amazon S3 — a storage service distinct from Lake Formation — provides object checksum algorithms (SHA-256, CRC32/CRC32C) validated on upload and retrievable for integrity re-verification, plus S3 Object Lock for immutability. These storage-layer features implement DI-05's integrity monitoring on AWS.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "google_dataplex", "requirement_id": "Cloud Storage — Object Integrity and Retention Lock", "fit": "partial", "rationale": "Google Cloud Storage — a distinct service from Dataplex — provides object checksums (CRC32C/MD5) validated on write and retrievable for re-verification, plus bucket retention lock for immutability. These storage-layer capabilities implement DI-05's integrity monitoring on Google Cloud.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://data/controls/DI-05", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Storage systems holding AI training datasets, model weights, and inference caches must produce integrity check results at least daily, with any detected storage-level corruption, unauthorized write, or structural degradation triggering immediate quarantine of affected data and an alert to the responsible operations and data governance teams.", "evidence_required": [ "storage_integrity_scan_log showing storage_system_id, scan_timestamp, objects_checked, anomalies_detected (count and type), and scan_result (clean/anomaly) for each scheduled scan", "quarantine_event_record for any storage anomaly, including object_id, storage_system_id, anomaly_type (bit-rot/unauthorized-write/structural-degradation/weight-hash-mismatch), detected_at, and remediation_status", "model_weight_hash_manifest showing expected SHA-256 hash per model artifact version with last_verified_at timestamp confirming hash verification against stored artifacts", "storage_monitoring_coverage_report identifying all AI-relevant storage systems under integrity monitoring and their configured scan frequency, confirming 100% coverage of in-scope systems" ], "machine_tests": [ "Flip a single bit in a test model weight file stored in the monitored storage system → assert the next integrity scan detects the hash deviation and generates a quarantine_event_record within one scan cycle", "Write a test file to a monitored storage path using a non-authorized service account → assert the unauthorized write is detected in the integrity scan log with object_id and writer_identity fields populated", "Check storage_integrity_scan_log for all in-scope storage systems → assert every system has at least one scan_result record within the last 24 hours", "Verify the model_weight_hash_manifest against current stored artifacts → assert all model artifact hashes match expected values and last_verified_at is within the defined verification window" ], "human_review": [ "Review storage_monitoring_coverage_report to confirm all AI-relevant storage systems — including archival and cold-tier stores — are enrolled in integrity monitoring, and assess whether scan frequency is appropriate for each system's risk classification", "Assess quarantine event records for patterns of recurring storage degradation in specific systems or storage regions, which may indicate hardware failure, misconfiguration, or targeted attack", "Verify that model weight hash manifests are updated when new model versions are deployed and that the manifest update process requires authorization from both the ML engineering and data governance teams" ], "blocking_effect": "advisory", "normative_status": "industry-framework", "anti_patterns": [ "Monitoring only active training data stores and omitting archival or cold-tier storage where older model weights and historical datasets reside, leaving superseded but recoverable artifacts unprotected", "Relying on storage-provider-native integrity checks (e.g., object ETags) as the sole verification mechanism without computing independent cryptographic hashes of model artifacts at a layer outside the storage provider's control", "Scanning storage for integrity on a weekly or monthly schedule for AI systems in production inference, leaving multi-day windows where tampered model weights could be loaded into serving infrastructure", "Not maintaining a hash manifest for model weight artifacts independently of the model registry, making it impossible to verify integrity if the model registry itself is compromised", "Treating storage integrity alerts as low-priority operational tickets rather than security incidents requiring immediate quarantine and root-cause investigation" ], "update_status": "current", "layer_code": "DI" }, { "id": "DI-06", "layer": "DI", "plane": "control", "name": "Model Memorization Risk Assessment and Output Leakage Prevention", "plain": "AI models trained on sensitive or confidential data must be assessed for memorization risk and monitored at inference time to prevent the extraction of memorized confidential content through adversarial queries or routine use.", "threat": { "tags": [ "model-memorization", "training-data-extraction", "sensitive-data-leakage-via-ai" ], "desc": "Language models and deep learning systems can memorize and verbatim reproduce sensitive training data including PII, credentials, proprietary documents, and medical records in response to adversarial extraction queries. Memorization risk is highest when training data contains repeated unique sequences or when models are fine-tuned on small confidential datasets. Extraction attacks are difficult to detect at inference time without runtime monitoring because the outputs appear to be legitimate model responses rather than security events." }, "standard": [ { "id": "gdpr", "section": "Art. 5(1)(f)", "title": "Integrity and confidentiality of personal data in AI training" }, { "id": "eu_ai_act", "section": "Art. 10(5)", "title": "Prohibition on processing sensitive data without safeguards" }, { "id": "nist_pf", "section": "CT.DP-P2", "title": "Data processed to limit identification of individuals" }, { "id": "iso_27701", "section": "§7.2.5", "title": "Privacy impact assessment" } ], "sources": [ { "id": "anthropic_privacy_policy_2024", "title": "Anthropic Data Handling & Privacy Policy", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-09-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/privacy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_privacy_policy_2024", "relationship": "informative_reference", "rationale": "Establishes Anthropic Data Handling & Privacy Policy requirements informing the apeiris://data/controls/DI-06 Model Memorization Risk Assessment and Output Leakage Prevention control.", "reviewed_on": "2026-07-01" }, { "id": "microsoft_purview_data_governance_2024", "title": "Microsoft Purview (Data Governance)", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/purview/purview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_purview_data_governance_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Purview (Data Governance) requirements informing the apeiris://data/controls/DI-06 Model Memorization Risk Assessment and Output Leakage Prevention control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Perform memorization risk assessment before deploying models trained on sensitive data using membership inference tests; implement output scanning for sensitive data pattern leakage at the inference gateway; establish a continuous red-team extraction testing program; apply training-time mitigations (differential privacy, data deduplication) for high-risk cases.", "steps": [ "Perform pre-deployment memorization risk assessment for all models trained on data classified at confidential or above, using membership inference attack simulations and canary record extraction tests.", "Deploy an output scanning layer at the inference gateway that screens AI responses for known sensitive data patterns (PII regex, credential formats, proprietary document signatures) and redacts or blocks leaking responses.", "Establish a continuous model extraction red-team program that tests production models for memorization leakage using novel prompt variants not covered by static output filters.", "Apply training-time memorization mitigations for high-risk models, including training data deduplication, differential privacy noise injection (epsilon ≤ 8 for high-sensitivity datasets), and minimum example count thresholds per unique record." ], "data_governance_officer": { "summary": "Memorization risk is a data governance failure that occurs when sensitive data governance controls are not extended to model training processes, creating a persistent data exposure vector in deployed models.", "actions": [ "Define the memorization risk assessment requirement in the AI data governance policy, specifying which data classifications trigger mandatory pre-deployment assessment.", "Require that memorization risk assessment results are documented and reviewed before sign-off on model deployment for models trained on confidential data.", "Establish a data subject rights process for handling extraction incident responses when a memorization leakage event involves personal data." ], "failure_signals": [ "Models deployed on confidential training data without documented memorization risk assessment.", "Output scanning coverage below 100% of inference endpoints for models with high memorization risk scores.", "Memorization leakage events not reported through the data breach notification process where personal data is involved." ] }, "data_engineer": { "summary": "Training-time mitigations and output scanning implementation are the primary engineering responsibilities for this control.", "actions": [ "Implement training data deduplication pipelines that remove near-duplicate records from sensitive training datasets before model training begins.", "Integrate differential privacy libraries (e.g., TensorFlow Privacy, Opacus) into the training pipeline for models classified as high memorization risk.", "Build and maintain the sensitive data pattern library used by the output scanning layer, updated whenever new sensitive data classes are identified." ], "failure_signals": [ "Training pipelines for sensitive models that do not include deduplication as a mandatory stage.", "Sensitive data pattern library not updated within the review cycle.", "Output scanning bypass discovered through red-team testing." ] }, "legal_counsel": { "summary": "Model memorization leakage involving personal data constitutes a data breach under GDPR and similar regulations, requiring legal involvement in both preventive controls and incident response.", "actions": [ "Advise on whether memorization risk assessment findings trigger any immediate regulatory disclosure obligations.", "Ensure the incident response plan for memorization leakage events includes breach notification timelines and regulator communication templates.", "Review differential privacy parameters to confirm they satisfy any applicable legal standards for pseudonymization or anonymization of personal data used in AI training." ], "failure_signals": [ "Memorization leakage events involving personal data not reported to legal counsel within the defined escalation window.", "No breach notification process defined specifically for AI output data leakage scenarios.", "Differential privacy parameters used without legal review of their adequacy for anonymization claims." ] }, "grc_auditor": { "summary": "Memorization risk assessment reports, output scanning coverage logs, and red-team test results are the primary evidence that this control is operating.", "actions": [ "Verify that all models trained on confidential data have a current memorization risk assessment report in the model governance record.", "Pull output scanning coverage logs and confirm 100% of inference requests through high-risk model endpoints are scanned.", "Review red-team test results and confirm that any extraction findings were remediated before the next production deployment of the affected model." ], "metrics": [ "Pre-deployment memorization assessment coverage: target 100% of models trained on confidential data.", "Output scanning coverage for high-risk model endpoints: target 100%.", "Red-team extraction test cadence: target quarterly for high-risk models." ], "failure_signals": [ "Models in production without memorization risk assessment documentation.", "Output scanning coverage gaps for high-risk model endpoints.", "Red-team extraction findings not remediated before subsequent model deployment." ] }, "it_operations": { "summary": "Memorization safeguards translate into operational controls: output filters in the serving path, scheduled scanning jobs, and rate limits on extraction-shaped query patterns.", "actions": [ "Deploy and monitor output-leakage filters — PII detectors, canary checks — in model serving infrastructure.", "Run scheduled memorization probes against production models and track results per model version.", "Rate-limit and alert on query patterns consistent with training-data extraction." ], "failure_signals": [ "Serving endpoints running without leakage filters after redeployment.", "Scheduled memorization probes not executed for a released model version.", "Extraction-pattern alerts with no response runbook." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Memorization risk assessment is an emerging practice; most enterprises are unaware of the scope of memorization exposure in fine-tuned models trained on proprietary or sensitive data." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "cloud-native" ], "implementers": [ "MLOps", "Legal Counsel", "Data Engineering", "Security Engineering" ], "frameworks": [ { "framework": "gdpr", "requirement_id": "Art. 5(1)(f)", "fit": "direct", "rationale": "GDPR Article 5(1)(f) requires that personal data be processed in a manner that ensures appropriate security, including protection against unauthorized disclosure. When personal data is included in AI training sets, the model becomes a persistent vector for unauthorized disclosure via memorization extraction attacks. This control's combination of risk assessment, training-time mitigation, and runtime output scanning implements the technical measures required to comply with the integrity and confidentiality principle for AI training contexts.", "normative_force": "binding-law", "source_version": "2018", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "eu_ai_act", "requirement_id": "Art. 10(5)", "fit": "direct", "rationale": "EU AI Act Article 10(5) prohibits processing special categories of personal data in AI training without appropriate safeguards and requires that such processing be strictly necessary and subject to technical measures to protect data subjects. Differential privacy and output scanning are among the technical safeguards required by this provision for high-risk AI systems trained on sensitive personal data. Pre-deployment memorization assessment provides evidence that the required safeguards were evaluated and implemented.", "normative_force": "binding-law", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_pf", "requirement_id": "CT.DP-P2", "fit": "direct", "rationale": "NIST Privacy Framework CT.DP-P2 requires data to be processed to limit the identification of individuals (e.g., de-identification privacy techniques). Memorization risk assessment and output leakage prevention verify that trained models do not defeat those disassociation measures at inference time.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_27701", "requirement_id": "§7.2.5", "fit": "partial", "rationale": "ISO/IEC 27701:2019 §7.2.5 requires a privacy impact assessment wherever new or changed processing of PII may result in privacy risk. Training an AI model on personal data is such processing; memorization risk assessment is the AI-specific privacy risk analysis this clause calls for. §7.4.8 concerns disposal, not risk assessment.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "anthropic_privacy", "requirement_id": "Privacy Policy", "fit": "adjacent", "rationale": "Anthropic's Privacy Policy describes at document level how user data is handled with respect to model training, including commitments about when conversation data is and is not used for training. It is a reference point for the training-data privacy commitments DI-06 requires organizations to make and verify — not a technical memorization-safeguards specification.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "microsoft_purview", "requirement_id": "Microsoft Purview — Sensitive Information Types and DLP", "fit": "partial", "rationale": "Microsoft Purview's sensitive information type library provides a comprehensive set of data patterns (PII, credentials, financial data) that can populate the output scanning layer's detection library for AI inference responses. Purview's data loss prevention policies can be extended to AI output channels, providing a governance framework for managing AI output leakage risk on Microsoft platforms. Integration with Azure OpenAI Service enables Purview-based output scanning for Azure-hosted AI inference endpoints.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "owasp_ai_exchange", "requirement_id": "sensitiveoutputhandling", "fit": "direct", "rationale": "DI-06 runs inference-time filters that detect and block verbatim reproduction of confidential training data in model responses, directly handling sensitive output to prevent leakage.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "mitre", "requirement_id": "AML.M0020", "fit": "supporting", "direction": "control-supports-requirement", "rationale": "Control \"AI models trained on sensitive or confidential data must undergo memorization risk…\" enacts ATLAS mitigation AML.M0020 Generative AI Guardrails; OpenCRE crosswalks this control’s OWASP AI Exchange concept (sensitiveoutputhandling) to this mitigation.", "source_version": "v2026.06", "reviewed_on": "2026-07-08", "source_status": "authoritative", "mapping_confidence": "medium", "legal_status": "not-applicable", "control_readiness": "approved", "implementation_maturity": "developing", "assurance_result": "not-assessed", "evidence_status": "no-evidence", "normative_force": "industry-framework", "basis": "asserted", "relation": "equivalent_to", "correction": "ap38-opencre-enhancement 2026-07-08" } ], "canonical_id": "apeiris://data/controls/DI-06", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "AI models trained on sensitive or confidential data must undergo memorization risk assessment before production deployment and must have inference-time output filters active that detect and block verbatim or near-verbatim reproduction of confidential training data in model responses.", "evidence_required": [ "memorization_risk_assessment_report for each model version, including model_id, training_data_sensitivity_classification, extraction_attack_tests_conducted, verbatim_reproduction_rate, and risk_level assigned", "inference_output_filter_configuration_record showing filter_id, filter_type (regex/embedding-similarity/PII-scanner), sensitivity_threshold, and models_protected for each active output filter", "output_filter_event_log showing detection events where inference outputs triggered filters, including request_id, model_id, filter_id, matched_pattern_type, and action_taken (blocked/redacted/flagged)", "differential_privacy_parameter_record (where differential privacy is applied during training) showing epsilon, delta, noise_mechanism, and dataset_scope for each trained model" ], "machine_tests": [ "Submit extraction attack prompts (e.g., 'Repeat the following text exactly: [known training data phrase]') against a deployed model → assert the inference output filter blocks or redacts verbatim reproduction of sensitive training data", "Run the memorization extraction test suite (canary string recall, membership inference probes) against the model pre-deployment → assert verbatim_reproduction_rate is below the defined threshold before deployment approval is granted", "Disable the inference output filter for a test model endpoint and submit a known extraction probe → assert the probe produces verbatim recall, confirming the filter is effective when active", "Query the inference output filter event log for the past 7 days → assert all models handling sensitive training data have at least one scan record, confirming filter activity monitoring is operational" ], "human_review": [ "Review the memorization risk assessment report for each model version prior to production deployment approval — assess whether extraction attack test coverage is sufficient for the sensitivity level of the training data and the model's deployment context", "Evaluate the inference output filter configuration for adequacy against current adversarial extraction techniques — confirm filters cover both exact-match and semantic-similarity extraction patterns, not only verbatim string matching", "Review differential privacy parameter choices (epsilon, delta) against current regulatory guidance on acceptable privacy loss bounds for the data types involved, and assess whether parameters were selected based on principled threat modeling or default settings" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Deploying AI models trained on sensitive data to production without conducting a memorization risk assessment, relying on the model provider's general safety claims rather than model-specific extraction testing", "Using only keyword-based or regex output filters to prevent training data extraction, which adversarial prompts can circumvent through paraphrasing, encoding variations, or semantic reformulation", "Treating memorization risk as a one-time pre-deployment check rather than an ongoing concern — model behavior can change with fine-tuning, and new extraction techniques may bypass filters that were effective at initial deployment", "Applying differential privacy only at training time without inference-time output filtering, assuming training-time privacy guarantees are sufficient to prevent all extraction attacks on deployed models", "Conflating PII redaction in training data with memorization risk elimination — de-identified training data can still be reconstructed through membership inference or quasi-identifier recombination in model outputs" ], "update_status": "current", "layer_code": "DI" }, { "id": "DI-07", "layer": "DI", "plane": "lifecycle", "name": "Data Integrity Incident Response", "plain": "The organization must have defined procedures for detecting, investigating, containing, and resolving data corruption or tampering incidents that affect AI training data, model assets, or inference pipelines, with documented recovery and post-incident review processes.", "threat": { "tags": [ "uncontained-corruption", "incident-response-failure", "recovery-delay" ], "desc": "Without a defined integrity incident response procedure, corruption or tampering events in AI data assets cascade before they are contained, leading to contaminated model retraining, poisoned downstream decisions, and loss of the forensic evidence needed to determine the root cause and scope of the breach. Prolonged mean time to containment in AI data integrity incidents directly correlates with the scope of downstream model re-work required and the regulatory reporting obligations triggered." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 13", "title": "Data Quality — issue management" }, { "id": "cobit_2019", "section": "DSS02.07", "title": "Track status and produce reports" }, { "id": "eu_ai_act", "section": "Art. 73", "title": "Reporting of serious incidents" }, { "id": "dcam", "section": "Capability 5.3", "title": "Data quality issues are identified and remediated" } ], "sources": [ { "id": "dama_dmbok", "title": "DAMA-DMBOK 2nd Ed", "authority": "DAMA International", "source_type": "standard", "normative_force": "best-practice", "version": "2 (2nd Edition, Revised 2024)", "published_on": "2017-07-05", "retrieved_on": "2026-07-02", "canonical_url": "https://www.dama.org/cpages/body-of-knowledge", "license": "proprietary", "status": "current", "flagship": false, "source_id": "dama_dmbok", "relationship": "informative_reference", "rationale": "Establishes DAMA-DMBOK 2nd Ed Ch. 13 requirements informing the apeiris://data/controls/DI-07 Data Integrity Incident Response control.", "reviewed_on": "2026-07-02" }, { "id": "cobit_2019", "title": "COBIT 2019", "authority": "ISACA", "source_type": "standard", "normative_force": "industry-framework", "version": "2019", "published_on": "2018-11-12", "retrieved_on": "2026-07-02", "canonical_url": "https://www.isaca.org/resources/cobit", "license": "proprietary", "status": "current", "flagship": false, "source_id": "cobit_2019", "relationship": "informative_reference", "rationale": "Establishes COBIT 2019 DSS02.07 requirements informing the apeiris://data/controls/DI-07 Data Integrity Incident Response control.", "reviewed_on": "2026-07-02" }, { "id": "eu_ai_act", "title": "EU AI Act", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2024/1689", "published_on": "2024-07-12", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2024/1689/oj/eng", "license": "open-access", "status": "current", "flagship": false, "source_id": "eu_ai_act", "relationship": "normative_requirement", "rationale": "Establishes EU AI Act Art. 73 requirements informing the apeiris://data/controls/DI-07 Data Integrity Incident Response control.", "reviewed_on": "2026-07-02" }, { "id": "dcam", "title": "DCAM v2.2", "authority": "EDM Council", "source_type": "standard", "normative_force": "industry-framework", "version": "2.2", "published_on": "2022-01-01", "retrieved_on": "2026-07-02", "canonical_url": "https://edmcouncil.org/frameworks/dcam/", "license": "proprietary", "status": "current", "flagship": false, "source_id": "dcam", "relationship": "informative_reference", "rationale": "Establishes DCAM v2.2 Capability 5.3 requirements informing the apeiris://data/controls/DI-07 Data Integrity Incident Response control.", "reviewed_on": "2026-07-02" }, { "id": "gdpr", "title": "GDPR", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2016/679", "published_on": "2016-04-27", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj", "license": "open-access", "status": "current", "flagship": false, "source_id": "gdpr", "relationship": "normative_requirement", "rationale": "Establishes GDPR Art. 5(1)(f) requirements informing the apeiris://data/controls/DI-07 Data Integrity Incident Response control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Maintain a data integrity incident response playbook with defined triggers, roles, escalation paths, containment procedures, forensic preservation steps, and recovery gates; integrate the playbook with the broader organizational incident response process; conduct post-incident reviews and feed findings back into preventive control improvements.", "steps": [ "Author and maintain a Data Integrity Incident Response Playbook covering: incident classification (minor corruption, major corruption, active tampering), RACI for response roles, detection-to-declaration SLA, quarantine and containment procedures, forensic evidence preservation, communication protocols, and recovery gates.", "Integrate data integrity incident triggers from the DI-01 through DI-06 alerting systems into the incident management platform with automatic ticket creation and priority routing based on affected asset classification.", "Define and document recovery gates that require cryptographic verification of restored data assets against pre-incident baselines before AI pipeline services are restored from quarantine.", "Conduct a structured post-incident review within 10 business days of any declared data integrity incident, producing a root-cause analysis and preventive control improvement recommendations that are tracked to closure." ], "data_governance_officer": { "summary": "The Data Governance Office is the incident response authority for data integrity incidents affecting AI data assets, responsible for declaring incidents, coordinating cross-functional response, and overseeing regulatory reporting.", "actions": [ "Own and maintain the Data Integrity Incident Response Playbook, reviewing it at least annually and after any declared incident.", "Serve as incident commander for data integrity incidents declared at the major or active-tampering classification level.", "Coordinate with Legal Counsel on regulatory notification requirements for incidents involving personal data or high-risk AI systems under the EU AI Act." ], "failure_signals": [ "Data integrity incidents declared without the playbook being followed.", "Incident response playbook last reviewed more than 12 months ago.", "Post-incident review findings not tracked to closure within the defined window." ] }, "data_engineer": { "summary": "Data engineering teams are responsible for the technical execution of containment, forensic preservation, and data restoration procedures during an integrity incident.", "actions": [ "Execute the quarantine and containment steps in the playbook including pipeline suspension, affected asset isolation, and SIEM evidence package creation.", "Perform data restoration from verified clean backups following the recovery gate verification procedure.", "Document the technical timeline and root cause findings for the post-incident review report." ], "failure_signals": [ "Containment procedures not executed within the playbook-defined SLA.", "Data restoration performed without cryptographic verification against pre-incident baselines.", "Forensic evidence not preserved before system remediation begins." ] }, "legal_counsel": { "summary": "Legal counsel must be engaged early in data integrity incidents to assess regulatory notification obligations and preserve legal privilege for investigation findings.", "actions": [ "Advise on regulatory notification obligations under GDPR, EU AI Act, and sector regulations within the notification window from incident declaration.", "Preserve legal privilege for the root-cause investigation report if litigation or regulatory enforcement is anticipated.", "Review and approve external communications about the incident before they are issued." ], "failure_signals": [ "Legal counsel not engaged within the playbook-defined escalation window for major incidents.", "Regulatory notifications issued without legal review.", "Incident investigation findings shared externally before privilege review." ] }, "grc_auditor": { "summary": "Incident records, post-incident reviews, and playbook maintenance history are the primary evidence that the integrity incident response control is operating and continuously improving.", "actions": [ "Review incident records for the audit period and verify that declared incidents were responded to in accordance with the playbook SLAs.", "Verify that post-incident reviews were conducted within the required timeframe and that improvement recommendations were tracked to closure.", "Assess playbook currency and confirm the last review date is within the annual review requirement." ], "metrics": [ "Playbook compliance rate for declared incidents: target 100%.", "Post-incident review completion rate within 10 business days: target 100%.", "Preventive control improvement recommendation closure rate within 90 days: target ≥ 95%." ], "failure_signals": [ "Declared incidents where playbook procedures were not followed.", "Post-incident reviews not completed within 10 business days.", "Improvement recommendations older than 90 days without closure." ] }, "it_operations": { "summary": "Integrity incident response is executed by operations: isolating tainted data, restoring from verified-good baselines, and coordinating downstream invalidation within playbook timelines.", "actions": [ "Maintain tested runbooks for isolating corrupted datasets and restoring from checksum-verified backups.", "Automate identification of downstream consumers of tainted data for invalidation.", "Run integrity-incident exercises and track time-to-isolate and time-to-restore." ], "failure_signals": [ "Restores performed from backups whose integrity was never verified.", "Downstream consumers of tainted data identified manually and late.", "Playbook steps that fail when exercised." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most organizations handle data integrity incidents through general IT incident response processes; AI-specific playbooks with data integrity focus and cryptographic recovery gates are rare." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "federated-enterprise", "eu-high-risk-ai" ], "implementers": [ "Data Governance Office", "Data Engineering", "Legal Counsel", "GRC" ], "frameworks": [ { "framework": "dama_dmbok", "requirement_id": "Ch. 13", "fit": "direct", "rationale": "DAMA-DMBOK2 Chapter 13 defines data quality issue management — logging, triaging and remediating quality incidents. The integrity incident playbook is that issue-management process hardened for tamper and corruption events.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "cobit_2019", "requirement_id": "DSS02.07", "fit": "direct", "rationale": "COBIT 2019 practice DSS02.07 (track status and produce reports) requires incident status tracking and reporting through resolution. Data integrity incident response depends on this practice for tracked, reportable incident lifecycles from detection through post-incident review.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "eu_ai_act", "requirement_id": "Art. 73", "fit": "direct", "rationale": "EU AI Act Article 73 requires providers of high-risk AI systems to report serious incidents to market surveillance authorities within defined timeframes — without undue delay and no later than 15 days after awareness, with shorter deadlines for widespread infringements or death. (Article 62 was the draft-text number; the adopted Regulation 2024/1689 renumbered serious-incident reporting to Article 73.) The data integrity incident response playbook operationalizes these notification timelines for incidents rooted in data corruption or tampering.", "normative_force": "binding-law", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "dcam", "requirement_id": "Capability 5.3", "fit": "direct", "rationale": "DCAM v2.2 capability 5.3 requires an established process for identifying and remediating data quality issues. The data integrity incident response playbook is that remediation process hardened for tampering and corruption events; the previously cited 6.2 does not address incident management.", "normative_force": "certification-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "gdpr", "requirement_id": "Art. 5(1)(f)", "fit": "partial", "rationale": "GDPR Article 5(1)(f)'s integrity and confidentiality principle requires not only preventive controls but also the ability to detect and respond to integrity failures affecting personal data. The data integrity incident response procedure satisfies the detective and corrective component of the integrity and confidentiality obligation. When integrity incidents involve personal data, the playbook must incorporate the 72-hour breach notification window under GDPR Article 33.", "normative_force": "binding-law", "source_version": "2018", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" } ], "canonical_id": "apeiris://data/controls/DI-07", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "The organization must have an operationalized Data Integrity Incident Response Playbook with defined triggers, RACI assignments, and SLA-governed containment procedures, evidenced by playbook currency within 12 months and incident records demonstrating playbook-compliant response timelines for every declared data integrity incident in the review period.", "evidence_required": [ "Data Integrity Incident Response Playbook with documented trigger classification, RACI, escalation paths, containment steps, and recovery gate procedures, with last-reviewed date within 12 months", "Incident ticket records for any declared data integrity incidents showing declaration time, containment action timestamps, and SLA compliance against playbook targets", "Post-incident review reports produced within 10 business days of each declared incident, with root-cause findings and tracked-to-closure improvement recommendations with owner and target-date fields", "Integration evidence showing DI-01 through DI-06 alerting system triggers routing into the incident management platform with automated ticket creation and priority classification", "Recovery gate verification logs confirming cryptographic baseline comparison was executed before pipeline restoration for any incident involving data asset quarantine" ], "machine_tests": [ "Inject a simulated DI-01 integrity alert into the incident management platform → assert automatic ticket creation with correct priority classification fires within the playbook-defined SLA", "Simulate an end-of-incident data restoration event without cryptographic baseline verification → assert recovery gate check blocks restoration confirmation status transition", "Query incident management system for data integrity tickets in the past 12 months → assert each closed ticket has a linked post-incident review record dated within 10 business days of ticket closure", "Verify playbook last-reviewed metadata field → assert review date is within 12 months of current date" ], "human_review": [ "Review declared incident records against playbook procedures to confirm RACI roles were followed and SLA timelines were met for containment and regulatory notification steps", "Assess post-incident review reports for root-cause depth and verify that improvement recommendations are tracked to closure with documented owners and target dates, not left open indefinitely", "Verify that legal counsel engagement timestamps align with the playbook escalation window for any incidents involving personal data or high-risk AI systems where GDPR Art. 33 notification obligations apply" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Routing data integrity incidents through general IT incident response processes without AI-specific playbook steps, resulting in missed containment procedures for training pipeline quarantine", "Restoring AI pipeline services from backup without executing cryptographic baseline verification against pre-incident snapshots, allowing potentially contaminated data to re-enter production", "Completing post-incident reviews more than 30 days after incident closure, by which point forensic evidence quality and team recall have substantially degraded", "Tracking improvement recommendations informally without assigned owners or target dates, causing findings to remain open beyond the 90-day closure target", "Not engaging legal counsel during incidents involving personal data in training sets, risking missed GDPR Art. 33 72-hour breach notification windows" ], "update_status": "current", "layer_code": "DI" }, { "id": "DI-08", "layer": "DI", "plane": "both", "name": "Data Integrity Evidence Package", "plain": "All DI-layer controls must produce structured, time-stamped evidence artifacts that are compiled into a cohesive Data Integrity Evidence Package demonstrating that integrity is protected across the full AI data lifecycle, from storage through inference output.", "threat": { "tags": [ "audit-gap", "compliance-evidence-failure", "unverifiable-integrity-claims" ], "desc": "Without a compiled evidence package, individual control outputs are fragmented across disparate systems and cannot be assembled into coherent proof of integrity governance at audit time. Regulators and enterprise procurement teams increasingly require structured evidence packages — not control descriptions — to accept AI system integrity claims. Incomplete or poorly structured evidence creates unresolved audit findings and delays in AI system deployment approvals in regulated sectors." }, "standard": [ { "id": "eu_ai_act", "section": "Art. 11 & Annex IV", "title": "Technical documentation and evidence requirements for high-risk AI" }, { "id": "dama_dmbok", "section": "Ch. 13", "title": "Data Quality — measurement and reporting" }, { "id": "dcam", "section": "Data Control Environment (Component 7)", "title": "Operational data controls and their evidence" }, { "id": "cobit_2019", "section": "MEA01.04", "title": "Assess and report assurance outcomes" } ], "sources": [ { "id": "microsoft_purview_data_governance_2024", "title": "Microsoft Purview (Data Governance)", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/purview/purview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_purview_data_governance_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Purview (Data Governance) requirements informing the apeiris://data/controls/DI-08 Data Integrity Evidence Package control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Define a standardized DataGovernanceAttestation (DV-08) evidence schema that aggregates outputs from DI-01 through DI-07; automate evidence collection into a signed, versioned package on a defined cadence; publish the package to the enterprise trust registry; make the package available for regulator, auditor, and procurement review.", "steps": [ "Define the DataGovernanceAttestation (DV-08) evidence schema including required fields: evidence_id, actor, intent, action, resource, policy, obligation, verdict, blocking_effect, confidence, confidence_basis, collected_at, valid_from, valid_until, reviewed_on, source_freshness_status, residual_risk, producer_verifier, consumer_verifiers, evidence_completeness_status, runtime_gate_required, integrity.hash (SHA-256), integrity.signature (Ed25519).", "Deploy an evidence collection pipeline that queries the output stores of DI-01 through DI-07 controls on a defined cadence (minimum monthly for standard tier, weekly for high-risk tier) and assembles them into a structured evidence package.", "Apply an Ed25519 digital signature to each evidence package using the organization's evidence signing key and publish the signed package to the enterprise trust registry with a retention period aligned to regulatory requirements.", "Establish a package review process where the Data Governance Officer reviews the evidence package completeness score and signs off before the package is made available for external review; packages scoring below 80% completeness are escalated for remediation before sign-off." ], "data_governance_officer": { "summary": "The Data Integrity Evidence Package is the primary accountability artifact for the DI layer, demonstrating to regulators, auditors, and business stakeholders that data integrity is governed across the AI data lifecycle.", "actions": [ "Review and sign off on each DI evidence package before it is published to the trust registry, confirming completeness and accuracy.", "Track evidence completeness score trends over time and escalate declining scores to the responsible control owners.", "Make the most recent signed evidence package available to regulatory and procurement reviewers upon request within the defined response window." ], "failure_signals": [ "Evidence packages published without Data Governance Officer sign-off.", "Evidence completeness score below 80% for two consecutive reporting periods without escalation.", "Evidence packages not available within the defined response window for regulatory or procurement requests." ] }, "data_engineer": { "summary": "The evidence collection pipeline must be maintained as production infrastructure with the same reliability and observability standards as operational AI pipelines.", "actions": [ "Build and maintain the evidence collection pipeline that queries DI-01 through DI-07 output stores and assembles the structured evidence package.", "Implement evidence package signing using the organization's Ed25519 key management infrastructure.", "Monitor evidence pipeline health and alert on failures within 30 minutes to ensure evidence packages are produced on the defined cadence." ], "failure_signals": [ "Evidence collection pipeline failures that result in missing reporting period packages.", "Evidence packages without valid digital signatures.", "DI control output stores that are not reachable by the evidence pipeline due to access configuration errors." ] }, "legal_counsel": { "summary": "The signed evidence package is a legal document that may be used in regulatory proceedings, procurement due diligence, and contractual compliance demonstrations.", "actions": [ "Advise on evidence retention requirements for signed DI packages under applicable regulations, including minimum retention periods for EU AI Act and GDPR compliance purposes.", "Review the evidence package structure to confirm it satisfies the technical documentation requirements under EU AI Act Annex IV for high-risk AI systems.", "Establish a process for controlled disclosure of evidence packages to regulators and third-party auditors with appropriate data protection safeguards." ], "failure_signals": [ "Signed evidence packages not retained for the legally required period.", "Evidence packages disclosed to third parties without a controlled disclosure process.", "Evidence package content that contains personal data without appropriate access controls." ] }, "grc_auditor": { "summary": "The DI-08 evidence package is the primary artifact for DI-layer audit coverage; complete and signed packages with high completeness scores significantly reduce audit sampling burden.", "actions": [ "Verify that signed DI evidence packages exist for each required reporting period and that no gaps exist in the package history.", "Assess evidence package completeness scores and trace any scores below 90% to the contributing control gaps.", "Cross-reference the evidence package with direct control output evidence for a 20% sample of DI controls to confirm the package accurately represents the underlying control operation." ], "metrics": [ "Evidence package production rate: target 100% of required reporting periods covered.", "Evidence completeness score: target ≥ 90% for standard tier, ≥ 95% for high-risk and EU AI Act tier.", "Data Governance Officer sign-off rate: target 100% of published packages." ], "failure_signals": [ "Missing evidence packages for any required reporting period.", "Evidence completeness scores below threshold without documented remediation plan.", "Unsigned or improperly signed evidence packages in the trust registry." ] }, "it_operations": { "summary": "The DI evidence package is assembled from operational telemetry — scan results, incident tickets, restore tests. Operations keeps those feeds complete, timestamped and exportable.", "actions": [ "Automate collection of integrity-monitoring outputs into the evidence pipeline.", "Ensure evidence artifacts carry timestamps, system identifiers and hashes for auditability.", "Verify evidence exports reproduce cleanly on demand for auditors." ], "failure_signals": [ "Evidence assembly requiring manual log searches each cycle.", "Telemetry retention shorter than the evidence period.", "Evidence exports failing integrity verification." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Structured, signed AI data integrity evidence packages are an emerging requirement driven by EU AI Act Annex IV and enterprise procurement due diligence; most organizations are at the ad hoc documentation stage." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise", "multi-tenant" ], "implementers": [ "Data Governance Office", "Data Engineering", "Legal Counsel", "GRC" ], "frameworks": [ { "framework": "eu_ai_act", "requirement_id": "Art. 11 & Annex IV", "fit": "direct", "rationale": "EU AI Act Article 11 and Annex IV require providers of high-risk AI systems to maintain technical documentation demonstrating compliance with data governance requirements, including evidence of data quality, integrity controls, and their effective operation. The signed DataGovernanceAttestation evidence package directly satisfies Annex IV's requirement for structured technical documentation covering data integrity. Absence of such documentation is a direct compliance gap that national market surveillance authorities can act on under Article 61.", "normative_force": "binding-law", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 13", "fit": "direct", "rationale": "DAMA-DMBOK2 Chapter 13 requires data quality programs to measure and report their results; the DI evidence package assembles those reports together with integrity baselines and incident records into one auditable artifact.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "dcam", "requirement_id": "Data Control Environment (Component 7)", "fit": "direct", "rationale": "DCAM v2.2 Component 7 (Data Control Environment) covers the operational controls over data and the evidence that they operate. The DI evidence package is the packaged output of that control environment for integrity controls.", "normative_force": "certification-standard", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_2019", "requirement_id": "MEA01.04", "fit": "direct", "rationale": "COBIT 2019 MEA01.04 requires that assurance outcomes are assessed and reported to stakeholders with sufficient detail to support governance decisions and compliance demonstrations. The DI evidence package implements the reporting artifact that MEA01.04 describes, structured as a signed attestation that can be reviewed by governance stakeholders, auditors, and regulators. The completeness score provides the quality metric that COBIT's assurance reporting objective requires.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "iso_8000", "requirement_id": "ISO 8000-61", "fit": "partial", "rationale": "ISO 8000-61 defines data quality management processes whose operation must be demonstrable; the DI evidence package records the outputs of those processes — baselines, scan results, incident records — in auditable form.", "normative_force": "voluntary-standard", "source_version": "2022", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "microsoft_purview", "requirement_id": "Purview Compliance Manager", "fit": "adjacent", "rationale": "Microsoft Purview Compliance Manager — a compliance solution distinct from the Purview data governance catalog — manages assessments and improvement actions with evidence upload. DI control outputs from Azure-hosted AI workloads can be attached as assessment evidence.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://data/controls/DI-08", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "A signed DataGovernanceAttestation evidence package must exist for each required reporting period, covering outputs from DI-01 through DI-07 with a completeness score at or above 90% for standard tier and 95% for high-risk tier, and Data Governance Officer sign-off must be recorded before the package is published to the trust registry.", "evidence_required": [ "Signed DataGovernanceAttestation evidence package with Ed25519 signature, SHA-256 content hash, and completeness score field covering outputs from DI-01 through DI-07, timestamped for each required reporting period", "Data Governance Officer sign-off records showing review date, completeness score at time of review, and approval status for each published package", "Evidence pipeline execution logs showing query timestamps for each DI control output store and assembly confirmation for the reporting period, with alert records for any pipeline failures", "Trust registry publication records confirming each signed package was deposited with retention metadata aligned to regulatory minimum retention periods", "Evidence completeness trend report covering the past four reporting periods showing scores at or above the applicable threshold" ], "machine_tests": [ "Request the latest evidence package from the trust registry → assert Ed25519 signature validates against the organization's evidence signing public key", "Parse the evidence package JSON → assert completeness_score field is present and meets the tier-appropriate threshold (>= 0.90 standard, >= 0.95 high-risk)", "Query the trust registry for packages covering each required reporting period in the past 12 months → assert no reporting period gaps exist in the package history", "Inspect evidence package timestamp fields → assert collected_at, valid_from, and valid_until are present and valid_until was in the future at the time of package publication" ], "human_review": [ "Sample three evidence packages and cross-reference DI control output artifacts against the package aggregation to confirm the package accurately represents the underlying control operation rather than summarizing it", "Assess the evidence collection pipeline architecture for single points of failure and confirm monitoring alerts are configured to fire within 30 minutes of pipeline execution failure", "Verify that evidence retention periods documented in each package align with applicable regulatory minimums including EU AI Act Annex IV and GDPR record retention requirements" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Publishing evidence packages without Data Governance Officer sign-off, creating an unvalidated record that may be relied upon in regulatory submissions or procurement due diligence", "Producing evidence packages as narrative PDF documents rather than structured JSON artifacts with cryptographic signatures, making them unverifiable by machine consumers", "Assembling evidence package contents manually from email attachments and shared drives rather than through an automated pipeline, creating reproducibility and completeness gaps", "Setting evidence completeness thresholds below 80% without documented risk acceptance, allowing structurally incomplete packages to pass governance review and be published", "Failing to retain signed evidence packages for the full regulatory retention period, creating gaps in historical compliance evidence during regulatory examinations or litigation discovery" ], "update_status": "current", "layer_code": "DI" }, { "id": "DM-01", "layer": "DM", "plane": "control", "name": "AI Data Asset Inventory and Cataloging", "plain": "Every dataset used for AI training, validation, fine-tuning, or inference must have an authoritative catalog entry recording ownership, classification, provenance lineage, sensitivity tier, and current lifecycle status.", "threat": { "tags": [ "data-sprawl", "undiscovered-pii", "orphaned-dataset", "shadow-ai-training" ], "desc": "Without a centralized data catalog, AI teams independently acquire and store training datasets outside governance visibility. Orphaned datasets accumulate sensitive or regulated content that is never reviewed for disposal. Attackers or insiders who locate ungoverned data stores can exfiltrate training sets that contain personal data, trade secrets, or privileged communications with no alert triggered." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 12", "title": "Metadata Management — data catalog and asset inventory" }, { "id": "eu_ai_act", "section": "Art. 10(2)(a)", "title": "Data governance — relevant design choices" }, { "id": "dcam", "section": "Capability 3.3", "title": "Identify the Data" }, { "id": "microsoft_purview", "section": "Data Map", "title": "Automated asset discovery and classification" } ], "sources": [ { "id": "microsoft_purview_data_governance_2024", "title": "Microsoft Purview (Data Governance)", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/purview/purview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_purview_data_governance_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Purview (Data Governance) requirements informing the apeiris://data/controls/DM-01 AI Data Asset Inventory and Cataloging control.", "reviewed_on": "2026-07-01" }, { "id": "databricks_unity_catalog_2024", "title": "Databricks Unity Catalog", "authority": "Databricks, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.databricks.com/en/data-governance/unity-catalog/index.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "databricks_unity_catalog_2024", "relationship": "informative_reference", "rationale": "Establishes Databricks Unity Catalog requirements informing the apeiris://data/controls/DM-01 AI Data Asset Inventory and Cataloging control.", "reviewed_on": "2026-07-02" }, { "id": "data_contract_spec_1_0", "title": "Open Data Contract Standard (Bitol ODCS)", "authority": "Bitol (LF AI & Data Foundation)", "source_type": "standard", "normative_force": "best-practice", "version": "3.1.0", "published_on": "2025-12-08", "retrieved_on": "2026-06-29", "canonical_url": "https://github.com/bitol-io/open-data-contract-standard", "license": "open-access", "status": "current", "flagship": false, "source_id": "data_contract_spec_1_0", "relationship": "informative_reference", "rationale": "Establishes Open Data Contract Standard (Bitol ODCS) requirements informing the apeiris://data/controls/DM-01 AI Data Asset Inventory and Cataloging control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Deploy an enterprise data catalog (Purview, Dataplex, or Unity Catalog) with automated scanner coverage. Each AI data asset receives a catalog entry at ingestion containing owner, classification, lineage graph, retention schedule, and AI-use designation. Catalog completeness is gated into data pipeline approvals.", "steps": [ "Select and deploy a centralized catalog platform with automated scanning connectors to all data stores used by AI workloads.", "Define the mandatory metadata schema for AI data assets: owner, classification label, sensitivity tier, AI-use type (training/validation/inference), data source, lineage reference, and retention schedule.", "Integrate catalog registration into data pipeline onboarding so that no dataset reaches a training or inference job without an approved catalog entry.", "Establish a monthly catalog reconciliation process to detect ungoverned data stores and escalate orphaned assets for remediation.", "Publish catalog completeness metrics to the data governance dashboard with alerting when coverage drops below the defined threshold." ], "data_governance_officer": { "summary": "The catalog is the authoritative system of record for all AI data assets. Your role is to set minimum metadata standards, assign stewards, and enforce completeness.", "actions": [ "Define the mandatory metadata schema and stewardship model for AI data catalog entries.", "Run quarterly catalog maturity reviews to validate completeness and lineage accuracy.", "Escalate ungoverned datasets identified during reconciliation for classification and disposition decisions." ], "failure_signals": [ "Catalog completeness rate below 95% for AI-designated data stores.", "Datasets in use by active AI models with no catalog entry present.", "Stewardship assignments missing on more than 5% of catalog entries." ] }, "data_engineer": { "summary": "Catalogs are only accurate if ingestion pipelines register assets automatically. Build registration as a first-class pipeline step, not an afterthought.", "actions": [ "Implement catalog API calls at dataset creation and schema change events in all data pipelines.", "Configure automated scanners to detect unregistered data stores and surface them to the governance team.", "Maintain lineage graphs by emitting OpenLineage events from ETL jobs to the catalog backend." ], "failure_signals": [ "Pipeline executions completing without catalog registration events in the audit log.", "Lineage graphs broken or missing for more than 10% of AI training datasets.", "Scanner coverage gaps in data stores added in the last 30 days." ] }, "grc_auditor": { "summary": "The data catalog is the primary artifact for demonstrating AI data assets are inventoried and governed. Catalog exports are the baseline for all DM-layer audit evidence.", "actions": [ "Request a full catalog export and cross-reference against known AI model training runs from the past 12 months.", "Sample 15% of catalog entries and verify owner, classification, and lineage fields are populated and accurate.", "Validate that catalog completeness metrics are measured and reported on a recurring schedule." ], "metrics": [ "Catalog completeness rate: target ≥ 95% of AI data stores covered.", "Orphaned dataset resolution SLA: target ≤ 30 days from discovery.", "Stewardship assignment rate: target 100% of active AI data assets." ], "failure_signals": [ "Active training runs referencing datasets with no catalog entry.", "Completeness rate below 95% for two consecutive quarters.", "No documented reconciliation process or evidence of reconciliation runs." ] }, "it_operations": { "summary": "Catalog scanners require connectivity to all data stores. Ensure scanner service accounts have read access to all AI-relevant storage systems and that scan schedules are maintained.", "actions": [ "Provision scanner service accounts with least-privilege read access to all data stores in scope.", "Monitor scanner job health and alert on scan failures exceeding 24 hours.", "Maintain an inventory of data store endpoints registered with the scanner to detect newly provisioned stores." ], "failure_signals": [ "Scanner jobs failing silently for data stores added outside the standard provisioning process.", "Service account permission errors blocking scans for more than 48 hours without incident ticket." ] }, "legal_counsel": { "summary": "You cannot honor legal duties over data you have not inventoried. The catalog is where counsel locates personal data for DSARs, identifies EU AI Act-relevant training sets, and scopes legal holds.", "actions": [ "Use the catalog as the authoritative scope for DSAR, deletion and legal-hold execution.", "Require legally relevant attributes — personal-data flag, lawful basis, jurisdiction — on catalog entries.", "Flag AI training datasets in the catalog that trigger EU AI Act documentation duties." ], "failure_signals": [ "DSAR or deletion requests answered from memory rather than from the catalog.", "AI training sets discovered outside the catalog during a regulatory inquiry.", "Catalog entries missing lawful-basis attributes for personal data." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most enterprises have a business data catalog but have not extended it to AI-specific data assets, training sets, or inference inputs." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "federated-enterprise", "eu-high-risk-ai" ], "implementers": [ "Data Governance Team", "Data Engineering", "IT Operations" ], "frameworks": [ { "framework": "dama_dmbok", "requirement_id": "Ch. 12", "fit": "direct", "rationale": "DAMA-DMBOK2 Chapter 12 (Metadata Management) defines the data catalog: an inventory of data assets with business, technical and operational metadata and recorded ownership. The AI data asset inventory is that catalog extended to AI-specific asset types, with architectural context from Chapter 4 (Data Architecture).", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "eu_ai_act", "requirement_id": "Art. 10(2)(a)", "fit": "direct", "rationale": "EU AI Act Article 10(2)(a) requires data governance practices covering the relevant design choices for training, validation and testing data sets. An authoritative data asset inventory documents which datasets embody those design choices and who owns them — the precondition for demonstrating Art. 10(2) governance in conformity assessment.", "normative_force": "binding-law", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "dcam", "requirement_id": "Capability 3.3", "fit": "direct", "rationale": "DCAM v2.2 capability 3.3 (Identify the Data) requires the organization to identify and define its data — the inventory-and-definition capability an AI data asset catalog implements. Capability 3.1 concerns architecture strategy, not asset identification.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "microsoft_purview", "requirement_id": "Data Map", "fit": "direct", "rationale": "Microsoft Purview Data Map provides automated scanning, classification, and lineage capabilities that directly implement this control for Azure and hybrid environments. Purview's asset discovery and sensitivity labeling features operationalize the catalog registration requirement. This framework entry covers technical implementation guidance for enterprises using Microsoft cloud infrastructure.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "cobit_2019", "requirement_id": "APO01.06", "fit": "direct", "rationale": "COBIT 2019 practice APO01.06 requires defined information (data) and system ownership. An authoritative AI data asset inventory is where that ownership is recorded and kept current for every dataset, model artifact and derived output.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "databricks_unity", "requirement_id": "Unity Catalog", "fit": "partial", "rationale": "Databricks Unity Catalog provides fine-grained governance for data assets in Databricks-based AI and analytics environments. Unity Catalog's automated lineage capture and asset tagging capabilities partially implement catalog registration for Databricks workloads. Organizations using Databricks for AI training should treat Unity Catalog as the catalog backend for Databricks-managed assets.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "data_contract_spec", "requirement_id": "Schema and Terms sections", "fit": "direct", "rationale": "The Bitol Open Data Contract Standard (ODCS) describes datasets in named sections — including schema and terms — rather than numbered clauses. Cataloged AI data assets can carry ODCS contracts so consumers receive schema, ownership and usage terms together with the asset.", "normative_force": "best-practice", "source_version": "3.1.0 (ODCS)", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://data/controls/DM-01", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every dataset actively used for AI training, validation, fine-tuning, or inference must have an authoritative catalog entry with all mandatory metadata fields (owner, classification label, sensitivity tier, AI-use type, lineage reference, retention schedule) populated and catalog completeness must be at or above 95% of AI-designated data stores, with no active model training or inference runs referencing uncatalogued dataset identifiers.", "evidence_required": [ "Data catalog export showing catalog entries for all AI-designated data stores with owner, classification label, sensitivity tier, AI-use type, lineage reference, and retention schedule fields populated and non-null", "Catalog completeness metric report covering the past three monthly reconciliation cycles, showing percentage of AI data stores with all mandatory metadata fields completed", "Pipeline audit logs confirming catalog registration API events were emitted for all dataset creation and schema change events during the review period", "Monthly catalog reconciliation report identifying ungoverned data stores discovered by automated scanning and their remediation status and resolution timestamps", "AI model training run metadata showing catalog entry IDs for each dataset consumed, confirming no training run referenced a dataset without a valid catalog entry" ], "machine_tests": [ "Query catalog API against the known AI workload data store inventory → assert 100% of stores return a catalog entry with all mandatory metadata fields populated and non-null", "Trigger a test dataset creation event in the ingestion pipeline → assert catalog registration API call is emitted and logged in the pipeline audit trail within the defined SLA", "Execute catalog scanner on a test data store containing a pre-planted unregistered dataset → assert scanner discovery alert fires and routes to the governance remediation queue", "Cross-reference active model training run metadata against catalog entry IDs → assert all referenced dataset IDs resolve to valid catalog entries with no 404 or missing-entry responses" ], "human_review": [ "Sample 15% of catalog entries and manually verify that owner, classification, and lineage fields contain accurate information and not auto-populated placeholder or default values", "Review the monthly reconciliation report and assess whether ungoverned datasets discovered during scanning were resolved within the 30-day SLA with documented disposition decisions", "Verify that the catalog stewardship model assigns named individuals (not shared team accounts) as data stewards for all active AI data assets, enabling accountability traceability" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Allowing AI training pipelines to reference datasets by filesystem path or object storage URI without requiring a corresponding catalog entry, creating an ungoverned bypass of the registration requirement", "Assigning a single generic shared account as data steward across all AI data assets, making individual accountability impossible to establish during incident investigation or regulatory review", "Running catalog scans on a weekly or longer cadence, creating windows where newly provisioned AI training data stores accumulate sensitive content for days before governance coverage is established", "Populating mandatory catalog fields with template placeholder values such as 'TBD' or 'Unknown' to satisfy completeness metrics without performing actual metadata governance", "Treating catalog registration as a one-time activity at dataset creation without updating entries when datasets are modified, reclassified, reassigned, or approach end-of-lifecycle" ], "update_status": "current", "layer_code": "DM" }, { "id": "DM-02", "layer": "DM", "plane": "control", "name": "Data Classification and Sensitivity Tagging at Rest", "plain": "All AI data assets stored at rest must have machine-readable sensitivity labels applied, with storage controls — encryption tier, access restrictions, audit logging level — enforced according to the assigned classification.", "threat": { "tags": [ "sensitive-data-exposure", "misclassification", "unauthorized-training-data-access", "regulatory-breach" ], "desc": "Unclassified AI training and inference data is routinely stored with default open permissions, enabling broad internal access to data that may contain personal information, financial records, or privileged communications. Misclassification errors cause either over-exposure of sensitive data or unnecessary restrictions that block legitimate use. Regulators examining AI system compliance treat absent classification labels as evidence of systemic data governance failure." }, "standard": [ { "id": "iso_27701", "section": "§6.5.2.1", "title": "Classification of information — PII considered in the classification scheme" }, { "id": "gdpr", "section": "Art. 5(1)(f)", "title": "Integrity and confidentiality of personal data" }, { "id": "nist_pf", "section": "ID.IM-P6", "title": "Data elements within the data actions are inventoried" }, { "id": "microsoft_purview", "section": "Information Protection — sensitivity labels", "title": "Automated classification and label enforcement" } ], "sources": [ { "id": "aws_lake_formation_macie_2024", "title": "AWS data governance services (Lake Formation, Macie, Glue, S3)", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/lake-formation/latest/dg/what-is-lake-formation.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_lake_formation_macie_2024", "relationship": "informative_reference", "rationale": "Establishes AWS data governance services (Lake Formation, Macie, Glue, S3) requirements informing the apeiris://data/controls/DM-02 Data Classification and Sensitivity Tagging at Rest control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_purview_data_governance_2024", "title": "Microsoft Purview (Data Governance)", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/purview/purview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_purview_data_governance_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Purview (Data Governance) requirements informing the apeiris://data/controls/DM-02 Data Classification and Sensitivity Tagging at Rest control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Deploy automated classification scanning on all AI data stores using trainable classifiers and regex patterns for known sensitive data types. Apply machine-readable sensitivity labels to objects and enforce label-based storage controls through policy engines. Remediate misclassified assets through a structured exception workflow with data steward approval.", "steps": [ "Define the enterprise classification taxonomy for AI data assets (e.g., Public, Internal, Confidential, Restricted-PII, Restricted-PHI) and map each tier to specific storage controls including encryption standard, access policy, and audit log retention.", "Deploy automated classification scanners on all AI-relevant data stores with coverage reports generated at minimum weekly.", "Configure label-enforcement policies that automatically apply storage controls — KMS key selection, object ACLs, audit logging tier — based on the assigned classification label.", "Establish a misclassification dispute and appeal process with a data steward review SLA of five business days.", "Produce monthly classification coverage and accuracy reports for the data governance committee." ], "data_governance_officer": { "summary": "Classification is the foundation of access control and regulatory compliance for AI data. Define and own the classification taxonomy and ensure enforcement policies align with organizational risk appetite.", "actions": [ "Define and publish the enterprise classification taxonomy with specific AI data categories and their mapped storage controls.", "Review and approve label enforcement policies before deployment to production data stores.", "Chair the quarterly classification accuracy review and sign off on remediation plans for misclassified assets." ], "failure_signals": [ "More than 5% of AI data assets with no classification label applied.", "Classification labels not enforced by policy on any storage tier.", "No documented classification taxonomy published for AI-specific data categories." ] }, "data_engineer": { "summary": "Classification labels must be applied at data ingestion and maintained through pipeline transformations. Build label propagation into ETL pipelines so derived datasets inherit classification from source assets.", "actions": [ "Implement label propagation logic in ETL pipelines so that derived datasets inherit the highest classification tier of their source datasets.", "Register classification label assignments in the data catalog as part of the dataset creation event.", "Instrument pipelines to block writes to storage tiers that are inconsistent with the assigned label." ], "failure_signals": [ "Derived training datasets with lower classification than source data after transformations.", "Label propagation events missing from pipeline audit logs for ingestion jobs.", "Objects written to storage tiers that do not match classification policy without an approved exception." ] }, "legal_counsel": { "summary": "Classification labels directly determine which legal obligations attach to AI data. Verify that personal data, regulated records, and privileged communications are correctly classified and that storage controls satisfy applicable legal requirements.", "actions": [ "Review the classification taxonomy to confirm personal data, PHI, PCI, and legally privileged categories are explicitly defined with correct control mappings.", "Verify that data subject to cross-border transfer restrictions is classified and access-restricted to authorized geographic regions.", "Review misclassification remediation logs to confirm no regulated data was exposed while misclassified." ], "failure_signals": [ "Personal data in AI training sets classified as Internal rather than Restricted-PII.", "Cross-border transfer controls not enforced on data classified as subject to localization requirements.", "No legal review of classification taxonomy updates in the past 12 months." ] }, "grc_auditor": { "summary": "Classification coverage and enforcement are primary indicators of data governance maturity. Audit both label coverage and the technical enforcement of controls downstream of labeling.", "actions": [ "Request classification coverage reports and verify that automated scanners cover all AI-relevant data stores.", "Sample 20% of Restricted-tier assets and confirm storage controls — encryption, access policy, audit logging — are enforced consistent with the taxonomy.", "Review the misclassification incident log and assess whether remediation SLAs were met." ], "metrics": [ "Classification coverage rate: target 100% of AI data assets with a label applied.", "Enforcement alignment rate: target ≥ 98% of labeled assets with storage controls matching taxonomy.", "Misclassification remediation SLA: target ≤ 5 business days from discovery." ], "failure_signals": [ "Classification coverage below 95% for AI training or inference data stores.", "Restricted-tier assets found without encryption or access controls enforced.", "Misclassification remediations consistently exceeding the five-day SLA." ] }, "it_operations": { "summary": "Label enforcement depends on storage-tier policies being correctly configured and scanner service accounts maintaining connectivity to all data stores.", "actions": [ "Configure KMS key policies, object storage ACLs, and network policies to enforce classification-tier controls automatically.", "Monitor scanner connectivity and alert on scan failures within four hours.", "Maintain a registry of all data store endpoints to detect unscanned stores added outside standard provisioning." ], "failure_signals": [ "KMS key assignment not aligned to classification tier on newly provisioned data stores.", "Scanner failures unreported for more than 24 hours.", "Data stores added outside the provisioning process with no scanner coverage established within 48 hours." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Most enterprises classify business data but have not extended classification to AI training and inference datasets, which are often stored in data science environments outside central governance." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "eu-high-risk-ai", "multi-tenant" ], "implementers": [ "Data Governance Team", "Security Operations", "IT Operations" ], "frameworks": [ { "framework": "iso_27701", "requirement_id": "§6.5.2.1", "fit": "direct", "rationale": "ISO/IEC 27701:2019 §6.5.2.1 extends ISO/IEC 27002 information-classification guidance to privacy: the organization's classification scheme should explicitly consider PII as part of its taxonomy. Classification and sensitivity tagging of AI datasets at rest directly implements this guidance. §8.4.1 concerns processors' temporary files, not classification. The certification-standard status of ISO 27701 makes this a key control for privacy management system certification.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "gdpr", "requirement_id": "Art. 5(1)(f)", "fit": "direct", "rationale": "GDPR Article 5(1)(f) requires personal data to be processed with appropriate security to protect against unauthorized access, applying the integrity and confidentiality principle. Classification-enforced storage controls — encryption, access restrictions — are the technical implementation of this legal obligation for AI systems processing personal data. Non-compliance exposes organizations to supervisory authority enforcement and fines up to 2% of global annual turnover.", "normative_force": "binding-law", "source_version": "2018", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_pf", "requirement_id": "ID.IM-P6", "fit": "direct", "rationale": "NIST Privacy Framework ID.IM-P6 requires the data elements within data actions to be inventoried. Classification and sensitivity tagging at rest is how that inventory carries protection-relevant attributes for every stored AI data element.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "microsoft_purview", "requirement_id": "Information Protection — sensitivity labels", "fit": "direct", "rationale": "Microsoft Purview Information Protection applies and enforces sensitivity labels on data at rest across Microsoft 365 and Azure storage, with auto-labeling policies driven by sensitive information types. It is a reference implementation for at-rest classification tagging on Microsoft estates.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "aws_lake_formation", "requirement_id": "Amazon Macie", "fit": "partial", "rationale": "Amazon Macie provides automated sensitive data discovery and classification for S3-hosted AI datasets, identifying PII, PHI, and other regulated data types. Lake Formation tag-based access control enforces classification-driven access policies at the data lake level. Together these services partially implement classification and enforcement for AWS-based AI data pipelines.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "owasp_ai_exchange", "requirement_id": "segregatedata", "fit": "supporting", "rationale": "DM-02 enforces per-classification storage, access, and encryption controls at rest, segregating sensitive AI data to limit exposure.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://data/controls/DM-02", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "All AI data assets stored at rest must have a machine-readable sensitivity label from the enterprise classification taxonomy applied, with storage controls (encryption tier, access policy, audit logging level) enforced by policy engine in alignment with the assigned label; classification coverage must reach 100% of AI-designated data stores and zero Restricted-tier assets may have non-compliant storage controls.", "evidence_required": [ "Classification scanner coverage report showing 100% of AI-designated data stores scanned within the past seven days, with sensitivity labels applied to all discovered objects", "Label enforcement policy configuration export showing label-to-storage-control mappings for each classification tier including encryption standard, access control list policy, and audit log retention period", "Storage access control audit confirming Restricted-PII and Restricted-PHI labeled assets have KMS encryption with restricted key policies and audit logging enabled at the required level", "Misclassification incident log for the past 12 months with remediation timestamps confirming the five-business-day SLA was met for each remediation", "ETL pipeline label propagation audit log confirming derived datasets inherit the highest classification label from their source datasets during transformation runs" ], "machine_tests": [ "Attempt to write an object labeled Restricted-PII to a storage bucket lacking KMS encryption and restricted ACL → assert the pipeline write is blocked by the label enforcement policy before the write completes", "Trigger a classification scanner run on a test data store pre-loaded with synthetic PII records → assert scanner correctly assigns Restricted-PII label to discovered objects within the defined weekly scan cadence", "Execute an ETL transformation job consuming a Restricted-PII labeled source dataset → assert the output dataset inherits the Restricted-PII label in the pipeline audit log rather than defaulting to a lower tier", "Query the access control policy for a sample of Restricted-tier AI data assets → assert each policy requires authenticated principal, denies unauthenticated or public access, and enforces a customer-managed KMS key" ], "human_review": [ "Sample 20% of Restricted-tier AI data assets and manually verify that encryption key assignment, access policy configuration, and audit logging level match the classification taxonomy requirements, not just the label metadata", "Review the classification taxonomy document with legal counsel to confirm personal data, PHI, and legally privileged categories are correctly defined with control mappings that satisfy GDPR Art. 5(1)(f) and ISO 27701 §6.5.2.1 obligations", "Assess the misclassification dispute workflow by reviewing resolved cases from the past quarter, verifying data steward review SLAs were met and approved exceptions are documented with business justification and expiry dates" ], "blocking_effect": "blocks-runtime-action", "normative_status": "binding-law", "anti_patterns": [ "Applying classification labels through manual human tagging without automated scanner enforcement, causing label drift as data volumes grow and manual processes fail to scale", "Classifying AI training datasets that contain personal data as Internal rather than Restricted-PII, understating sensitivity and activating weaker storage controls than legally required under GDPR Art. 5(1)(f)", "Treating label assignment as a one-time event at dataset creation without configuring label propagation for derived datasets, allowing training and validation sets to carry lower classification than their source data", "Using classification labels as documentation-only metadata without binding them to enforceable storage policies, creating the appearance of classification governance without actual access control effect", "Failing to update classification labels when data enrichment processes add new sensitive categories to an existing dataset that was initially classified at a lower tier" ], "update_status": "current", "layer_code": "DM" }, { "id": "DM-03", "layer": "DM", "plane": "lifecycle", "name": "Data Retention and Disposal Governance", "plain": "AI data assets must have documented retention schedules aligned to legal, regulatory, and operational requirements, with secure disposal executed at end-of-retention and verified through deletion certificates or cryptographic erasure evidence.", "threat": { "tags": [ "unauthorized-data-retention", "disposal-failure", "regulatory-non-compliance", "data-subject-right-violation" ], "desc": "Training datasets and inference logs accumulate indefinitely because no retention schedule is defined or enforced. Retained data that should have been deleted creates ongoing regulatory exposure under GDPR right-to-erasure requests, CCPA deletion rights, and sector-specific record retention laws. Adversaries who later compromise stale data stores extract historical training data — including personal information — that the organization had long stopped monitoring." }, "standard": [ { "id": "gdpr", "section": "Art. 5(1)(e)", "title": "Storage limitation principle for personal data" }, { "id": "iso_27701", "section": "§7.4.5", "title": "Disposal of PII and retention scheduling" }, { "id": "dama_dmbok", "section": "Ch. 6", "title": "Data Storage and Operations — retention and disposal" }, { "id": "cobit_2019", "section": "APO01.06", "title": "Define information (data) and system ownership" } ], "sources": [ { "id": "gdpr", "title": "GDPR", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2016/679", "published_on": "2016-04-27", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj", "license": "open-access", "status": "current", "flagship": false, "source_id": "gdpr", "relationship": "normative_requirement", "rationale": "Establishes GDPR Art. 5(1)(e) requirements informing the apeiris://data/controls/DM-03 Data Retention and Disposal Governance control.", "reviewed_on": "2026-07-02" }, { "id": "iso_27701", "title": "ISO/IEC 27701:2019", "authority": "ISO/IEC", "source_type": "standard", "normative_force": "certification-standard", "version": "2019 (superseded by ISO/IEC 27701:2025, published 2025-10-14; 24-36 month certification transition)", "published_on": "2019-08-06", "retrieved_on": "2026-07-02", "canonical_url": "https://www.iso.org/standard/71670.html", "license": "proprietary", "status": "current", "flagship": false, "source_id": "iso_27701", "relationship": "implementation_pattern", "rationale": "Establishes ISO/IEC 27701:2019 §7.4.5 requirements informing the apeiris://data/controls/DM-03 Data Retention and Disposal Governance control.", "reviewed_on": "2026-07-02" }, { "id": "dama_dmbok", "title": "DAMA-DMBOK 2nd Ed", "authority": "DAMA International", "source_type": "standard", "normative_force": "best-practice", "version": "2 (2nd Edition, Revised 2024)", "published_on": "2017-07-05", "retrieved_on": "2026-07-02", "canonical_url": "https://www.dama.org/cpages/body-of-knowledge", "license": "proprietary", "status": "current", "flagship": false, "source_id": "dama_dmbok", "relationship": "informative_reference", "rationale": "Establishes DAMA-DMBOK 2nd Ed Ch. 6 requirements informing the apeiris://data/controls/DM-03 Data Retention and Disposal Governance control.", "reviewed_on": "2026-07-02" }, { "id": "cobit_2019", "title": "COBIT 2019", "authority": "ISACA", "source_type": "standard", "normative_force": "industry-framework", "version": "2019", "published_on": "2018-11-12", "retrieved_on": "2026-07-02", "canonical_url": "https://www.isaca.org/resources/cobit", "license": "proprietary", "status": "current", "flagship": false, "source_id": "cobit_2019", "relationship": "informative_reference", "rationale": "Establishes COBIT 2019 APO01.06 requirements informing the apeiris://data/controls/DM-03 Data Retention and Disposal Governance control.", "reviewed_on": "2026-07-02" }, { "id": "nist_pf", "title": "NIST Privacy Framework 1.0", "authority": "National Institute of Standards and Technology", "source_type": "standard", "normative_force": "voluntary-standard", "version": "1.0", "published_on": "2020-01-16", "retrieved_on": "2026-07-02", "canonical_url": "https://www.nist.gov/privacy-framework", "license": "open-access", "status": "current", "flagship": false, "source_id": "nist_pf", "relationship": "implementation_pattern", "rationale": "Establishes NIST Privacy Framework 1.0 CT.DM-P4 requirements informing the apeiris://data/controls/DM-03 Data Retention and Disposal Governance control.", "reviewed_on": "2026-07-02" }, { "id": "eu_ai_act", "title": "EU AI Act", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2024/1689", "published_on": "2024-07-12", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2024/1689/oj/eng", "license": "open-access", "status": "current", "flagship": false, "source_id": "eu_ai_act", "relationship": "normative_requirement", "rationale": "Establishes EU AI Act Art. 10(5) requirements informing the apeiris://data/controls/DM-03 Data Retention and Disposal Governance control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Define per-data-category retention schedules in a centralized retention schedule registry. Automate retention enforcement through storage lifecycle policies with hard deletion or cryptographic erasure at expiry. Produce deletion certificates for regulated data categories and log disposal events in the data catalog.", "steps": [ "Develop a retention schedule for each AI data category — training sets, validation sets, inference logs, model outputs, feedback data — specifying retention duration, legal basis for retention, and disposal method.", "Configure automated lifecycle policies on all storage systems to enforce retention schedules, using hard deletion for non-regulated data and cryptographic erasure for regulated categories where regulatory standards permit.", "Integrate disposal events with the data catalog so that catalog entries are updated to reflect end-of-life status and deletion certification is attached.", "Establish a data subject deletion request workflow that identifies all AI data stores containing the requesting individual's data and executes deletion within the legally required window.", "Conduct annual retention schedule reviews with legal counsel to align schedules with current regulatory requirements." ], "data_governance_officer": { "summary": "Retention schedules are a legal and regulatory instrument. Own the schedule registry, coordinate legal review annually, and ensure disposal execution is traceable and evidenced.", "actions": [ "Publish and maintain the enterprise retention schedule registry covering all AI data categories with legal basis citations.", "Coordinate annual retention schedule reviews with legal counsel and regulatory affairs.", "Review quarterly disposal execution reports and escalate missed disposal events for root cause analysis." ], "failure_signals": [ "AI data categories with no documented retention schedule.", "Retention schedules not reviewed with legal counsel in the past 12 months.", "Data subject deletion requests not completed within the legally required window." ] }, "legal_counsel": { "summary": "Retention schedules must reflect applicable laws and regulations. Review schedules annually, ensure data subject rights processes satisfy legal timelines, and verify that disposal methods meet regulatory requirements for different data categories.", "actions": [ "Review retention schedules for all AI data categories and confirm alignment with GDPR, CCPA, and any applicable sector-specific laws.", "Verify that data subject deletion request workflows satisfy the 30-day GDPR window and any stricter sector requirements.", "Confirm that disposal methods — hard deletion vs. cryptographic erasure — are legally sufficient for each regulated data category." ], "failure_signals": [ "Retention schedules citing outdated or superseded legal requirements.", "Data subject deletion requests completed outside the statutory window.", "Disposal method for regulated categories not legally reviewed or documented." ] }, "grc_auditor": { "summary": "Retention and disposal evidence is a primary artifact in regulatory examinations and litigation holds. Audit both the existence of schedules and the evidence of disposal execution.", "actions": [ "Request the retention schedule registry and verify all active AI data categories are covered with legal basis citations.", "Review deletion certificates or cryptographic erasure logs for regulated data disposed of in the past 12 months.", "Sample active AI training datasets and confirm their retention period is documented and has not lapsed without disposal." ], "metrics": [ "Retention schedule coverage: target 100% of AI data categories with documented schedule.", "Disposal execution rate: target 100% of assets reaching end-of-retention disposed within 30 days.", "Data subject deletion compliance rate: target 100% completed within statutory window." ], "failure_signals": [ "AI data categories with no documented retention schedule.", "Assets past their scheduled disposal date still present in storage.", "No deletion certificates or disposal logs available for regulated categories." ] }, "it_operations": { "summary": "Retention enforcement depends on storage lifecycle policies being correctly configured and monitored. Ensure automated policies are active on all AI data stores and that lifecycle rule changes go through change management.", "actions": [ "Configure and monitor automated lifecycle policies on all AI-relevant storage systems, covering object expiration, archival transitions, and deletion rules.", "Gate changes to lifecycle policies through the change management process with data governance approval.", "Alert on lifecycle policy failures or manual overrides that bypass scheduled deletion." ], "failure_signals": [ "Lifecycle policies not configured on AI data stores provisioned in the last 90 days.", "Lifecycle policy override events without approved change records.", "Deletion job failures unreported for more than 24 hours." ] }, "data_engineer": { "summary": "Retention policy becomes real in code: TTLs, scheduled deletion jobs, and propagation of disposal through derived datasets, feature stores, caches and backups.", "actions": [ "Implement retention TTLs and scheduled deletion jobs for every AI data store, driven by the retention schedule registry.", "Propagate deletions to derived data — features, embeddings, caches — and verify completion with post-deletion scans.", "Emit deletion-execution logs as evidence that disposal actually ran, not just that policy existed." ], "failure_signals": [ "Datasets past their retention period still present in storage scans.", "Deletions applied at source but persisting in derived stores or backups beyond policy.", "No execution logs proving scheduled disposal jobs ran." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "AI teams typically retain all training and inference data indefinitely as storage costs are low. Formal retention scheduling for AI-specific data categories is rare outside highly regulated sectors." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "eu-high-risk-ai", "high-risk-sector", "federated-enterprise" ], "implementers": [ "Data Governance Team", "Legal Counsel", "IT Operations" ], "frameworks": [ { "framework": "gdpr", "requirement_id": "Art. 5(1)(e)", "fit": "direct", "rationale": "GDPR Article 5(1)(e) establishes the storage limitation principle, requiring personal data to be kept no longer than necessary for the purposes for which it was collected. AI training and inference logs containing personal data must have documented retention schedules that enforce this principle. Non-compliance with storage limitation is among the most commonly cited GDPR violations by supervisory authorities.", "normative_force": "binding-law", "source_version": "2018", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "iso_27701", "requirement_id": "§7.4.5", "fit": "direct", "rationale": "ISO/IEC 27701:2019 §7.4.5 requires PII controllers to establish and document retention periods and disposal procedures for personal data. Retention schedule documentation and disposal execution records are primary evidence artifacts for ISO 27701 certification audits. The standard requires retention schedules to be reviewed when legal or regulatory requirements change.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 6", "fit": "direct", "rationale": "DAMA-DMBOK2 Chapter 6 (Data Storage and Operations) covers the operational data lifecycle including retention scheduling, archival and disposal. Retention and disposal governance for AI data implements that lifecycle guidance.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "cobit_2019", "requirement_id": "APO01.06", "fit": "partial", "rationale": "COBIT 2019 practice APO01.06 (define information/data and system ownership) makes named owners accountable for the data lifecycle — including how long data is retained and when it is disposed. Retention and disposal governance assigns those decisions to the accountable owners.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_pf", "requirement_id": "CT.DM-P4", "fit": "direct", "rationale": "NIST Privacy Framework 1.0 CT.DM-P4 addresses data retention and destruction as a data management control, requiring organizations to maintain data only as long as needed and dispose of it securely. The framework provides a voluntary governance model for retention scheduling that complements binding legal requirements. This control directly implements the CT.DM-P4 practices.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "eu_ai_act", "requirement_id": "Art. 10(5)", "fit": "partial", "rationale": "EU AI Act Article 10(5) permits processing of special categories of personal data only where strictly necessary for bias detection and correction in high-risk AI systems, subject to strict safeguards — including the duty to delete that data once the bias has been corrected or the data reaches the end of its retention period. Retention and disposal governance is the mechanism that operationalizes this deletion duty for AI training sets; Art. 10(5) is not a general retention basis.", "normative_force": "binding-law", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "owasp_ai_exchange", "requirement_id": "shortretain", "fit": "direct", "rationale": "DM-03 requires a documented retention schedule per AI data category with disposal within 30 days of the scheduled date, directly implementing shortest-necessary retention.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://data/controls/DM-03", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every AI data category must have a documented retention schedule with legal basis citation in the enterprise retention registry, all assets at or past their scheduled disposal date must be disposed of within 30 days, and deletion certificates or cryptographic erasure logs must be available as verifiable evidence for all regulated data categories disposed of in the review period.", "evidence_required": [ "Enterprise retention schedule registry export showing retention duration, legal basis citation, and disposal method for each AI data category (training sets, validation sets, inference logs, model outputs, feedback data)", "Automated lifecycle policy configuration export for all AI-relevant storage systems confirming hard deletion or cryptographic erasure rules are active and map to retention schedule registry entries", "Deletion certificates or cryptographic erasure verification logs for all regulated data categories disposed of in the past 12 months, with disposal timestamps and data category identifiers", "Data subject deletion request log showing request receipt dates, identified data store locations, deletion execution timestamps, and confirmation of statutory window compliance", "Legal counsel annual retention schedule review sign-off record dated within 12 months confirming schedule alignment with current applicable regulations including GDPR Art. 5(1)(e) and the EU AI Act Art. 10(5) deletion duty for special-category data processed for bias detection and correction" ], "machine_tests": [ "Query all AI data stores for objects with creation timestamps older than their scheduled retention period → assert zero objects are present beyond their retention expiry date plus the 30-day disposal window", "Trigger a lifecycle policy test run on a test storage bucket configured with a 1-day retention rule → assert test objects are deleted and a disposal event with timestamp and object identifier is logged in the audit trail", "Submit a test data subject deletion request through the workflow system → assert all identified data store locations receive deletion tasks and deletion confirmation events are logged within the configured SLA", "Query lifecycle policy configurations for all AI-designated data stores → assert each store has at least one active retention rule referencing a scheduled retention period from the retention registry" ], "human_review": [ "Review the retention schedule registry with legal counsel to verify legal basis citations are accurate and current, with particular attention to special-category data processed for bias detection and correction under EU AI Act Art. 10(5) — which must be deleted once the bias is corrected or the retention period expires — and GDPR Art. 6 basis claims", "Sample five AI data categories and trace disposal evidence from the past 12 months, assessing whether deletion certificates or erasure logs are present and whether disposal methods are legally sufficient for each category's regulatory classification", "Assess the data subject deletion request workflow by reviewing completed requests from the past quarter and verifying statutory window compliance, completeness of data store identification, and absence of data store gaps" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Retaining all AI training and inference data indefinitely under a minimum-storage-cost rationale with no defined retention schedule or legal basis, creating accumulating regulatory exposure under GDPR Art. 5(1)(e) storage limitation", "Relying on ad hoc manual deletion requests to storage administrators rather than automated lifecycle policies, creating disposal gaps when personnel change or tickets are deprioritized", "Applying retention lifecycle rules only to relational databases while leaving object storage buckets containing AI training datasets and inference logs with no expiration rules configured", "Issuing deletion certificates based on catalog metadata record updates rather than verified storage system deletion events, creating unverifiable disposal claims during regulatory examination", "Excluding AI model artifacts, embedding stores, and cached inference outputs from retention schedules, leaving data derived from personal information outside the governance scope of the retention program" ], "update_status": "current", "layer_code": "DM" }, { "id": "DM-04", "layer": "DM", "plane": "control", "name": "Master Data Management for AI Systems", "plain": "Authoritative master data records that feed AI training or inference pipelines must be governed through a formal master data management process with a designated golden record, versioning, conflict resolution, and downstream change notification.", "threat": { "tags": [ "training-data-version-conflict", "authoritative-source-ambiguity", "model-contamination", "silent-data-drift" ], "desc": "AI systems trained or operated against conflicting versions of reference data — customer segments, product hierarchies, entity classifications — produce inconsistent outputs that cannot be traced to a single authoritative source. When master data changes without downstream notification, models silently operate against stale reference data until output degradation becomes observable. Data poisoning attacks that corrupt an unversioned master data source propagate to all AI systems consuming that source without detection." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 10", "title": "Master Data Management framework and golden record" }, { "id": "dcam", "section": "Capability 5.2", "title": "Data quality is profiled and measured" }, { "id": "iso_8000", "section": "ISO 8000-110", "title": "Master data exchange — syntax and semantic encoding" }, { "id": "cobit_2019", "section": "BAI04.01", "title": "Assess current availability, performance and capacity and create a baseline" } ], "sources": [ { "id": "snowflake_horizon_data_governance_2024", "title": "Snowflake Horizon (Data Governance)", "authority": "Snowflake Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.snowflake.com/en/data-cloud/workloads/data-governance/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "snowflake_horizon", "relationship": "informative_reference", "rationale": "Establishes Snowflake Horizon (Data Governance) requirements informing the apeiris://data/controls/DM-04 Master Data Management for AI Systems control.", "reviewed_on": "2026-07-01" }, { "id": "databricks_unity_catalog_2024", "title": "Databricks Unity Catalog", "authority": "Databricks, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.databricks.com/en/data-governance/unity-catalog/index.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "databricks_unity_catalog_2024", "relationship": "informative_reference", "rationale": "Establishes Databricks Unity Catalog requirements informing the apeiris://data/controls/DM-04 Master Data Management for AI Systems control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Designate an MDM hub or authoritative data source for each master data domain consumed by AI systems. Publish versioned golden records with change history and issue change events to subscribed AI pipelines. Implement conflict resolution workflows with data steward adjudication for conflicting records from multiple source systems.", "steps": [ "Inventory all master data domains — entities, hierarchies, taxonomies, reference tables — consumed by AI training and inference systems and designate the authoritative source and steward for each.", "Implement or integrate an MDM hub that maintains versioned golden records with full change history and supports conflict detection from multiple source systems.", "Publish change notification events to all downstream AI pipelines when master data records change, enabling pipelines to trigger retraining or revalidation assessments.", "Define and enforce conflict resolution workflows with data steward adjudication SLAs for records where multiple sources provide conflicting attribute values.", "Include master data version references in AI model metadata so that the exact version of master data used in training is traceable from model artifacts." ], "data_governance_officer": { "summary": "MDM governance for AI is an extension of enterprise MDM. Ensure AI systems are subscribed to authoritative master data sources and that change events reach AI pipeline owners within defined SLAs.", "actions": [ "Maintain a registry of master data domains and their designated authoritative sources for AI consumption.", "Establish and enforce SLAs for conflict resolution adjudication by data stewards.", "Review quarterly MDM health reports covering golden record completeness, conflict volume, and downstream notification delivery rates." ], "failure_signals": [ "AI training pipelines consuming master data from non-authoritative source systems.", "Unresolved master data conflicts older than the defined SLA in any domain consumed by production AI systems.", "No change notification mechanism for master data domains that feed active AI models." ] }, "data_engineer": { "summary": "AI pipelines must consume master data through controlled interfaces that enforce version pinning and change subscription. Avoid direct database queries against source systems that bypass MDM governance.", "actions": [ "Refactor AI pipeline integrations to consume master data through MDM hub APIs rather than direct source system queries.", "Implement version pinning in training pipelines so that the exact golden record version used is logged in training run metadata.", "Subscribe AI pipelines to change notification events and implement automated retraining assessment triggers." ], "failure_signals": [ "Training pipeline configurations referencing source system databases rather than MDM hub endpoints.", "No version reference for master data in model training run metadata.", "Change notification subscriptions missing for master data domains consumed by active models." ] }, "grc_auditor": { "summary": "MDM governance for AI is audited by confirming that models can be traced to specific master data versions and that conflict resolution processes are functioning.", "actions": [ "Request the master data domain registry and verify designated authoritative sources are documented for all AI-consumed domains.", "Sample five AI model artifacts and verify that master data version references are present in training metadata.", "Review conflict resolution logs and confirm adjudication SLAs are being met." ], "metrics": [ "MDM domain coverage: target 100% of AI-consumed master data domains with designated authoritative source.", "Conflict resolution SLA compliance: target ≥ 95% of conflicts resolved within defined SLA.", "Training run traceability: target 100% of production model training runs with master data version references." ], "failure_signals": [ "Master data domains consumed by AI models with no authoritative source designation.", "Conflict resolution backlogs in domains feeding production AI systems.", "Model training artifacts with no master data version reference." ] }, "it_operations": { "summary": "MDM hub availability is a dependency for AI pipeline reliability. Ensure MDM hub SLAs support AI training and inference schedules and that change event delivery is monitored.", "actions": [ "Monitor MDM hub availability and alert when uptime falls below the SLA required by AI pipeline dependencies.", "Instrument change event publishing pipelines with delivery confirmation and dead-letter queues for failed notifications.", "Maintain change event subscription registries to prevent orphaned subscriptions from accumulating." ], "failure_signals": [ "MDM hub downtime causing AI pipeline failures without prior notification to pipeline owners.", "Change event delivery failures unreported for more than one hour.", "Subscription registries with stale entries pointing to decommissioned AI pipeline endpoints." ] }, "legal_counsel": { "summary": "Master data errors propagate into every AI decision that consumes them — and into the representations the organization makes about people and counterparties. Counsel cares that golden records are correct where legal accuracy duties attach.", "actions": [ "Identify master data entities subject to accuracy obligations — customers, counterparties, sanctioned parties.", "Review the correction workflow so data-subject rectification rights propagate to golden records and downstream AI consumers.", "Assess exposure when AI decisions were made on stale or incorrect master data." ], "failure_signals": [ "Rectified personal data persisting in downstream AI features after correction at source.", "Sanctions or KYC screening running on unreconciled duplicate masters.", "No traceability from an AI decision back to the master-data version it used." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Enterprise MDM programs exist in most large organizations but are rarely extended to govern master data as an explicit input dependency of AI systems, with version traceability into model artifacts." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "federated-enterprise", "high-risk-sector" ], "implementers": [ "Data Governance Team", "Data Engineering", "Platform Engineering" ], "frameworks": [ { "framework": "dama_dmbok", "requirement_id": "Ch. 10", "fit": "direct", "rationale": "DAMA DMBOK2 Chapter 10 is the definitive reference for master data management, covering golden record management, hub architecture, stewardship, and conflict resolution. The requirement to designate authoritative sources and version golden records directly implements DMBOK's MDM capability framework. DMBOK treats MDM maturity as a core data governance indicator assessed in DCAM and similar frameworks.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "dcam", "requirement_id": "Capability 5.2", "fit": "direct", "rationale": "DCAM v2.2 capability 5.2 requires data quality to be profiled and measured against defined criteria. Master data management depends on this measurement to certify golden records and detect divergence in consuming systems.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_8000", "requirement_id": "ISO 8000-110", "fit": "partial", "rationale": "ISO 8000-110 specifies syntax and semantic-encoding requirements for exchanging master data between systems. Master data management for AI relies on these exchange requirements to keep golden records consistent across consuming pipelines.", "normative_force": "voluntary-standard", "source_version": "2015", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "databricks_unity", "requirement_id": "Unity Catalog", "fit": "partial", "rationale": "Databricks Unity Catalog provides a centralized governance layer for data assets including reference tables and lookup datasets consumed by AI training workflows. Unity Catalog's table versioning and access control features partially implement MDM governance for Databricks-based AI pipelines. Enterprises using Databricks should configure Unity Catalog as the authoritative interface for master data access within Databricks workloads.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "snowflake_horizon", "requirement_id": "Object tagging", "fit": "partial", "rationale": "Snowflake Horizon object tagging lets governance teams tag authoritative master data objects and propagate that designation to consumers, making the golden-record source discoverable and enforceable in queries and policies.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://data/controls/DM-04", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every master data domain consumed by AI training or inference systems must have a designated authoritative source and MDM steward in the master data domain registry; all production AI training runs must embed the specific golden record version ID in training metadata; and change notification events must be delivered to all subscribed AI pipelines within the defined SLA when master data records are updated.", "evidence_required": [ "Master data domain registry export listing each AI-consumed domain with designated authoritative source system, steward assignment, MDM hub endpoint reference, and conflict resolution SLA", "AI model training run metadata records showing golden record version IDs for each master data domain consumed, confirming version pinning was active and captured during training execution", "Change notification event delivery logs for master data domains consumed by active AI models, showing event publish timestamps and delivery confirmation to each subscribed pipeline endpoint", "Conflict resolution adjudication log for the past six months showing open conflict creation dates, adjudication dates, and SLA compliance ratios for each AI-consumed domain", "MDM hub audit trail showing golden record version history for AI-consumed domains with change event timestamps enabling before-and-after record comparison" ], "machine_tests": [ "Execute an AI training pipeline run against the MDM hub test environment → assert training run metadata includes golden record version IDs for all master data domains declared in the pipeline configuration", "Publish a test golden record update to a master data domain with registered AI pipeline subscribers → assert change notification event is delivered to all subscriber endpoints within the defined SLA", "Attempt to configure an AI training pipeline to query a source system database directly for a designated master data domain → assert pipeline configuration validation rejects non-MDM hub endpoints for governed master data domains", "Query the MDM hub conflict log for AI-consumed domains → assert zero unresolved conflicts exceed the defined adjudication SLA threshold" ], "human_review": [ "Review AI model artifact metadata for a sample of five production training runs and verify master data version references are present and resolve to specific golden record snapshots in the MDM hub audit trail, not just version labels", "Assess the conflict resolution adjudication process for AI-consumed master data domains, verifying that data steward adjudication is documented with business rationale and conflicts are not resolved by automatic default selection", "Review the change notification subscription registry to confirm all active AI pipelines are subscribed to domains they consume, with no orphaned subscriptions pointing to decommissioned pipeline endpoints" ], "blocking_effect": "blocks-deployment", "normative_status": "industry-framework", "anti_patterns": [ "Consuming master data via direct queries to source system databases rather than MDM hub APIs, bypassing versioning, conflict resolution, and change notification infrastructure entirely", "Training AI models without capturing the specific golden record version used, making it impossible to reproduce training results or assess model impact when master data is subsequently modified", "Using a single shared service account for all AI pipelines to access master data rather than per-pipeline subscriptions, preventing targeted change notification delivery to specifically affected consumers", "Treating conflict resolution as an automated merge operation without data steward adjudication, allowing algorithmic resolution to silently propagate incorrect reference data to AI training pipelines", "Failing to notify downstream AI model owners when master data schema changes occur, causing silent data format mismatches that corrupt feature extraction during subsequent training runs" ], "update_status": "current", "layer_code": "DM" }, { "id": "DM-05", "layer": "DM", "plane": "control", "name": "Data Quality Standards and Target Setting", "plain": "Minimum data quality thresholds — completeness, accuracy, consistency, timeliness — must be defined for AI training datasets, validation sets, and inference inputs, with automated quality gates blocking pipeline progression when thresholds are not met.", "threat": { "tags": [ "training-data-degradation", "quality-drift", "garbage-in-garbage-out", "silent-model-performance-collapse" ], "desc": "AI models trained on data that silently degrades in quality produce outputs that are less accurate, more biased, or systematically wrong in ways that are not immediately visible. Inference pipelines fed low-quality inputs amplify errors into downstream decisions. Without defined thresholds and automated gates, quality degradation accumulates until model performance failures become externally visible, by which point downstream harms may already have occurred." }, "standard": [ { "id": "iso_8000", "section": "ISO 8000-8", "title": "Information and data quality — concepts and measuring" }, { "id": "dama_dmbok", "section": "Ch. 13", "title": "Data Quality Management framework" }, { "id": "eu_ai_act", "section": "Art. 10(2)(f)", "title": "Examination of training data in view of possible biases" }, { "id": "gdpr", "section": "Art. 5(1)(d)", "title": "Accuracy principle for personal data" } ], "sources": [ { "id": "google_dataplex_bigquery_2024", "title": "Google Cloud Dataplex & BigQuery", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://cloud.google.com/dataplex/docs/introduction", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "google_dataplex_bigquery_2024", "relationship": "informative_reference", "rationale": "Establishes Google Cloud Dataplex & BigQuery requirements informing the apeiris://data/controls/DM-05 Data Quality Standards and Target Setting control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Define data quality dimensions and target thresholds per AI data category. Implement automated quality profiling on pipeline ingestion with score calculation against defined dimensions. Integrate quality gate checks into pipeline orchestration to block downstream steps when scores fall below thresholds. Publish quality dashboards with trend analysis.", "steps": [ "Define quality dimensions and numeric thresholds for each AI data category — completeness %, accuracy rate, consistency score, timeliness in hours — in a quality standards registry.", "Implement automated data quality profiling using a DQ framework (Great Expectations, dbt tests, Soda, or cloud-native equivalents) integrated into ingestion and transformation pipelines.", "Configure pipeline orchestration to enforce quality gates: block training runs when quality scores fall below defined thresholds and route failed datasets to a quality remediation workflow.", "Publish real-time quality dashboards showing current scores against thresholds for all active AI data pipelines.", "Conduct quarterly quality threshold reviews with model owners to adjust targets based on observed model performance correlations." ], "data_governance_officer": { "summary": "Quality thresholds are a governance commitment that must be set based on risk, not engineering convenience. Own the quality standards registry and ensure thresholds are reviewed when model performance issues are reported.", "actions": [ "Establish and publish the quality standards registry with thresholds defined for all AI data categories.", "Coordinate quarterly threshold reviews with model owners and data stewards.", "Escalate persistent quality threshold failures to data owners for root cause remediation." ], "failure_signals": [ "AI data categories with no documented quality thresholds.", "Quality thresholds set to 0% or otherwise trivially met with no governance review.", "Persistent threshold failures with no escalation to data owners." ] }, "data_engineer": { "summary": "Quality gates must be built into pipelines as first-class checks, not optional validation steps. Automated profiling should run before any data reaches training or inference workloads.", "actions": [ "Integrate quality profiling frameworks into all ingestion and transformation pipelines with quality scores emitted to the monitoring stack.", "Implement quality gate logic in pipeline orchestration to halt execution and alert when threshold breaches occur.", "Design remediation queues for datasets that fail quality gates, including automated notification to data owners." ], "failure_signals": [ "Training runs completing with quality gate checks disabled or bypassed.", "Quality profiling not running on inference input streams for real-time AI systems.", "Quality threshold failures generating no alerts or remediation tickets." ] }, "grc_auditor": { "summary": "Quality threshold governance is audited by verifying that thresholds exist, gates are enforced, and failures trigger remediation rather than silent bypass.", "actions": [ "Request the quality standards registry and verify thresholds are defined for all active AI data categories.", "Review pipeline execution logs for evidence that quality gates ran and blocked at least one pipeline in the review period (or confirm no failures occurred).", "Sample five training runs and confirm quality scores at the time of execution are recorded and met defined thresholds." ], "metrics": [ "Quality standards coverage: target 100% of AI data categories with documented thresholds.", "Quality gate enforcement rate: target 100% of pipeline runs with quality gate execution logged.", "Threshold breach remediation SLA: target ≤ 5 business days from breach to resolved status." ], "failure_signals": [ "AI data categories with no quality thresholds in the registry.", "Pipeline execution logs showing quality gate steps skipped or disabled.", "Threshold breach tickets unresolved beyond the defined SLA with no escalation." ] }, "it_operations": { "summary": "Quality profiling infrastructure — DQ frameworks, monitoring agents, dashboards — must be maintained with the same reliability standards as production data pipelines.", "actions": [ "Monitor quality profiling job health and alert on execution failures within 30 minutes.", "Maintain quality dashboard availability at the same SLA as production monitoring systems.", "Ensure quality profiling infrastructure scales with AI pipeline data volume growth." ], "failure_signals": [ "Quality profiling jobs failing silently for more than one pipeline execution cycle.", "Quality dashboards unavailable during an active threshold breach incident.", "Profiling infrastructure capacity causing false threshold failures due to timeouts rather than actual data quality issues." ] }, "legal_counsel": { "summary": "Quality thresholds are where 'accurate' stops being an aspiration and becomes a testable claim. Counsel ensures thresholds on regulated data reflect legal accuracy duties, not just engineering convenience.", "actions": [ "Review quality targets for datasets under GDPR accuracy and EU AI Act data-governance duties.", "Confirm threshold breaches on regulated data trigger documented remediation rather than silent waiver.", "Use quality measurement records as evidence in accuracy-related disputes and filings." ], "failure_signals": [ "Quality targets on regulated datasets relaxed without legal review.", "Threshold breaches accumulating on data feeding consequential decisions.", "Accuracy claims in documentation with no measurement evidence behind them." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Basic data profiling is common but formalized quality thresholds with automated pipeline gates for AI-specific data are uncommon outside regulated sectors. Most AI teams rely on model performance degradation as a lagging indicator of data quality issues." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "Data Engineering", "Data Governance Team", "ML Engineering" ], "frameworks": [ { "framework": "iso_8000", "requirement_id": "ISO 8000-8", "fit": "direct", "rationale": "ISO 8000-8 defines concepts and methods for measuring information and data quality across syntactic, semantic and pragmatic dimensions. Data quality target setting adopts these measurable dimensions when defining thresholds for AI training and inference data.", "normative_force": "voluntary-standard", "source_version": "2015", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 13", "fit": "direct", "rationale": "DAMA DMBOK2 Chapter 13 is the comprehensive reference for data quality management, covering quality dimension definition, profiling, threshold setting, and remediation workflows. The control implements DMBOK's data quality management framework for AI-specific data assets. DMBOK treats quality threshold governance as a maturity indicator distinguishing managed from repeatable organizations.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "eu_ai_act", "requirement_id": "Art. 10(3)", "fit": "direct", "rationale": "EU AI Act Article 10(2)(f) requires data governance to include examination of training data in view of possible biases; the dataset quality criteria themselves — relevant, sufficiently representative, and to the best extent possible free of errors and complete — are stated in Article 10(3). Data quality target setting operationalizes both: thresholds implement the Art. 10(3) criteria and profiling for skew implements the Art. 10(2)(f) bias examination.", "normative_force": "binding-law", "source_version": "2024", "reviewed_on": "2026-07-08", "basis": "anchored", "correction": "ai-exchange-verify 2026-07-08", "relation": "satisfies" }, { "framework": "gdpr", "requirement_id": "Art. 5(1)(d)", "fit": "direct", "rationale": "GDPR Article 5(1)(d) establishes the accuracy principle, requiring personal data to be kept accurate and up to date with inaccurate data erased or rectified. AI training and inference datasets containing personal data must meet accuracy quality thresholds to comply with this principle. Automated quality gates that block inaccurate personal data from reaching AI systems operationalize this legal requirement.", "normative_force": "binding-law", "source_version": "2018", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "google_dataplex", "requirement_id": "Auto Data Quality", "fit": "partial", "rationale": "Google Cloud Dataplex Auto Data Quality provides managed data quality scanning and rule definition for BigQuery and Cloud Storage datasets. For enterprises on Google Cloud infrastructure, Dataplex quality rules and dashboards partially implement the quality profiling and threshold enforcement requirements of this control. The platform's integration with BigQuery and Vertex AI makes it relevant for Google-based AI training pipelines.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "dcam", "requirement_id": "Capability 5.1", "fit": "partial", "rationale": "DCAM v2.2 capability 5.1 requires the data quality management program to be established — including defined quality dimensions, standards and targets. Data quality target setting is the target-definition activity of that program; capability 4.3 concerns technology operating risk, not quality standards.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "owasp_ai_exchange", "requirement_id": "dataqualitycontrol", "fit": "direct", "rationale": "DM-05 defines minimum quality thresholds and blocks training progression when automated quality-gate scores fall below them, directly controlling training-data quality.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "mitre", "requirement_id": "AML.M0007", "fit": "supporting", "direction": "control-supports-requirement", "rationale": "Control \"Minimum data quality thresholds must be defined and published in the quality standards…\" enacts ATLAS mitigation AML.M0007 Sanitize Training Data; OpenCRE crosswalks this control’s OWASP AI Exchange concept (dataqualitycontrol) to this mitigation.", "source_version": "v2026.06", "reviewed_on": "2026-07-08", "source_status": "authoritative", "mapping_confidence": "medium", "legal_status": "not-applicable", "control_readiness": "approved", "implementation_maturity": "developing", "assurance_result": "not-assessed", "evidence_status": "no-evidence", "normative_force": "industry-framework", "basis": "asserted", "relation": "equivalent_to", "correction": "ap38-opencre-enhancement 2026-07-08" } ], "canonical_id": "apeiris://data/controls/DM-05", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Minimum data quality thresholds must be defined and published in the quality standards registry for each active AI data category across all required dimensions (completeness, accuracy, consistency, timeliness); automated quality gate checks must execute on every pipeline run and block training progression when scores fall below thresholds; and quality gate results with scores must be logged for every training run in the pipeline audit trail.", "evidence_required": [ "Quality standards registry export showing defined numeric thresholds for completeness, accuracy, consistency, and timeliness dimensions for each active AI data category, with governance review dates", "Pipeline execution audit logs for the past 90 days showing quality gate check results including dimension scores and pass/fail status for each training and inference pipeline run", "Quality gate enforcement evidence showing at least one threshold breach triggered a pipeline block and routed the dataset to the remediation workflow in the review period, or a clean run attestation if no breaches occurred", "Data quality profiling framework configuration showing profiling rules mapped to quality standards registry thresholds, with evidence that profiling runs execute on all active AI data pipelines", "Quarterly quality threshold review records showing Data Governance Officer sign-off and any threshold adjustments with business justification tied to observed model performance correlations" ], "machine_tests": [ "Inject a test dataset with completeness score below the defined threshold into an AI training pipeline → assert quality gate halts pipeline execution and generates a remediation ticket before the training step fires", "Submit a training pipeline run with quality gate checks disabled in the pipeline configuration → assert pipeline configuration validation rejects the run before execution begins", "Execute a quality profiling scan on a reference dataset with known quality characteristics → assert profiling dimension scores match expected values within defined tolerance, confirming profiling rule accuracy", "Query pipeline execution audit logs for the past 30 days → assert every training run record contains a quality_gate_result field with dimension scores and pass/fail status, with no runs missing this field" ], "human_review": [ "Review the quality standards registry and assess whether thresholds are set at risk-appropriate levels, sampling three data categories for threshold justification and verifying no thresholds are set at trivially low values such as 0% or single-field not-null checks", "Examine pipeline execution logs for threshold breach events and verify that datasets routed to remediation queues received resolution within the five-business-day SLA with documented root cause findings", "Assess the quarterly threshold review process to confirm model owners actively participate in threshold calibration based on observed model performance correlations with data quality scores, not just governance sign-off on unchanged values" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Setting data quality thresholds at 0% or not-null-only checks that all data trivially passes, creating quality gates that generate compliance evidence without providing any protection against degraded training data", "Running quality profiling as an optional advisory pre-run step that produces warnings but does not block pipeline execution, allowing training jobs to proceed on data that fails defined minimum quality standards", "Defining quality thresholds at system launch without quarterly review, causing thresholds to become misaligned with model performance requirements as data distributions and model architectures evolve over time", "Measuring quality only at training dataset ingestion while skipping quality checks on inference input streams for real-time AI systems, creating a governance gap where production quality degradation is undetected", "Logging quality gate pass/fail status without retaining the underlying dimension scores, preventing trend analysis and making it impossible to detect gradual quality degradation before a hard threshold breach occurs" ], "update_status": "current", "layer_code": "DM" }, { "id": "DM-06", "layer": "DM", "plane": "control", "name": "Data Schema Governance and Breaking Change Control", "plain": "All schemas for data assets consumed by AI training and inference pipelines must have designated owners, be version-controlled, and require approval for breaking changes, with downstream pipeline owners notified before any breaking change is deployed.", "threat": { "tags": [ "schema-drift", "undocumented-breaking-change", "downstream-pipeline-breakage", "silent-model-input-corruption" ], "desc": "Uncontrolled schema changes to upstream data sources silently corrupt AI pipeline inputs. A column rename or type change in a source table propagates downstream, causing training pipelines to fail on incorrect types, inference pipelines to read null values instead of feature data, or models to operate against silently wrong feature encodings. Without schema ownership and breaking-change gates, upstream teams make schema changes without awareness of their AI pipeline dependencies." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 8", "title": "Data architecture and schema management" }, { "id": "dcam", "section": "Capability 3.3", "title": "Identify the Data" }, { "id": "databricks_unity", "section": "Delta Lake schema enforcement and evolution", "title": "Table-format-level schema enforcement and evolution controls" }, { "id": "google_dataplex", "section": "BigQuery schema management", "title": "Controlled schema change in BigQuery plus catalog metadata" } ], "sources": [ { "id": "google_dataplex_bigquery_2024", "title": "Google Cloud Dataplex & BigQuery", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://cloud.google.com/dataplex/docs/introduction", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "google_dataplex_bigquery_2024", "relationship": "informative_reference", "rationale": "Establishes Google Cloud Dataplex & BigQuery requirements informing the apeiris://data/controls/DM-06 Data Schema Governance and Breaking Change Control control.", "reviewed_on": "2026-07-01" }, { "id": "snowflake_horizon_data_governance_2024", "title": "Snowflake Horizon (Data Governance)", "authority": "Snowflake Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.snowflake.com/en/data-cloud/workloads/data-governance/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "snowflake_horizon", "relationship": "informative_reference", "rationale": "Establishes Snowflake Horizon (Data Governance) requirements informing the apeiris://data/controls/DM-06 Data Schema Governance and Breaking Change Control control.", "reviewed_on": "2026-07-01" }, { "id": "databricks_unity_catalog_2024", "title": "Databricks Unity Catalog", "authority": "Databricks, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.databricks.com/en/data-governance/unity-catalog/index.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "databricks_unity_catalog_2024", "relationship": "informative_reference", "rationale": "Establishes Databricks Unity Catalog requirements informing the apeiris://data/controls/DM-06 Data Schema Governance and Breaking Change Control control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Register all AI-consumed schemas in a central schema registry with versioning and ownership assignment. Classify schema changes as breaking or non-breaking. Require impact assessment and downstream owner sign-off before deploying breaking changes. Publish schema change events to subscribed pipeline owners.", "steps": [ "Deploy a schema registry covering all data assets consumed by AI training and inference pipelines, capturing schema version history, owner, and downstream consumer list.", "Define the breaking change classification rules: column deletion, type narrowing, rename without alias, and removal of required fields are breaking; adding nullable columns and backward-compatible type expansions are non-breaking.", "Implement a change approval workflow requiring schema owners to identify downstream AI pipeline consumers and obtain sign-off from pipeline owners before deploying breaking changes.", "Publish schema change events to downstream pipeline owners via a notification channel — email, Slack, JIRA ticket — on every schema version increment.", "Integrate schema compatibility checks into CI/CD pipelines for schema changes so that incompatible schema deployments are blocked without an approved breaking-change ticket." ], "data_governance_officer": { "summary": "Schema governance is a dependency management problem as much as a data quality problem. Ensure schema ownership is assigned and that the breaking-change approval process is enforced rather than bypassed under time pressure.", "actions": [ "Establish and publish the schema governance policy defining ownership, versioning, and breaking-change approval requirements.", "Review quarterly schema change logs to confirm breaking changes were approved and downstream owners notified.", "Escalate repeated bypasses of the breaking-change process for process reinforcement or tooling enforcement." ], "failure_signals": [ "Active AI pipeline schemas with no designated owner in the schema registry.", "Breaking changes deployed to production without an approved change ticket.", "Downstream pipeline owners reporting they were not notified of schema changes that affected their systems." ] }, "data_engineer": { "summary": "Schema registries are only effective if pipelines enforce compatibility at runtime. Implement schema validation at pipeline ingestion so incompatible schema versions fail fast rather than silently corrupting features.", "actions": [ "Implement schema validation checks at the start of each AI pipeline execution that compare the incoming schema version against the expected version and fail if incompatible.", "Register all AI data pipeline schemas in the schema registry with owner assignments and downstream consumer lists at pipeline onboarding.", "Automate breaking-change detection in CI/CD using schema compatibility tools (Confluent Schema Registry, Buf, or cloud-native equivalents)." ], "failure_signals": [ "Pipeline executions processing data under an incompatible schema version without a validation error.", "Schema registry entries for active AI pipelines not updated following schema changes.", "CI/CD schema compatibility checks disabled for any AI data pipeline." ] }, "grc_auditor": { "summary": "Schema governance is audited by verifying ownership coverage, change log completeness, and evidence that breaking changes went through the approval process.", "actions": [ "Request the schema registry and verify designated owners for all schemas consumed by active AI production pipelines.", "Review the schema change log for the past 12 months and identify any breaking changes deployed without approved change tickets.", "Confirm that schema compatibility checks are active in CI/CD pipelines for AI data pipeline repositories." ], "metrics": [ "Schema ownership coverage: target 100% of AI-consumed schemas with designated owner.", "Breaking-change compliance rate: target 100% of breaking changes deployed with approved tickets.", "Downstream notification delivery rate: target ≥ 99% of schema change events delivered to registered consumers." ], "failure_signals": [ "Schemas in active AI pipelines with no owner assignment.", "Breaking changes deployed without evidence of impact assessment or downstream owner sign-off.", "Schema change events not delivered to downstream consumers due to stale subscription registries." ] }, "it_operations": { "summary": "Schema registry availability is a dependency for pipeline deployment. Ensure the registry service is monitored and that CI/CD integration points are maintained.", "actions": [ "Monitor schema registry service availability and alert on outages within 15 minutes.", "Maintain CI/CD integration points for schema compatibility checks across all AI pipeline repositories.", "Ensure schema change event notification infrastructure is monitored for delivery failures." ], "failure_signals": [ "Schema registry outages causing CI/CD pipeline deployments to bypass compatibility checks.", "Schema change event notification failures unreported for more than one hour.", "CI/CD schema compatibility check integrations broken in AI pipeline repositories without incident tracking." ] }, "legal_counsel": { "summary": "Schema changes can silently change the meaning of regulated fields — what counts as consent, residency, or a protected attribute. Counsel reviews breaking changes that alter legally significant semantics.", "actions": [ "Flag schema fields with legal significance — consent flags, jurisdiction, retention markers — for mandatory review on change.", "Review breaking changes that redefine or drop legally significant fields before approval.", "Confirm deprecation timelines preserve continuity of compliance-relevant history." ], "failure_signals": [ "Consent or retention fields dropped or re-typed without legal sign-off.", "Historical compliance queries broken by unreviewed schema changes.", "Legally significant semantic changes shipped as 'non-breaking'." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Schema registries exist in streaming-first architectures using Kafka but are rarely applied to AI training data pipelines. Breaking-change governance for AI data dependencies is typically informal, relying on coordination between teams rather than enforced process." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "federated-enterprise", "multi-tenant" ], "implementers": [ "Data Engineering", "Platform Engineering", "DevOps" ], "frameworks": [ { "framework": "dama_dmbok", "requirement_id": "Ch. 8", "fit": "direct", "rationale": "DAMA DMBOK2 Chapter 8 addresses data architecture and schema management as a core data management discipline, covering schema standards, ownership, and change management. The control implements DMBOK's requirement for formal schema governance including versioning and change approval processes. DMBOK treats schema management maturity as an indicator of overall data architecture governance capability.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "dcam", "requirement_id": "Capability 3.3", "fit": "direct", "rationale": "DCAM v2.2 capability 3.3 (Identify the Data) requires data structures and definitions to be identified and agreed. Schema governance keeps those agreed definitions stable, making breaking changes explicit and controlled.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "databricks_unity", "requirement_id": "Delta Lake schema enforcement and evolution", "fit": "partial", "rationale": "Schema enforcement and controlled schema evolution are Delta Lake table-format features — a Linux Foundation open-source project — surfaced through Databricks; Unity Catalog adds the governance layer over those tables. Breaking-change control builds on the format-level enforcement.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "google_dataplex", "requirement_id": "BigQuery schema management", "fit": "partial", "rationale": "Dataplex ships no schema-registry product; schema change control on Google Cloud is implemented through BigQuery schema management (controlled column addition and relaxation rules) plus Dataplex/Knowledge Catalog metadata. DM-06 implementations should anchor on those real capabilities.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "snowflake_horizon", "requirement_id": "Access History", "fit": "partial", "rationale": "Snowflake Access History records which objects and columns are read and written by which workloads; before a breaking schema change, it provides the dependency evidence needed to assess downstream impact.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "eu_ai_act", "requirement_id": "Art. 10(2)", "fit": "adjacent", "rationale": "EU AI Act Article 10(2) requires documented data governance practices for high-risk AI systems covering data management processes. Schema governance documentation — ownership records, change logs, approval workflows — contributes to the technical documentation required under Article 11. While not a direct schema governance requirement, maintaining schema change records supports conformity assessment evidence.", "normative_force": "binding-law", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" } ], "canonical_id": "apeiris://data/controls/DM-06", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "All schemas for AI-consumed data assets must have a designated owner registered in the schema registry, and every breaking schema change deployed to production must have a documented approval ticket with downstream pipeline owner sign-off recorded before deployment. Schema compatibility checks in CI/CD must block incompatible schema deployments without an approved ticket.", "evidence_required": [ "schema_registry_export listing each AI-consumed schema with designated owner, current version, and downstream consumer list — confirming 100% ownership coverage", "breaking_change_approval_ticket for each breaking change deployed in the audit period, containing impact assessment and explicit downstream pipeline owner sign-offs", "ci_cd_schema_compatibility_check_logs confirming compatibility enforcement was active and un-bypassed for every AI pipeline deployment in the audit period", "schema_change_event_delivery_log showing notification delivery to registered downstream consumers on each schema version increment" ], "machine_tests": [ "Attempt to deploy a breaking schema change (column deletion) to a CI/CD environment without an approved change ticket → assert deployment is blocked with error=schema_change_not_approved", "Query the schema registry for all AI-consumed data assets → assert each asset has a non-null owner field and at least one registered downstream consumer", "Submit a type-narrowing schema change to the CI/CD compatibility checker → assert the check returns INCOMPATIBLE status and halts the pipeline deployment", "Simulate a schema version increment and verify the notification system → assert all registered downstream consumers receive a change event within the defined delivery window" ], "human_review": [ "Review the schema change log for the past 12 months and verify every breaking change has a corresponding approved ticket with documented downstream owner sign-off before the deployment timestamp", "Assess whether the breaking vs. non-breaking change classification rules are clearly defined, consistently applied, and enforced in tooling rather than relying on per-engineer judgment", "Verify that the breaking-change approval workflow cannot be bypassed under release pressure by reviewing any exceptions or emergency-change records for completeness and post-hoc approval evidence" ], "blocking_effect": "blocks-deployment", "normative_status": "industry-framework", "anti_patterns": [ "Classifying column renames as non-breaking changes without providing backward-compatible aliases, silently breaking downstream AI pipeline feature ingestion without triggering the approval gate", "Assigning schema ownership to a team or role rather than a named individual, making it impossible to determine accountable ownership or obtain the correct sign-off during a breaking-change review", "Bypassing the breaking-change approval workflow under release pressure by temporarily misclassifying changes as non-breaking without a formal classification review", "Relying on post-deployment notifications to inform downstream consumers of breaking changes rather than requiring pre-deployment sign-off, allowing pipeline breakage before consumers are aware", "Maintaining the schema registry as a manual spreadsheet rather than an API-driven registry integrated with CI/CD, resulting in stale consumer lists and missed compatibility enforcement" ], "update_status": "current", "layer_code": "DM" }, { "id": "DM-07", "layer": "DM", "plane": "control", "name": "Synthetic Data Governance", "plain": "Synthetic datasets used for AI training must be governed for representativeness of the target distribution, bias inheritance from the generative model, copyright status of the generation method, and re-identification risk before being approved for training use.", "threat": { "tags": [ "bias-inheritance", "re-identification-via-synthesis", "copyright-infringement", "representativeness-failure" ], "desc": "Synthetic data generated from biased source datasets inherits and can amplify the source bias, producing models that discriminate in ways the organization cannot attribute to real data. Synthetic records generated from small, high-dimensional real datasets may be re-identifiable through model inversion or membership inference attacks, creating privacy liability despite the synthetic label. Synthetic data generated using copyrighted source materials or proprietary generative models may create downstream intellectual property exposure for models trained on that data." }, "standard": [ { "id": "eu_ai_act", "section": "Art. 10(2)(f)", "title": "Examination of training data in view of possible biases" }, { "id": "gdpr", "section": "Art. 5(1)(a)", "title": "Lawfulness and fairness principles applied to synthetic data" }, { "id": "iso_8000", "section": "ISO 8000-8", "title": "Information and data quality — concepts and measuring" }, { "id": "nist_pf", "section": "CT.DP-P2", "title": "Data processed to limit identification of individuals" } ], "sources": [ { "id": "anthropic_privacy_policy_2024", "title": "Anthropic Data Handling & Privacy Policy", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-09-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/privacy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_privacy_policy_2024", "relationship": "informative_reference", "rationale": "Establishes Anthropic Data Handling & Privacy Policy requirements informing the apeiris://data/controls/DM-07 Synthetic Data Governance control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Establish a synthetic data approval workflow requiring representativeness testing, bias inheritance assessment, copyright clearance of the generation method, and re-identification risk scoring before a synthetic dataset is approved for training use. Results are recorded in the data catalog alongside the synthetic dataset.", "steps": [ "Define the mandatory synthetic data governance checklist: representativeness test (distribution comparison against the target population), bias inheritance assessment (comparison of demographic distributions between source and synthetic datasets), copyright clearance (documentation of the generative model's training data provenance and license), and re-identification risk scoring (differential privacy epsilon value or membership inference attack success rate).", "Implement automated representativeness and bias testing using statistical comparison tools integrated into the synthetic data generation pipeline.", "Require legal counsel review of copyright clearance documentation for synthetic datasets generated using external or commercial generative models.", "Define re-identification risk thresholds and block training use of synthetic datasets that exceed the defined risk threshold without a documented risk acceptance sign-off.", "Record all governance assessment results — representativeness score, bias delta, copyright status, re-identification risk score, approval status — in the data catalog entry for the synthetic dataset." ], "data_governance_officer": { "summary": "Synthetic data is not automatically safe data. Govern it with the same rigor as real data, with additional checks for generation-method provenance and re-identification risk.", "actions": [ "Establish and publish the synthetic data governance policy and mandatory checklist.", "Ensure all synthetic datasets in use are registered in the catalog with governance assessment results attached.", "Escalate synthetic datasets failing re-identification thresholds for risk acceptance review by the data governance committee." ], "failure_signals": [ "Synthetic datasets in use for AI training with no governance assessment recorded.", "Re-identification risk scores above threshold approved without documented risk acceptance.", "No legal review of copyright clearance documentation for synthetic data generated with commercial models." ] }, "data_engineer": { "summary": "Representativeness and bias tests should run automatically during synthetic data generation, not as post-hoc manual reviews. Build governance checks into the generation pipeline.", "actions": [ "Integrate statistical representativeness and bias comparison tests into the synthetic data generation pipeline, blocking output if tests fail.", "Emit governance assessment results as structured metadata at generation time for catalog registration.", "Implement re-identification risk scoring using differential privacy libraries or membership inference probes as part of the generation workflow." ], "failure_signals": [ "Synthetic datasets generated without automated representativeness and bias test results.", "Re-identification risk scoring not present in synthetic dataset metadata.", "Generation pipelines bypassing governance checks when operating under time pressure." ] }, "legal_counsel": { "summary": "Copyright and privacy law create specific obligations for synthetic data. Review generation method provenance and confirm that the use of synthetic data does not create IP or privacy liability.", "actions": [ "Review copyright clearance documentation for all generative models used to produce AI training datasets, confirming the model's training data license permits commercial use.", "Assess whether GDPR or CCPA obligations apply to synthetic datasets that may be re-identifiable, and advise on whether the synthetic data qualifies as personal data.", "Document legal risk assessments for synthetic datasets generated using foundation models with uncertain training data provenance." ], "failure_signals": [ "Synthetic data generated using commercial foundation models with no copyright clearance review.", "Re-identifiable synthetic datasets treated as out of scope for GDPR without a legal assessment.", "No legal review process triggered when new external generative models are onboarded." ] }, "grc_auditor": { "summary": "Synthetic data governance is audited by verifying that assessment results exist in the catalog and that approval decisions are documented with evidence.", "actions": [ "Request the catalog entries for all synthetic datasets used in training runs in the past 12 months and verify governance assessment results are attached.", "Review re-identification risk scores and confirm datasets above threshold were either rejected or approved via the risk acceptance process.", "Verify legal counsel reviewed copyright clearance for synthetic datasets generated using external generative models." ], "metrics": [ "Synthetic dataset governance coverage: target 100% of training-use synthetic datasets with catalog entry and assessment results.", "Re-identification risk compliance rate: target 100% of above-threshold datasets with documented risk acceptance.", "Copyright clearance rate: target 100% of externally generated synthetic datasets with legal review documented." ], "failure_signals": [ "Training runs using synthetic datasets with no governance assessment in the catalog.", "Above-threshold re-identification risk approved without a documented risk acceptance record.", "No legal counsel review of copyright clearance in the governance record." ] }, "it_operations": { "summary": "Synthetic data pipelines are production infrastructure: generators, seeds and versions must be operated, versioned and isolated so synthetic outputs are reproducible and cannot contaminate real-data stores.", "actions": [ "Version and pin synthetic data generators, configurations and seed datasets for reproducibility.", "Segregate synthetic outputs in clearly labeled storage with propagated markers.", "Monitor for synthetic data leaking into real-data training stores and vice versa." ], "failure_signals": [ "Synthetic datasets whose generator version or seed data cannot be identified.", "Synthetic records found unlabeled inside production training corpora.", "Regeneration producing materially different data with no configuration change recorded." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Synthetic data use in AI training is growing rapidly but governance frameworks for synthetic data are nascent. Most organizations treat synthetic data as inherently safe and do not apply re-identification or bias inheritance assessment." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "eu-high-risk-ai", "high-risk-sector", "cloud-native", "federated-enterprise" ], "implementers": [ "Data Engineering", "Legal Counsel", "ML Engineering" ], "frameworks": [ { "framework": "eu_ai_act", "requirement_id": "Art. 10(2)(f)", "fit": "direct", "rationale": "EU AI Act Article 10(2)(f) requires examination of training data in view of possible biases likely to affect health, safety or fundamental rights. Synthetic data governance must demonstrate that generated data does not introduce or amplify such biases, making the bias-examination requirement directly applicable to synthetic corpora.", "normative_force": "binding-law", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "gdpr", "requirement_id": "Art. 5(1)(a)", "fit": "partial", "rationale": "GDPR Article 5(1)(a) requires lawful, fair and transparent processing. Where synthetic data is generated from personal data, the generation step is itself processing of personal data and needs a lawful basis; fairness further requires that synthetic derivatives not be used to circumvent data-subject rights. Synthetic data governance documents both.", "normative_force": "binding-law", "source_version": "2018", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_pf", "requirement_id": "CT.DP-P2", "fit": "direct", "rationale": "NIST Privacy Framework CT.DP-P2 requires processing that limits the identification of individuals. Synthetic data generation is such a disassociation technique — but only if governance verifies the synthetic output cannot re-identify individuals from the source data.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_8000", "requirement_id": "ISO 8000-8", "fit": "partial", "rationale": "ISO 8000-8's syntactic, semantic and pragmatic quality measures apply to generated data as much as to collected data: synthetic datasets must meet the same measurable quality criteria before standing in for real data.", "normative_force": "voluntary-standard", "source_version": "2015", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 14", "fit": "adjacent", "rationale": "DAMA DMBOK2 Chapter 14 addresses big data and emerging data sources governance, providing frameworks for governing novel data sources including synthetically generated data. While DMBOK predates widespread AI synthetic data use, its principles for governance of non-traditional data sources apply. The DMBOK's data governance framework provides the process model for the synthetic data approval workflow.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "anthropic_privacy", "requirement_id": "Privacy Policy", "fit": "adjacent", "rationale": "Anthropic's Privacy Policy documents provider-side data handling commitments at document level. For synthetic data governance it serves as a reference disclosure format; it does not itself specify synthetic data controls.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "owasp_ai_exchange", "requirement_id": "obfuscatetrainingdata", "fit": "supporting", "rationale": "DM-07 gates synthetic training datasets on a re-identification risk score below threshold, governing synthetic data as a privacy-preserving substitute for real training data.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://data/controls/DM-07", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every synthetic dataset approved for AI training use must have a data catalog entry containing a passing representativeness score, a bias delta within defined thresholds, a copyright clearance status, and a re-identification risk score below the defined threshold — or a risk acceptance record signed by the data governance committee. Training job launchers must block runs referencing synthetic datasets without an approved catalog entry.", "evidence_required": [ "synthetic_dataset_catalog_entry for each training-use synthetic dataset with representativeness_score, bias_delta, copyright_clearance_status, re_identification_risk_score, and approval_status fields populated", "bias_inheritance_assessment_report comparing demographic feature distributions between source and synthetic dataset with statistical methodology and threshold comparison results", "re_identification_risk_record containing differential_privacy_epsilon_value or membership_inference_attack_success_rate with methodology documentation and threshold comparison", "copyright_clearance_review_record signed by legal counsel for synthetic datasets generated using external or commercial generative models, confirming the model's license permits the intended training use", "risk_acceptance_record signed by the data governance committee for any synthetic dataset approved above the re-identification risk threshold, including documented residual risk" ], "machine_tests": [ "Submit a synthetic dataset to the generation pipeline without a representativeness test result → assert pipeline output is blocked with status=missing_governance_assessment", "Query the data catalog for all synthetic datasets referenced in training runs in the past 12 months → assert 100% have catalog entries with all mandatory governance fields populated and non-null", "Trigger a training job referencing a synthetic dataset with re-identification risk above the defined threshold and no risk acceptance record → assert the job launcher blocks with error=synthetic_data_governance_not_satisfied", "Attempt to register a synthetic dataset with a bias delta exceeding the defined maximum → assert the catalog registration is rejected with status=bias_threshold_exceeded" ], "human_review": [ "Review legal counsel sign-off records for copyright clearance of synthetic datasets generated using external or commercial generative models and confirm all were reviewed before the dataset was approved for training use", "Assess the re-identification risk scoring methodology for statistical rigor and verify that defined thresholds are appropriate for the sensitivity classification of the underlying source data", "Verify that the synthetic data governance checklist is enforced programmatically by the generation pipeline rather than relying on manual completion by data engineers at their discretion" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Treating synthetic data as inherently privacy-safe and skipping re-identification risk scoring because the dataset was generated rather than collected from individuals", "Using bias metrics from the source dataset as a proxy for the synthetic dataset's bias profile instead of directly measuring demographic feature distributions in the generated output", "Accepting copyright clearance at the generative model product level without reviewing whether the model's training data license explicitly permits downstream commercial AI training on outputs", "Storing synthetic datasets in the data lake without a catalog entry, making governance assessment results inaccessible during training job validation and audit", "Approving synthetic datasets for training use based on informal engineering review rather than the mandatory governance checklist, with no signed approval record" ], "update_status": "current", "layer_code": "DM" }, { "id": "DM-08", "layer": "DM", "plane": "both", "name": "Data Management Evidence Package", "plain": "A DM-layer evidence package must be compiled and maintained, aggregating attestation artifacts from DM-01 through DM-07 to demonstrate that AI data assets are cataloged, classified, lifecycle-governed, quality-assured, schema-controlled, and synthetically governed.", "threat": { "tags": [ "evidence-gap", "audit-failure", "unattested-data-management", "compliance-documentation-drift" ], "desc": "Without a compiled evidence package, individual DM controls may be implemented but their collective adequacy cannot be demonstrated to regulators, auditors, or counterparties. Fragmented evidence scattered across catalog exports, quality dashboards, and change logs cannot be efficiently produced during regulatory examinations or litigation. Evidence documentation that drifts out of date creates the appearance of non-compliance even when controls are technically operating." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 3", "title": "Data governance documentation and evidence requirements" }, { "id": "eu_ai_act", "section": "Art. 11", "title": "Technical documentation and conformity assessment records" }, { "id": "cobit_2019", "section": "MEA01", "title": "Monitor and evaluate performance and conformance" }, { "id": "dcam", "section": "Capability 2.1", "title": "The data management program is established and funded" } ], "sources": [ { "id": "eu_ai_act", "title": "EU AI Act", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2024/1689", "published_on": "2024-07-12", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2024/1689/oj/eng", "license": "open-access", "status": "current", "flagship": false, "source_id": "eu_ai_act", "relationship": "normative_requirement", "rationale": "Establishes EU AI Act Art. 11 requirements informing the apeiris://data/controls/DM-08 Data Management Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "dama_dmbok", "title": "DAMA-DMBOK 2nd Ed", "authority": "DAMA International", "source_type": "standard", "normative_force": "best-practice", "version": "2 (2nd Edition, Revised 2024)", "published_on": "2017-07-05", "retrieved_on": "2026-07-02", "canonical_url": "https://www.dama.org/cpages/body-of-knowledge", "license": "proprietary", "status": "current", "flagship": false, "source_id": "dama_dmbok", "relationship": "informative_reference", "rationale": "Establishes DAMA-DMBOK 2nd Ed Ch. 3 requirements informing the apeiris://data/controls/DM-08 Data Management Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "cobit_2019", "title": "COBIT 2019", "authority": "ISACA", "source_type": "standard", "normative_force": "industry-framework", "version": "2019", "published_on": "2018-11-12", "retrieved_on": "2026-07-02", "canonical_url": "https://www.isaca.org/resources/cobit", "license": "proprietary", "status": "current", "flagship": false, "source_id": "cobit_2019", "relationship": "informative_reference", "rationale": "Establishes COBIT 2019 MEA01 requirements informing the apeiris://data/controls/DM-08 Data Management Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "dcam", "title": "DCAM v2.2", "authority": "EDM Council", "source_type": "standard", "normative_force": "industry-framework", "version": "2.2", "published_on": "2022-01-01", "retrieved_on": "2026-07-02", "canonical_url": "https://edmcouncil.org/frameworks/dcam/", "license": "proprietary", "status": "current", "flagship": false, "source_id": "dcam", "relationship": "informative_reference", "rationale": "Establishes DCAM v2.2 Capability 2.1 requirements informing the apeiris://data/controls/DM-08 Data Management Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "gdpr", "title": "GDPR", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2016/679", "published_on": "2016-04-27", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj", "license": "open-access", "status": "current", "flagship": false, "source_id": "gdpr", "relationship": "normative_requirement", "rationale": "Establishes GDPR Art. 5(2) requirements informing the apeiris://data/controls/DM-08 Data Management Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "iso_8000", "title": "ISO 8000 Data Quality", "authority": "ISO", "source_type": "standard", "normative_force": "voluntary-standard", "version": "multi-part (8:2015, 61:2016, 110:2021, 120:2016)", "published_on": "2022-01-01", "retrieved_on": "2026-07-02", "canonical_url": "https://www.iso.org/standard/60805.html", "license": "proprietary", "status": "current", "flagship": false, "source_id": "iso_8000", "relationship": "implementation_pattern", "rationale": "Establishes ISO 8000 Data Quality ISO 8000-61 requirements informing the apeiris://data/controls/DM-08 Data Management Evidence Package control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Compile a DM evidence package at minimum quarterly, aggregating catalog exports (DM-01), classification coverage reports (DM-02), retention schedule registry and disposal logs (DM-03), MDM golden record status (DM-04), quality threshold compliance reports (DM-05), schema change logs (DM-06), and synthetic data governance records (DM-07) into a structured package with a signed attestation statement.", "steps": [ "Define the DM evidence package template specifying the required artifact from each DM control (DM-01 through DM-07), the format, the currency requirement (maximum age of each artifact), and the responsible contributor.", "Automate artifact collection from source systems — catalog exports, quality dashboards, schema registries, disposal logs — into a centralized evidence repository on a quarterly schedule.", "Produce a DM-layer attestation statement signed by the Data Governance Officer confirming that all required artifacts were collected, reviewed, and that material gaps are documented with remediation plans.", "Store the compiled evidence package in an access-controlled, tamper-evident repository with a minimum three-year retention period to support regulatory examination and litigation response.", "Register the evidence package completion in the enterprise GRC system and link it to the DM-08 control as the primary compliance artifact." ], "data_governance_officer": { "summary": "The DM evidence package is your primary compliance deliverable for the DM layer. Own the attestation statement and ensure the package is current, complete, and stored where it can be retrieved under regulatory demand.", "actions": [ "Review and sign the DM-layer attestation statement quarterly, confirming all artifact currency requirements are met.", "Escalate material gaps — controls with missing or expired artifacts — to data owners for remediation before the attestation is issued.", "Maintain a three-year archive of DM evidence packages in the access-controlled evidence repository." ], "failure_signals": [ "DM evidence package not compiled in the past quarter.", "Attestation statement signed with known material gaps not documented in the package.", "Evidence package not retrievable within 48 hours when requested by legal or compliance." ] }, "legal_counsel": { "summary": "The DM evidence package is a litigation and regulatory response asset. Confirm that the package satisfies documentation obligations under applicable laws and that it is stored with appropriate legal hold controls.", "actions": [ "Review the DM evidence package template to confirm it covers the documentation requirements of applicable laws — EU AI Act Art. 11, GDPR accountability obligations, sector-specific data governance regulations.", "Confirm that the evidence repository is subject to legal hold procedures and that retention periods align with applicable statutes of limitations.", "Advise on privilege protection for attestation statements and gap analysis documents produced in anticipation of regulatory examination." ], "failure_signals": [ "DM evidence packages stored outside the legal hold infrastructure.", "Package template not reviewed by legal counsel to confirm regulatory documentation sufficiency.", "Gap analysis documents stored in the evidence package without privilege review." ] }, "grc_auditor": { "summary": "The DM evidence package is the primary artifact for DM-layer audit. Review it for completeness, artifact currency, and attestation statement integrity — then cross-reference sampled artifacts against source systems.", "actions": [ "Request the most recent DM evidence package and review the attestation statement for completeness against the template.", "Verify artifact currency: confirm each artifact meets the defined maximum age requirement at the time of attestation.", "Sample three to five artifacts from the package — for example, the catalog completeness report and a disposal log — and verify them against source systems to confirm accuracy." ], "metrics": [ "Evidence package currency: target quarterly compilation with no quarter missed in the audit period.", "Artifact completeness rate: target 100% of required artifacts present in each package.", "Attestation sign-off rate: target 100% of packages with a signed DGO attestation statement." ], "failure_signals": [ "No DM evidence package compiled in one or more quarters during the audit period.", "Required artifacts missing from the package with no documented remediation plan.", "Attestation statement signed without all required artifacts present." ] }, "it_operations": { "summary": "Evidence repository availability and access controls are infrastructure responsibilities. Ensure the repository is tamper-evident, access-controlled, and available for retrieval under time-sensitive regulatory demand.", "actions": [ "Configure the evidence repository with immutable storage settings and access logging to provide tamper evidence.", "Implement role-based access controls on the evidence repository limiting write access to authorized evidence contributors and read access to governance and legal personnel.", "Monitor evidence repository availability and ensure it meets the retrieval SLA required for regulatory response." ], "failure_signals": [ "Evidence repository without immutable storage or object-lock configurations.", "Access logs showing unauthorized modifications to evidence package artifacts.", "Evidence repository unavailable when evidence retrieval was requested under regulatory or legal demand." ] }, "data_engineer": { "summary": "The DM evidence package draws on pipeline-generated artifacts. Engineers instrument catalog, retention and quality systems so their outputs land in the evidence pipeline automatically and verifiably.", "actions": [ "Automate exports of catalog snapshots, retention execution logs and quality measurements into the evidence pipeline.", "Hash and timestamp emitted artifacts so the package can be integrity-verified end to end.", "Alert when an expected evidence feed stops producing before the attestation period closes." ], "failure_signals": [ "Evidence artifacts produced by hand each attestation cycle.", "Evidence feeds silently stopping mid-period.", "Package verification failing because artifacts lack hashes or provenance." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "DM-layer evidence packages are uncommon even in regulated enterprises. GRC programs typically capture point-in-time audit evidence rather than maintaining compiled, recurring evidence packages for data management controls." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "eu-high-risk-ai", "high-risk-sector", "federated-enterprise" ], "implementers": [ "Data Governance Team", "GRC Team", "Legal Counsel" ], "frameworks": [ { "framework": "eu_ai_act", "requirement_id": "Art. 11", "fit": "direct", "rationale": "EU AI Act Article 11 requires providers of high-risk AI systems to draw up and maintain technical documentation demonstrating the system's compliance with the regulation. The DM evidence package directly contributes to the Article 11 technical documentation by providing the data governance evidence component required for conformity assessment and notified body review. Compiled and signed evidence packages are the primary mechanism for demonstrating AI Act Art. 10 data governance compliance.", "normative_force": "binding-law", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 3", "fit": "direct", "rationale": "DAMA DMBOK2 Chapter 3 addresses data governance frameworks including documentation, reporting, and evidence requirements. The DMBOK requires data governance programs to produce documented evidence of governance activities for accountability and audit purposes. The DM evidence package directly implements the DMBOK's documentation and reporting requirements for the data management layer.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "cobit_2019", "requirement_id": "MEA01", "fit": "direct", "rationale": "COBIT 2019 MEA01 (Monitor, Evaluate and Assess Performance and Conformance) requires organizations to collect, validate, and evaluate metrics and evidence related to governance objectives. The DM evidence package is the primary artifact satisfying MEA01 for the data management domain. COBIT's MEA01 governance objective requires recurring evidence compilation and attestation, directly aligning with this control.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "dcam", "requirement_id": "Capability 2.1", "fit": "direct", "rationale": "EDM Council DCAM Capability 2.1 requires data management programs to produce evidence of program execution for internal and external reporting. DCAM assessments use evidence packages as primary scoring artifacts to validate capability claims. The DM evidence package directly implements the DCAM reporting requirement and provides the documentation artifacts assessed during DCAM maturity evaluations.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "gdpr", "requirement_id": "Art. 5(2)", "fit": "direct", "rationale": "GDPR Article 5(2) establishes the accountability principle, requiring controllers to be able to demonstrate compliance with the data processing principles in Article 5(1). The DM evidence package provides the structured documentation required to demonstrate compliance with the accuracy, storage limitation, and integrity principles as applied to AI data assets. Without a compiled evidence package, GDPR accountability obligations cannot be satisfied efficiently under supervisory authority inquiry.", "normative_force": "binding-law", "source_version": "2018", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "iso_8000", "requirement_id": "ISO 8000-61", "fit": "adjacent", "rationale": "ISO 8000-61 defines the data quality management process model; the DM evidence package demonstrates those processes operated over the attestation period — inventory, classification, retention and quality outputs in one artifact.", "normative_force": "voluntary-standard", "source_version": "2015", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://data/controls/DM-08", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "A complete DM-layer evidence package must be compiled at minimum quarterly, containing all required artifacts from DM-01 through DM-07 each meeting their currency requirements, accompanied by a DGO-signed attestation statement documenting material gaps and remediation plans, stored in an immutable access-controlled repository and retrievable within 48 hours of request.", "evidence_required": [ "dm_evidence_package containing all 7 required sub-control artifacts (DM-01 catalog export, DM-02 classification coverage report, DM-03 retention schedule and disposal logs, DM-04 MDM golden record status, DM-05 quality threshold compliance report, DM-06 schema change log, DM-07 synthetic data governance records) with each artifact within its defined currency window", "dgo_attestation_statement signed by the Data Governance Officer with date, scope, complete artifact inventory, and documented remediation plans for any material gaps identified at time of compilation", "evidence_repository_configuration_record confirming immutable storage (object-lock or equivalent) is active and that write access is restricted to authorized evidence contributors", "grc_system_registration_record linking the evidence package to DM-08 in the enterprise GRC platform with package_id, compilation_date, and attestation_reference" ], "machine_tests": [ "Query the evidence repository for the most recent DM evidence package → assert the package was compiled within the last 90 days and contains artifacts for all 7 DM sub-controls with non-null artifact references", "Attempt to overwrite or delete a stored evidence package artifact in the repository → assert the operation is rejected with error=immutable_storage_violation", "Validate the evidence package against the defined template schema → assert zero missing required artifacts and zero artifact currency violations at the time of the most recent attestation" ], "human_review": [ "Review the signed DGO attestation statement and verify it was issued after all required artifacts were collected, and that documented gap remediation plans are specific, time-bound, and address every identified material gap", "Assess whether the evidence package template has been reviewed by legal counsel and confirmed as sufficient to meet EU AI Act Art. 11 technical documentation requirements and GDPR Art. 5(2) accountability obligations", "Verify that the evidence repository is subject to legal hold procedures, that retention periods align with applicable statutes of limitations, and that the retrieval SLA has been tested in the past 12 months" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Compiling the evidence package as an informal folder of screenshots and exports rather than a structured, versioned artifact conforming to the defined package template with artifact currency tracking", "Signing the DGO attestation statement before all required artifacts have been collected, creating a compliance record that asserts completeness that does not exist at the time of signing", "Storing evidence packages in a standard shared file system without immutable storage controls, allowing artifact modification after attestation and undermining the evidentiary value of the package", "Omitting gap remediation plans from the attestation statement when material artifacts are missing, treating the evidence package as a binary pass/fail rather than a continuous improvement record", "Failing to register the evidence package in the enterprise GRC system, making it invisible to compliance management workflows and preventing timely retrieval under regulatory examination" ], "update_status": "current", "layer_code": "DM" }, { "id": "DL-01", "layer": "DL", "plane": "data", "name": "End-to-End Data Lineage Tracking", "plain": "Every data asset consumed by an AI system must have a complete, machine-readable lineage graph tracing its origin through all transformations, enrichments, and pipelines to the point of AI consumption, enabling full upstream and downstream impact analysis.", "threat": { "tags": [ "untracked-data-flow", "lineage-gap", "audit-evasion", "supply-chain-opacity" ], "desc": "Without end-to-end lineage, organizations cannot determine which source data contributed to an AI decision, making root-cause analysis after errors or bias incidents impossible. Regulatory investigations stall when lineage records are incomplete, exposing the organization to enforcement action. Attackers who poison upstream data sources remain undetected if the lineage graph does not surface the contaminated path to AI consumers." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 12", "title": "Metadata Management — data lineage" }, { "id": "dcam", "section": "Business & Data Architecture (Component 3)", "title": "Data flows documented in the data architecture" }, { "id": "eu_ai_act", "section": "Art. 10(2)(b)", "title": "Training, validation and testing data governance" }, { "id": "cobit_2019", "section": "DSS06.05", "title": "Ensure traceability and accountability for information events" } ], "sources": [ { "id": "microsoft_purview_data_governance_2024", "title": "Microsoft Purview (Data Governance)", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/purview/purview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_purview_data_governance_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Purview (Data Governance) requirements informing the apeiris://data/controls/DL-01 End-to-End Data Lineage Tracking control.", "reviewed_on": "2026-07-01" }, { "id": "openlineage_spec_1_0", "title": "OpenLineage — Open Standard for Data Lineage", "authority": "OpenLineage (LF AI & Data Foundation)", "source_type": "standard", "normative_force": "best-practice", "version": "1.0.0", "published_on": "2023-08-01", "retrieved_on": "2026-06-29", "canonical_url": "https://openlineage.io/", "license": "open-access", "status": "current", "flagship": false, "source_id": "openlineage_spec_1_0", "relationship": "informative_reference", "rationale": "Establishes OpenLineage — Open Standard for Data Lineage requirements informing the apeiris://data/controls/DL-01 End-to-End Data Lineage Tracking control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Deploy a lineage metadata store integrated with all ingestion, transformation, and serving layers. Capture lineage events at each pipeline stage using an open lineage protocol (OpenLineage) and persist them in an immutable, queryable graph. Surface lineage APIs to downstream consumers and AI model registries.", "steps": [ "Instrument all data pipelines (batch and streaming) to emit OpenLineage events at each run, capturing input datasets, output datasets, and transformation metadata.", "Deploy a lineage metadata backend (Apache Atlas, Marquez, or native cloud catalog) to ingest, store, and serve the lineage graph.", "Integrate the lineage graph with the AI model registry so each model deployment record links to the lineage DAG of its training and inference data.", "Configure alerting for lineage coverage gaps: any data asset reaching an AI pipeline without a complete upstream graph triggers a blocking or advisory gate." ], "data_governance_officer": { "summary": "End-to-end lineage is the foundational data governance artifact for AI systems. Without it, no other data quality, compliance, or audit capability can be fully realized.", "actions": [ "Define the enterprise lineage coverage standard: which data assets require lineage tracking and at what granularity.", "Establish lineage completeness as a data quality KPI tracked in the data governance dashboard.", "Review lineage coverage reports quarterly and escalate persistent gaps to data domain owners." ], "failure_signals": [ "Lineage coverage for AI-bound data assets falls below 90%.", "Any AI model deployment proceeds without a linked lineage record.", "Lineage graph has unresolved orphaned nodes for more than 30 days." ] }, "data_engineer": { "summary": "Lineage instrumentation must be built into pipeline templates so that coverage is automatic rather than manual. Every new pipeline should emit OpenLineage events by default.", "actions": [ "Embed OpenLineage client libraries into the standard pipeline template and enforce via CI/CD linting.", "Test lineage event emission in the pipeline integration test suite before deployment.", "Build a lineage coverage dashboard showing percentage of pipeline runs with complete upstream-to-downstream records." ], "failure_signals": [ "Pipelines deployed without lineage instrumentation.", "Lineage events failing to reach the metadata backend for more than 5% of pipeline runs.", "Lineage graph contains nodes with no upstream or downstream edges." ] }, "grc_auditor": { "summary": "Lineage completeness is the primary evidence artifact for demonstrating AI data governance to regulators. Audit sampling should verify that AI outputs can be traced end-to-end to source data.", "actions": [ "Request a lineage coverage report from the data governance team and verify it covers 100% of AI-bound data assets.", "Select 5 AI model outputs at random and trace their lineage to source, documenting any gaps.", "Verify that the lineage metadata store is append-only and tamper-evident." ], "metrics": [ "Lineage coverage rate for AI-bound assets: target 100%.", "Percentage of AI model deployments with linked lineage DAG: target 100%.", "Mean time to close lineage gaps: target ≤ 5 business days." ], "failure_signals": [ "Coverage below 90% for two consecutive quarters.", "AI model deployments proceeding without lineage records.", "Lineage metadata store is not append-only or lacks integrity controls." ] }, "it_operations": { "summary": "The lineage metadata store must be treated as critical infrastructure with high availability, backup, and monitoring equivalent to the data systems it tracks.", "actions": [ "Configure monitoring and alerting for lineage ingest lag and event loss.", "Ensure the lineage backend has automated backup with a recovery time objective matching the primary data systems.", "Patch and update lineage tooling on the same cycle as other data platform components." ], "failure_signals": [ "Lineage ingest lag exceeds 1 hour for streaming pipelines.", "Lineage backend unavailability exceeds the 99.5% monthly SLA.", "Backup or restore tests have not been run in the past 90 days." ] }, "legal_counsel": { "summary": "Lineage is how the organization proves where data came from and what touched it — the factual backbone of GDPR accountability and EU AI Act data-governance documentation. Counsel is a primary consumer of lineage evidence.", "actions": [ "Specify the lineage detail needed to answer regulator and data-subject questions about AI data flows.", "Use lineage records to verify representations about data sourcing before they are made.", "Confirm lineage coverage of personal-data flows feeding AI systems." ], "failure_signals": [ "Sourcing questions from regulators answered by interview rather than lineage records.", "Personal-data flows into AI training with no lineage capture.", "Lineage gaps discovered only during incident response." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Most enterprises have partial lineage coverage from individual tools but lack a unified lineage graph spanning all AI-bound data flows." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "federated-enterprise", "eu-high-risk-ai" ], "implementers": [ "Data Engineering", "Data Governance Office", "Platform Engineering" ], "frameworks": [ { "framework": "dama_dmbok", "requirement_id": "Ch. 12", "fit": "direct", "rationale": "DAMA-DMBOK2 Chapter 12 (Metadata Management) treats data lineage as metadata: capturing the origin, movement and transformation of data as it flows through systems. End-to-end lineage tracking implements the lineage guidance of the metadata chapter; Chapter 7 covers Data Security, not lineage.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "dcam", "requirement_id": "Business & Data Architecture (Component 3)", "fit": "direct", "rationale": "DCAM v2.2 Component 3 (Business & Data Architecture) requires data flows to be documented as part of the defined data architecture. End-to-end lineage is the operational realization of that documented-flow requirement; capability 4.2 concerns the technology stack, not lineage.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "eu_ai_act", "requirement_id": "Art. 10(2)(b)", "fit": "direct", "rationale": "EU AI Act Article 10(2)(b) requires high-risk AI systems to use data subject to appropriate data governance practices, including documentation of data origin. End-to-end lineage is the mechanism by which this origin documentation is maintained and demonstrable to regulators.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "cobit_2019", "requirement_id": "DSS06.05", "fit": "partial", "rationale": "COBIT 2019 practice DSS06.05 requires the ability to trace information events and the accountability for them. End-to-end lineage is the enterprise implementation of that traceability for AI data flows. DSS06.03 concerns roles and access privileges, not records or traceability.", "normative_force": "best-practice", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "microsoft_purview", "requirement_id": "Data lineage (lineage extraction)", "fit": "direct", "rationale": "Microsoft Purview extracts lineage from scanned sources and pipeline integrations (for example Azure Data Factory) into its data map — automated lineage extraction rather than a dedicated 'lineage scanning' product. Coverage varies by connector, so extraction gaps must be tracked against DL-01's completeness requirement.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "openlineage", "requirement_id": "RunEvent object (OpenLineage.json JSON Schema)", "fit": "direct", "rationale": "OpenLineage defines lineage as RunEvent objects — runs of jobs consuming and producing datasets — in its OpenLineage.json JSON Schema; the specification has no numbered '§2'. Emitting RunEvents from pipeline orchestrators gives DL-01 a vendor-neutral lineage interchange format.", "normative_force": "best-practice", "source_version": "1.0.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "owasp_ai_exchange", "requirement_id": "supplychainmanage", "fit": "supporting", "rationale": "DL-01 maintains a complete machine-readable lineage graph tracing every AI-consumed asset from source to consumption, managing the AI data supply chain provenance.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "mitre", "requirement_id": "AML.M0023", "fit": "supporting", "direction": "control-supports-requirement", "rationale": "Control \"Every data asset consumed by an AI system must have a complete, machine-readable lineage…\" enacts ATLAS mitigation AML.M0023 AI Bill of Materials; OpenCRE crosswalks this control’s OWASP AI Exchange concept (supplychainmanage) to this mitigation.", "source_version": "v2026.06", "reviewed_on": "2026-07-08", "source_status": "authoritative", "mapping_confidence": "medium", "legal_status": "not-applicable", "control_readiness": "approved", "implementation_maturity": "developing", "assurance_result": "not-assessed", "evidence_status": "no-evidence", "normative_force": "industry-framework", "basis": "asserted", "relation": "equivalent_to", "correction": "ap38-opencre-enhancement 2026-07-08" }, { "framework": "mitre", "requirement_id": "AML.M0014", "fit": "supporting", "direction": "control-supports-requirement", "rationale": "Control \"Every data asset consumed by an AI system must have a complete, machine-readable lineage…\" enacts ATLAS mitigation AML.M0014 Verify AI Artifacts; OpenCRE crosswalks this control’s OWASP AI Exchange concept (supplychainmanage) to this mitigation.", "source_version": "v2026.06", "reviewed_on": "2026-07-08", "source_status": "authoritative", "mapping_confidence": "medium", "legal_status": "not-applicable", "control_readiness": "approved", "implementation_maturity": "developing", "assurance_result": "not-assessed", "evidence_status": "no-evidence", "normative_force": "industry-framework", "basis": "asserted", "relation": "equivalent_to", "correction": "ap38-opencre-enhancement 2026-07-08" }, { "framework": "nist_ai_100_2", "requirement_id": "NISTAML.05", "fit": "supporting", "rationale": "Control \"Every data asset consumed by an AI system must have a complete, machine-readable lineage…\" defends against NIST AI 100-2 attack class NISTAML.05 \"Supply Chain Attacks\"; OpenCRE crosswalks this control’s OWASP AI Exchange concept (supplychainmanage) to this NIST class. (Anchor captures the attack taxonomy, so we cite the threat class as defends_against, not the NIST mitigation named by OpenCRE.)", "normative_force": "informative-reference", "source_version": "100-2e2025", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "defends_against", "mapping_confidence": "medium", "correction": "ap38-opencre-enhancement 2026-07-08" } ], "canonical_id": "apeiris://data/controls/DL-01", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every data asset consumed by an AI system must have a complete, machine-readable lineage graph in the lineage metadata store tracing its full upstream path from source to AI consumption, with lineage event coverage at 100% of pipeline runs, no orphaned nodes unresolved beyond 30 days, and each AI model deployment record linked to a lineage DAG reference in the model registry.", "evidence_required": [ "lineage_coverage_report from the lineage metadata store showing percentage of AI-bound data assets with complete upstream lineage graphs, with zero assets missing lineage at the time of compilation", "openlineage_job_run_events_log confirming events were emitted by all instrumented pipeline runs in the audit period with input_dataset_ids, output_dataset_ids, and transformation_metadata present for each run", "model_registry_lineage_link_audit confirming each model deployment record contains a non-null lineage_dag_reference for all training and inference data assets", "lineage_graph_integrity_report confirming the metadata store is configured as append-only and that no orphaned nodes have persisted beyond 30 days" ], "machine_tests": [ "Deploy a data pipeline to a non-production environment without OpenLineage instrumentation → assert the CI/CD lineage coverage check flags the deployment with status=missing_lineage_instrumentation", "Query the lineage graph for a known AI-bound data asset by dataset_id → assert the upstream graph traverses to at least one source node with no missing edges in the path", "Attempt to submit an OpenLineage job run event with a missing output_dataset_id field → assert the lineage metadata store rejects the event with error=invalid_lineage_event_schema" ], "human_review": [ "Select 5 AI model outputs at random and manually trace their lineage through the lineage graph to source, documenting any gaps, orphaned nodes, or unexplained provenance breaks discovered", "Review the lineage coverage dashboard for the past quarter and verify no AI-bound data assets have had persistent lineage gaps exceeding 30 days without documented remediation", "Assess whether the lineage metadata store's append-only configuration has been independently tested and that unauthorized modification attempts generate alerts within the defined SLA" ], "blocking_effect": "advisory", "normative_status": "binding-law", "anti_patterns": [ "Relying on manual data flow documentation in wikis or spreadsheets as the primary lineage record instead of machine-readable graphs automatically maintained by pipeline instrumentation", "Implementing lineage only for batch pipelines while excluding streaming pipelines, leaving real-time AI data flows without provenance records despite carrying the same regulatory obligations", "Storing lineage metadata in a mutable data store without append-only constraints, allowing modification or deletion of lineage records after they are created", "Treating lineage as a point-in-time catalog snapshot generated quarterly rather than a continuously updated graph, allowing gaps to accumulate between snapshot cycles", "Linking AI model registry entries to lineage DAGs manually after model deployment rather than automating the link at training job completion, resulting in stale or missing references" ], "update_status": "current", "layer_code": "DL" }, { "id": "DL-02", "layer": "DL", "plane": "data", "name": "Training Data Lineage Documentation", "plain": "All data used to train, fine-tune, or adapt an AI model must have documented lineage including source identity, collection date, collection method, consent or license basis, transformation history, and the version of the dataset used in each training run.", "threat": { "tags": [ "unlicensed-training-data", "provenance-loss", "bias-amplification", "regulatory-noncompliance" ], "desc": "Models trained on data with undocumented lineage cannot be audited for license compliance, consent validity, or bias introduction. When a model produces discriminatory outputs, absence of training data provenance prevents identification of the offending source dataset. Regulators under the EU AI Act and emerging global AI governance frameworks require training data documentation as a condition of deployment for high-risk AI systems." }, "standard": [ { "id": "eu_ai_act", "section": "Art. 10(2)", "title": "Training data governance requirements for high-risk AI" }, { "id": "dama_dmbok", "section": "Ch. 12", "title": "Metadata Management — provenance documentation" }, { "id": "anthropic_privacy", "section": "Privacy Policy", "title": "Provider disclosures on training data sourcing" }, { "id": "dcam", "section": "Capability 5.1", "title": "Data quality management program established — sourcing and provenance in scope" } ], "sources": [ { "id": "anthropic_privacy_policy_2024", "title": "Anthropic Data Handling & Privacy Policy", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-09-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/privacy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_privacy_policy_2024", "relationship": "informative_reference", "rationale": "Establishes Anthropic Data Handling & Privacy Policy requirements informing the apeiris://data/controls/DL-02 Training Data Lineage Documentation control.", "reviewed_on": "2026-07-01" }, { "id": "databricks_unity_catalog_2024", "title": "Databricks Unity Catalog", "authority": "Databricks, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.databricks.com/en/data-governance/unity-catalog/index.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "databricks_unity_catalog_2024", "relationship": "informative_reference", "rationale": "Establishes Databricks Unity Catalog requirements informing the apeiris://data/controls/DL-02 Training Data Lineage Documentation control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Maintain a training dataset registry where each entry records source identity, collection date and method, license or consent basis, transformation history, version hash, and a link to the lineage DAG. Bind each model training run record to the specific dataset versions used, producing an immutable training provenance artifact.", "steps": [ "Create a training dataset registry with mandatory fields: source name and URL, collection date range, collection method (web scrape, API, licensed feed, synthetic, human-labeled), license or consent basis, and a SHA-256 hash of the dataset at the version used.", "Require every model training job to reference specific dataset registry entries by version ID; block training runs that reference unregistered or incomplete dataset records.", "Generate a training provenance artifact at the conclusion of each training run, embedding dataset version IDs, lineage DAG references, and training job metadata into the model card.", "Conduct an annual review of all active training datasets to verify that license and consent bases remain valid and that collection dates are within the acceptable freshness window." ], "data_governance_officer": { "summary": "Training data lineage is the most legally consequential lineage record an AI organization maintains. It is the primary artifact for demonstrating compliance with copyright, consent, and data protection obligations.", "actions": [ "Define the mandatory fields for the training dataset registry and enforce completeness as a gate on model deployment.", "Establish a training data license review process with Legal that runs before any new dataset is added to the registry.", "Track the percentage of deployed models with complete training provenance artifacts as a top-level governance KPI." ], "failure_signals": [ "Any model deployed without a complete training provenance artifact linked to the model registry entry.", "Training dataset registry entries with missing or expired license records.", "Training runs referencing datasets not in the approved registry." ] }, "data_engineer": { "summary": "Training data pipelines must be instrumented to automatically populate the dataset registry and produce provenance artifacts without manual intervention by data scientists.", "actions": [ "Build dataset intake pipelines that automatically compute SHA-256 hashes and populate registry fields at ingest time.", "Integrate dataset registry validation into the model training job launcher so jobs are blocked without valid registry references.", "Automate generation of the training provenance artifact at job completion and push it to the model registry." ], "failure_signals": [ "Training jobs completing without a provenance artifact in the model registry.", "Dataset hashes in the registry not matching the actual dataset files.", "Manual overrides of registry validation gates in training job launchers." ] }, "legal_counsel": { "summary": "Training data provenance records are the organization's primary defense in copyright, consent, and data protection disputes. They must be maintained with the same rigor as legal hold documents.", "actions": [ "Review and approve the license or consent basis for each new dataset before the registry entry is finalized.", "Establish a retention policy for training provenance artifacts that aligns with the statute of limitations in all jurisdictions where the model is deployed.", "Define the legal hold process for training provenance records when litigation or regulatory investigation is anticipated." ], "failure_signals": [ "Datasets with expired, revoked, or legally contested license bases remaining active in the registry.", "Training provenance artifacts deleted or overwritten before the retention period expires.", "No legal review process documented for new dataset onboarding." ] }, "grc_auditor": { "summary": "Training data lineage completeness is the primary audit artifact for EU AI Act Article 10 compliance and for responding to regulatory data inquiries about AI model behavior.", "actions": [ "Verify that 100% of models in the model registry have linked training provenance artifacts.", "Sample 10% of training dataset registry entries and validate license or consent basis documentation.", "Test the lineage DAG for a sample of training datasets to confirm end-to-end traceability to source." ], "metrics": [ "Percentage of deployed models with complete training provenance artifact: target 100%.", "Percentage of training dataset registry entries with valid, current license documentation: target 100%.", "Mean age of training datasets in active use: monitored for freshness." ], "failure_signals": [ "Models deployed without training provenance artifacts.", "License gaps in the training dataset registry for more than 5% of entries.", "Lineage DAG cannot be traversed from training data to source for any sampled dataset." ] }, "it_operations": { "summary": "Training data lineage is captured by pipeline instrumentation. Operations ensures every training run records its dataset versions, preprocessing steps and environment automatically — no manual lineage.", "actions": [ "Instrument training pipelines to emit dataset versions, hashes and preprocessing lineage for every run.", "Block or flag training runs that start without lineage capture enabled.", "Retain training lineage records per the model's regulatory retention needs." ], "failure_signals": [ "Model versions in production with no captured training lineage.", "Lineage capture disabled to speed up runs.", "Training lineage stored on infrastructure retired before the model it documents." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most AI development teams document training data informally. Structured registry-based provenance tied to model deployment gates is uncommon outside regulated sectors." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "Data Engineering", "ML Platform Team", "Legal / IP Counsel", "Data Governance Office" ], "frameworks": [ { "framework": "eu_ai_act", "requirement_id": "Art. 10(2)", "fit": "direct", "rationale": "EU AI Act Article 10(2) enumerates specific data governance requirements for training, validation, and testing data used in high-risk AI systems, including documentation of origin, collection methods, and relevant characteristics. Training data lineage documentation is the direct technical implementation of this statutory requirement.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 12", "fit": "direct", "rationale": "DAMA-DMBOK2 Chapter 12 (Metadata Management) includes provenance metadata — where data originated and under what terms it was acquired — among the metadata a mature program captures. Training data lineage documentation applies that guidance to AI training corpora.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "anthropic_privacy", "requirement_id": "Privacy Policy", "fit": "adjacent", "rationale": "Anthropic's Privacy Policy states at document level how data sources are used in training; the policy has no numbered 'Data Handling §3' section. It illustrates the provider-side documentation that training data lineage records must capture when third-party models are in the supply chain.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "dcam", "requirement_id": "Capability 5.1", "fit": "direct", "rationale": "DCAM v2.2 capability 5.1 (the data quality management program is established) includes defining quality expectations for data sourcing. Training data lineage documentation records where data came from and under what terms — the sourcing evidence the DQ program's scope presupposes; capability 4.3 concerns technology operating risk.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "databricks_unity", "requirement_id": "Unity Catalog lineage", "fit": "direct", "rationale": "Unity Catalog captures table- and column-level lineage automatically for workloads that run through it, giving training pipelines a queryable record of which datasets fed which model artifacts.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "owasp_ai_exchange", "requirement_id": "supplychainmanage", "fit": "supporting", "rationale": "DL-02 binds each model to training-dataset provenance records carrying source_identity, collection_method, and sha256_hash, managing training-data supply-chain integrity.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "owasp_ai_exchange", "requirement_id": "alloweddata", "fit": "supporting", "rationale": "DL-02 requires each training dataset entry to record a license_or_consent_basis, evidencing that only lawfully-permitted data was used for training.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "mitre", "requirement_id": "AML.M0023", "fit": "supporting", "direction": "control-supports-requirement", "rationale": "Control \"Every model in the model registry must have a training provenance artifact linking to…\" enacts ATLAS mitigation AML.M0023 AI Bill of Materials; OpenCRE crosswalks this control’s OWASP AI Exchange concept (supplychainmanage) to this mitigation.", "source_version": "v2026.06", "reviewed_on": "2026-07-08", "source_status": "authoritative", "mapping_confidence": "medium", "legal_status": "not-applicable", "control_readiness": "approved", "implementation_maturity": "developing", "assurance_result": "not-assessed", "evidence_status": "no-evidence", "normative_force": "industry-framework", "basis": "asserted", "relation": "equivalent_to", "correction": "ap38-opencre-enhancement 2026-07-08" }, { "framework": "mitre", "requirement_id": "AML.M0014", "fit": "supporting", "direction": "control-supports-requirement", "rationale": "Control \"Every model in the model registry must have a training provenance artifact linking to…\" enacts ATLAS mitigation AML.M0014 Verify AI Artifacts; OpenCRE crosswalks this control’s OWASP AI Exchange concept (supplychainmanage) to this mitigation.", "source_version": "v2026.06", "reviewed_on": "2026-07-08", "source_status": "authoritative", "mapping_confidence": "medium", "legal_status": "not-applicable", "control_readiness": "approved", "implementation_maturity": "developing", "assurance_result": "not-assessed", "evidence_status": "no-evidence", "normative_force": "industry-framework", "basis": "asserted", "relation": "equivalent_to", "correction": "ap38-opencre-enhancement 2026-07-08" }, { "framework": "nist_ai_100_2", "requirement_id": "NISTAML.05", "fit": "supporting", "rationale": "Control \"Every model in the model registry must have a training provenance artifact linking to…\" defends against NIST AI 100-2 attack class NISTAML.05 \"Supply Chain Attacks\"; OpenCRE crosswalks this control’s OWASP AI Exchange concept (supplychainmanage) to this NIST class. (Anchor captures the attack taxonomy, so we cite the threat class as defends_against, not the NIST mitigation named by OpenCRE.)", "normative_force": "informative-reference", "source_version": "100-2e2025", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "defends_against", "mapping_confidence": "medium", "correction": "ap38-opencre-enhancement 2026-07-08" } ], "canonical_id": "apeiris://data/controls/DL-02", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every model in the model registry must have a training provenance artifact linking to specific training dataset registry entries by version ID, with each registry entry containing source_identity, collection_date_range, collection_method, license_or_consent_basis, and sha256_hash; and no training job may complete without the artifact being generated as an immutable record bound to that model version.", "evidence_required": [ "training_dataset_registry_export for all datasets referenced in training runs in the audit period, with source_name, collection_date_range, collection_method, license_or_consent_basis, and sha256_hash fields populated for every entry", "training_provenance_artifact for each deployed model in the model registry, containing dataset_version_ids, lineage_dag_references, and training_job_metadata as immutable records", "training_job_validation_log confirming each training run referenced registered dataset version IDs and was blocked when referencing unregistered or incomplete entries", "legal_review_record for each new dataset added to the registry in the audit period, signed by legal counsel before the registry entry was finalized, confirming the license or consent basis is sufficient for the intended training use" ], "machine_tests": [ "Submit a training job referencing a dataset_id not present in the training dataset registry → assert the job launcher blocks with error=unregistered_dataset_reference before any compute is allocated", "Query the model registry for all deployed models → assert 100% have a training_provenance_artifact field containing at least one non-null dataset_version_id reference", "Compute the SHA-256 hash of a sampled training dataset file on disk and compare it to the corresponding registry entry sha256_hash → assert hashes match exactly with zero discrepancy" ], "human_review": [ "Sample 10% of training dataset registry entries and validate that the license or consent basis documentation is current, complete, and legally sufficient for the intended training use in all jurisdictions where the model is deployed", "Review the training job launcher configuration to confirm that the dataset registry validation gate cannot be bypassed by operators via environment variable flags, command-line overrides, or emergency access procedures", "Verify that legal counsel reviewed and documented copyright clearance for all datasets sourced from web scraping, licensed commercial data feeds, or third-party data providers before those datasets were admitted to the registry" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Documenting training dataset provenance as free-text descriptions in model cards or experiment tracker run notes rather than as structured registry entries with version-controlled, machine-readable fields", "Using dataset identifiers from local experiment tracker runs or file paths rather than linking to a central, auditable training dataset registry with controlled entry and approval", "Accepting copyright clearance based solely on the data source website's terms of service without explicit legal review of whether those terms permit AI model training use", "Computing dataset SHA-256 hashes at training job start rather than at ingest time, allowing dataset files to be modified between ingestion and training without the hash discrepancy being detected", "Allowing training jobs to complete successfully without generating a training provenance artifact and retroactively reconstructing provenance after model deployment when audit requests arrive" ], "update_status": "current", "layer_code": "DL" }, { "id": "DL-03", "layer": "DL", "plane": "data", "name": "Inference-Time Data Lineage (Per-Decision Provenance)", "plain": "For each individual AI decision or output, the system must record which specific data records, retrieved context, and real-time inputs contributed to that output, creating a per-decision provenance record that enables reconstruction and explanation of any individual inference.", "threat": { "tags": [ "unexplainable-decision", "stale-context-use", "rag-poisoning", "right-to-explanation-gap" ], "desc": "Without per-decision provenance, organizations cannot explain to affected individuals or regulators which data drove a specific AI outcome, violating explainability obligations under the EU AI Act and GDPR. RAG-augmented models that retrieve poisoned or outdated context records cannot be investigated after the fact if retrieval records are not retained. Absence of inference-time lineage makes it impossible to distinguish model errors from data errors during post-incident analysis." }, "standard": [ { "id": "eu_ai_act", "section": "Art. 12(1)", "title": "Logging requirements for high-risk AI systems" }, { "id": "gdpr", "section": "Art. 5(1)(a)", "title": "Lawfulness, fairness, transparency of processing" }, { "id": "nist_pf", "section": "CT.PO-P1", "title": "Policies for data processing transparency" }, { "id": "dama_dmbok", "section": "Ch. 12", "title": "Metadata Management — operational lineage" } ], "sources": [ { "id": "eu_ai_act", "title": "EU AI Act", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2024/1689", "published_on": "2024-07-12", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2024/1689/oj/eng", "license": "open-access", "status": "current", "flagship": false, "source_id": "eu_ai_act", "relationship": "normative_requirement", "rationale": "Establishes EU AI Act Art. 12(1) requirements informing the apeiris://data/controls/DL-03 Inference-Time Data Lineage (Per-Decision Provenance) control.", "reviewed_on": "2026-07-02" }, { "id": "gdpr", "title": "GDPR", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2016/679", "published_on": "2016-04-27", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj", "license": "open-access", "status": "current", "flagship": false, "source_id": "gdpr", "relationship": "normative_requirement", "rationale": "Establishes GDPR Art. 5(1)(a) requirements informing the apeiris://data/controls/DL-03 Inference-Time Data Lineage (Per-Decision Provenance) control.", "reviewed_on": "2026-07-02" }, { "id": "nist_pf", "title": "NIST Privacy Framework 1.0", "authority": "National Institute of Standards and Technology", "source_type": "standard", "normative_force": "voluntary-standard", "version": "1.0", "published_on": "2020-01-16", "retrieved_on": "2026-07-02", "canonical_url": "https://www.nist.gov/privacy-framework", "license": "open-access", "status": "current", "flagship": false, "source_id": "nist_pf", "relationship": "implementation_pattern", "rationale": "Establishes NIST Privacy Framework 1.0 CT.PO-P1 requirements informing the apeiris://data/controls/DL-03 Inference-Time Data Lineage (Per-Decision Provenance) control.", "reviewed_on": "2026-07-02" }, { "id": "dama_dmbok", "title": "DAMA-DMBOK 2nd Ed", "authority": "DAMA International", "source_type": "standard", "normative_force": "best-practice", "version": "2 (2nd Edition, Revised 2024)", "published_on": "2017-07-05", "retrieved_on": "2026-07-02", "canonical_url": "https://www.dama.org/cpages/body-of-knowledge", "license": "proprietary", "status": "current", "flagship": false, "source_id": "dama_dmbok", "relationship": "informative_reference", "rationale": "Establishes DAMA-DMBOK 2nd Ed Ch. 12 requirements informing the apeiris://data/controls/DL-03 Inference-Time Data Lineage (Per-Decision Provenance) control.", "reviewed_on": "2026-07-02" }, { "id": "iso_27701", "title": "ISO/IEC 27701:2019", "authority": "ISO/IEC", "source_type": "standard", "normative_force": "certification-standard", "version": "2019 (superseded by ISO/IEC 27701:2025, published 2025-10-14; 24-36 month certification transition)", "published_on": "2019-08-06", "retrieved_on": "2026-07-02", "canonical_url": "https://www.iso.org/standard/71670.html", "license": "proprietary", "status": "current", "flagship": false, "source_id": "iso_27701", "relationship": "implementation_pattern", "rationale": "Establishes ISO/IEC 27701:2019 §7.2.8 requirements informing the apeiris://data/controls/DL-03 Inference-Time Data Lineage (Per-Decision Provenance) control.", "reviewed_on": "2026-07-02" }, { "source_id": "ms_ifc_agents", "normative_force": "best-practice", "relationship": "informative_reference", "rationale": "Grounds inference-time data provenance: IFC propagates labels with derivative results through the agent loop, recording provenance at each step.", "reviewed_on": "2026-07-03", "title": "Information-flow control: Moving toward secure, autonomous agents", "authority": "Microsoft", "canonical_url": "https://commandline.microsoft.com/information-flow-control-moving-toward-secure-autonomous-agents/" } ], "implementation": { "pattern": "At each inference call, capture and persist a provenance record containing: inference ID, timestamp, model version, input data identifiers, retrieved context records (for RAG systems) with their source dataset references, and a reference to the output record. Store provenance records in an append-only log linked to the inference ID. Expose a provenance lookup API for post-hoc explanation workflows.", "steps": [ "Instrument the model serving layer to generate a unique inference ID for each request and emit a provenance event capturing input record IDs, retrieved context IDs (if RAG), model version, and timestamp.", "Store inference provenance records in an append-only log with a retention period aligned to the applicable regulatory obligation (minimum 3 years for EU AI Act high-risk systems).", "Expose a provenance lookup API that accepts an inference ID and returns the full per-decision provenance record, enabling explanation workflows for individual decisions.", "For RAG systems, instrument the retrieval layer to record the specific document chunks retrieved, their source dataset and version, and the relevance scores used for selection." ], "data_governance_officer": { "summary": "Per-decision provenance is the mechanism by which the organization can honor right-to-explanation obligations and respond to individual data subject requests about AI decisions affecting them.", "actions": [ "Establish the retention period for inference provenance records based on legal review of applicable regulations in all deployment jurisdictions.", "Define the process for responding to individual subject access requests that involve AI decision provenance.", "Include inference provenance coverage in the data governance dashboard alongside training lineage metrics." ], "failure_signals": [ "AI systems making decisions about individuals without capturing per-decision provenance records.", "Provenance records unavailable for decisions within the applicable retention window.", "No process defined for responding to subject access requests involving AI decisions." ] }, "data_engineer": { "summary": "Inference provenance instrumentation must be built into the model serving framework and retrieval layer as a non-optional component of the inference stack.", "actions": [ "Add provenance event emission to the model serving wrapper so it cannot be disabled by individual model deployments.", "Implement the provenance lookup API with appropriate access controls and audit logging of lookups.", "For RAG systems, modify the retrieval pipeline to tag each retrieved chunk with its source dataset version before passing it to the model." ], "failure_signals": [ "Model serving instances running without provenance event emission.", "Provenance records with missing retrieved context IDs for RAG model calls.", "Provenance lookup API returning empty results for valid inference IDs within the retention window." ] }, "legal_counsel": { "summary": "Inference-time lineage records are the documentary basis for responding to regulatory inquiries, data subject rights requests, and litigation discovery related to individual AI decisions.", "actions": [ "Confirm that the retention period for provenance records satisfies GDPR Article 5 accountability requirements and EU AI Act Article 12 logging obligations.", "Define the legal hold procedure for provenance records when a specific AI decision is under dispute or investigation.", "Review the provenance record schema to ensure it captures sufficient information to reconstruct the factual basis of an individual decision." ], "failure_signals": [ "Provenance records deleted before the legally required retention period.", "Provenance records insufficient to reconstruct the data basis of a decision subject to legal challenge.", "No legal hold procedure documented for inference provenance records." ] }, "grc_auditor": { "summary": "Inference-time lineage completeness is the key audit evidence for explainability and logging compliance under EU AI Act Article 12 and GDPR accountability obligations.", "actions": [ "Verify that all high-risk AI model serving instances are emitting provenance events for 100% of inference requests.", "Sample 20 inference IDs from production logs and verify that complete provenance records are retrievable via the lookup API.", "Confirm that provenance record retention periods are documented and enforced by the data platform." ], "metrics": [ "Percentage of inference requests with captured provenance records: target 100% for high-risk AI systems.", "Provenance lookup success rate for records within retention window: target 100%.", "Mean provenance record completeness score (required fields populated): target 100%." ], "failure_signals": [ "High-risk AI inference requests without provenance records.", "Provenance lookup failures for records within the retention window.", "Provenance records missing required fields for more than 1% of sampled records." ] }, "it_operations": { "summary": "Per-decision provenance is a high-volume runtime capture problem. Operations sizes, stores and indexes decision-level lineage so it is queryable within SLA when a specific decision is challenged.", "actions": [ "Operate decision-provenance capture with throughput and retention sized to production volume.", "Index provenance records for retrieval by decision identifier within the response SLA.", "Monitor capture completeness — every consequential decision must produce a provenance record." ], "failure_signals": [ "Decision volumes exceeding provenance capture capacity, dropping records.", "Provenance lookups for challenged decisions taking days.", "Sampling silently substituted for full capture on consequential systems." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Per-decision provenance is an emerging practice even in regulated industries. Most AI deployments log inputs and outputs but do not link them to source data lineage records or retrieved context provenance." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "high-risk-sector", "eu-high-risk-ai", "federated-enterprise", "multi-tenant" ], "implementers": [ "ML Platform Team", "Data Engineering", "Legal / Privacy Counsel" ], "frameworks": [ { "framework": "eu_ai_act", "requirement_id": "Art. 12(1)", "fit": "direct", "rationale": "EU AI Act Article 12(1) requires high-risk AI systems to be designed and developed with capabilities enabling automatic logging of events to the degree appropriate for their intended purpose. Per-decision provenance records are the primary mechanism for satisfying this logging obligation at the granularity required by the regulation.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "gdpr", "requirement_id": "Art. 5(1)(a)", "fit": "direct", "rationale": "GDPR Article 5(1)(a) transparency requires organisations to be able to explain what data informed a decision about an individual. Per-decision provenance is the technical record that makes such explanation possible for AI-assisted decisions, and underpins Art. 15 access-request and Art. 22 automated-decision responses.", "normative_force": "binding-law", "source_version": "2016/679", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_pf", "requirement_id": "CT.PO-P1", "fit": "partial", "rationale": "NIST Privacy Framework CT.PO-P1 requires policies for data processing to be communicated, covering how data is used in automated decisions. Per-decision provenance operationalizes this policy requirement by producing the technical record from which processing explanations can be drawn.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 12", "fit": "partial", "rationale": "DAMA-DMBOK2 Chapter 12's metadata guidance extends to operational metadata — runtime records of which data was touched by which process. Per-decision provenance is operational lineage metadata captured at the transaction level.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "iso_27701", "requirement_id": "§7.2.8", "fit": "adjacent", "rationale": "ISO/IEC 27701 §7.2.8 addresses ensuring accuracy of personal information, which for AI systems requires the ability to trace which data records contributed to a specific automated output. Inference-time lineage provides the technical record needed to verify accuracy claims and respond to accuracy challenges from data subjects.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "ms_ifc_agents", "requirement_id": "Propagate labels — labels travel with derivative results as data flows through the agent loop (Communicating labels)", "fit": "partial", "rationale": "IFC's propagate-labels step means labels travel with data and its derivatives as they flow through the agent loop, yielding per-decision provenance of where data came from and how it may be used at inference time.", "normative_force": "best-practice", "source_version": "2026", "reviewed_on": "2026-07-03", "basis": "anchored", "relation": "informs", "correction": "ap07-fit-audit 2026-07-08 (direct->partial)" } ], "canonical_id": "apeiris://data/controls/DL-03", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "For every inference request processed by a high-risk AI system, a provenance record must be created and stored in an append-only log containing inference_id, timestamp, model_version, input_data_identifiers, retrieved_context_record_ids (for RAG systems), and output_record_reference — and this record must be retrievable via the provenance lookup API for the full duration of the applicable regulatory retention period.", "evidence_required": [ "inference_provenance_log_sample covering a statistically representative set of inference requests, confirming all mandatory fields (inference_id, timestamp, model_version, input_record_ids, output_record_reference) are populated with no null values", "rag_retrieval_provenance_records for RAG-augmented inference systems, showing retrieved_chunk_ids, source_dataset_version, and relevance_scores for each retrieval event linked to its inference_id", "provenance_lookup_api_test_results confirming successful retrieval of complete provenance records for 100% of sampled inference IDs within the retention window, with HTTP 200 responses and all required fields present", "retention_policy_documentation specifying the legal basis for the defined retention period, confirmation that the platform enforces deletion only after the period expires, and evidence the policy has been reviewed by legal counsel" ], "machine_tests": [ "Submit an inference request to the model serving layer → assert a provenance record with all required fields (inference_id, timestamp, model_version, input_record_ids, output_record_reference) is written to the append-only log within 5 seconds", "Issue a provenance lookup API request for an inference_id within the retention window → assert the API returns HTTP 200 with a complete record containing all mandatory fields and non-null values", "Attempt to delete a provenance record for an inference_id within the active retention period → assert the operation is rejected with error=retention_period_active and the record remains intact" ], "human_review": [ "Sample 20 inference IDs from production logs and manually verify that complete provenance records are retrievable via the lookup API, including all retrieved context chunk IDs for RAG calls, with no missing mandatory fields", "Review the retention period policy documentation and confirm it satisfies GDPR Art. 5 accountability requirements and EU AI Act Art. 12 logging obligations for all jurisdictions in which the high-risk AI system is deployed", "Assess the provenance lookup API access controls to verify that subject access request fulfillment workflows are documented, tested, and that lookup audit logs are retained for the full compliance period" ], "blocking_effect": "blocks-runtime-action", "normative_status": "binding-law", "anti_patterns": [ "Logging AI inputs and outputs to application log streams without linking them to source data record identifiers, producing records too coarse to satisfy per-decision provenance requirements for individual explanation", "Implementing provenance event emission as an optional feature flag that can be disabled per model deployment, allowing individual deployment teams to skip provenance recording for convenience or performance reasons", "Storing per-decision provenance records in the same mutable application database as operational state, allowing records to be modified or deleted during routine database maintenance and cleanup operations", "For RAG systems, logging only the final synthesized context string passed to the model rather than the individual retrieved chunks with their source dataset version references, preventing reconstruction of which specific documents influenced the output", "Setting provenance record retention periods shorter than the applicable regulatory retention obligation because long-term storage costs are perceived as prohibitive, without legal sign-off on the shortened period" ], "update_status": "current", "layer_code": "DL" }, { "id": "DL-04", "layer": "DL", "plane": "data", "name": "Transformation and Enrichment Lineage", "plain": "Every transformation, normalization, enrichment, join, aggregation, or derivation applied to data between its source and AI consumption must be documented in the lineage record, including the transformation logic, the system that applied it, the operator who configured it, and the timestamp of each transformation step.", "threat": { "tags": [ "undocumented-transformation", "semantic-drift", "silent-enrichment", "pipeline-tampering" ], "desc": "Undocumented transformations are a primary vector for unintentional bias introduction and a barrier to debugging model behavior. When a normalization step silently changes the semantics of a feature, the lineage record cannot surface the change for review. Malicious insiders or compromised pipeline stages may introduce manipulative transformations that alter AI outcomes without detection if transformation records are incomplete." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 12", "title": "Metadata Management — transformation lineage" }, { "id": "iso_8000", "section": "ISO 8000-120", "title": "Master data — provenance of characteristic data" }, { "id": "eu_ai_act", "section": "Art. 10(2)(c)", "title": "Data-preparation processing operations (annotation, cleaning, enrichment, aggregation)" } ], "sources": [ { "id": "google_dataplex_bigquery_2024", "title": "Google Cloud Dataplex & BigQuery", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://cloud.google.com/dataplex/docs/introduction", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "google_dataplex_bigquery_2024", "relationship": "informative_reference", "rationale": "Establishes Google Cloud Dataplex & BigQuery requirements informing the apeiris://data/controls/DL-04 Transformation and Enrichment Lineage control.", "reviewed_on": "2026-07-01" }, { "id": "databricks_unity_catalog_2024", "title": "Databricks Unity Catalog", "authority": "Databricks, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.databricks.com/en/data-governance/unity-catalog/index.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "databricks_unity_catalog_2024", "relationship": "informative_reference", "rationale": "Establishes Databricks Unity Catalog requirements informing the apeiris://data/controls/DL-04 Transformation and Enrichment Lineage control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Capture transformation metadata at each pipeline stage using a combination of automated instrumentation and explicit transformation registration. Each transformation record must include: source dataset reference, transformation type and logic hash, applying system, configuration version, operator identity, and output dataset reference. Store transformation records as first-class lineage nodes in the lineage graph.", "steps": [ "Define a transformation metadata schema covering: transformation type (normalization, aggregation, join, enrichment, derivation), logic hash, applying system identifier, configuration version, operator identity, and timestamps.", "Instrument transformation engines (Spark, dbt, Airflow) to automatically emit transformation metadata events using the defined schema at each pipeline execution.", "Register all manually authored transformation logic (SQL scripts, Python functions, configuration files) in a version-controlled transformation catalog with semantic change detection.", "Link each transformation record to its upstream source and downstream output in the lineage graph, enabling complete transformation path reconstruction from any dataset." ], "data_governance_officer": { "summary": "Transformation lineage is essential for understanding how raw source data has been semantically altered before AI consumption. It is the primary tool for investigating bias and data quality issues in AI model behavior.", "actions": [ "Define the policy for transformation approval: which transformation types require pre-approval by the data governance team before production deployment.", "Establish a quarterly review of transformation records for AI-bound datasets to identify undocumented or semantically significant changes.", "Track the transformation documentation completeness rate as a data quality KPI." ], "failure_signals": [ "Transformation records missing logic hashes or operator identity for more than 5% of transformations in AI pipelines.", "Undocumented transformations discovered during post-incident investigation.", "Transformation changes deployed to production without governance review for approved transformation types." ] }, "data_engineer": { "summary": "Transformation instrumentation must be standardized across all pipeline frameworks so that coverage is consistent regardless of the tooling used for a given pipeline.", "actions": [ "Build or configure transformation metadata emission into all pipeline framework templates (dbt macros, Spark job wrappers, Airflow operator base classes).", "Implement semantic change detection for registered transformation logic so that modifications to transformation code trigger review workflows.", "Create a transformation lineage visualization in the data catalog showing the complete transformation path for each AI feature." ], "failure_signals": [ "Pipeline templates deployed without transformation metadata emission.", "Transformation logic changes not triggering semantic change detection.", "Lineage graph has transformation path gaps for AI-bound features." ] }, "grc_auditor": { "summary": "Transformation lineage completeness is the evidence basis for demonstrating that data preparation operations for AI systems are documented as required by EU AI Act Article 10(3) and for validating data integrity in regulated AI workflows.", "actions": [ "Verify that all transformation stages in AI data pipelines have complete transformation records in the lineage graph.", "Sample 5 AI features and trace their transformation path from source to model input, documenting any gaps.", "Review transformation approval records for high-impact transformations to verify the governance process was followed." ], "metrics": [ "Transformation documentation completeness rate for AI pipelines: target 100%.", "Percentage of transformation changes with complete operator identity and timestamp: target 100%.", "Percentage of high-impact transformations with documented governance approval: target 100%." ], "failure_signals": [ "Transformation records missing required fields in AI data pipelines.", "Transformation changes without operator identity attribution.", "High-impact transformation deployments without documented governance approval." ] }, "it_operations": { "summary": "Transformation metadata systems must be monitored for event loss and schema drift that would cause gaps in the transformation lineage record.", "actions": [ "Monitor transformation event emission rates and alert on drops below expected rates for active pipelines.", "Enforce schema version management for transformation metadata to prevent silent schema drift causing unmapped fields.", "Include transformation metadata stores in the disaster recovery scope with appropriate backup frequency." ], "failure_signals": [ "Transformation event emission rate drops below 95% of expected for active pipelines.", "Schema version mismatches causing transformation metadata fields to be dropped silently.", "Transformation metadata store not included in backup and recovery testing." ] }, "legal_counsel": { "summary": "Transformations can create legal exposure — enrichment can turn benign data into profiling, and aggregation can fail to anonymize. Counsel reviews what transformations do to the legal status of data.", "actions": [ "Review enrichment steps that combine datasets for re-identification and profiling implications.", "Confirm claimed anonymization or pseudonymization steps are documented in lineage with method and version.", "Advise when a transformation changes lawful-basis requirements for downstream use." ], "failure_signals": [ "'Anonymized' datasets with no recorded anonymization step or method.", "Enrichment joins creating special-category inferences without review.", "Downstream teams relying on transformed data whose legal status nobody can state." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "dbt and modern data stack tools provide partial transformation lineage for SQL transformations. Coverage gaps are common for Spark workloads, Python scripts, and third-party enrichment services." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "federated-enterprise" ], "implementers": [ "Data Engineering", "Data Governance Office", "Platform Engineering" ], "frameworks": [ { "framework": "dama_dmbok", "requirement_id": "Ch. 12", "fit": "direct", "rationale": "DAMA-DMBOK2 Chapter 12 calls for lineage metadata to record the transformations applied to data, not just its movement. Transformation and enrichment lineage captures the transformation logic identity and version behind each derived field.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "iso_8000", "requirement_id": "ISO 8000-120", "fit": "direct", "rationale": "ISO 8000-120 specifies requirements for capturing and exchanging data provenance as characteristic data — the record of where a value came from and how it changed. Transformation and enrichment lineage is the pipeline-level implementation of provenance capture.", "normative_force": "voluntary-standard", "source_version": "2015", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "eu_ai_act", "requirement_id": "Art. 10(2)(c)", "fit": "direct", "rationale": "EU AI Act Article 10(2)(c) requires data governance covering relevant data-preparation processing operations, such as annotation, labelling, cleaning, updating, enrichment and aggregation. Transformation and enrichment lineage records exactly these operations, satisfying the documentation dimension of the requirement; Article 10(3) states dataset quality criteria, not preparation-operation documentation.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "databricks_unity", "requirement_id": "Column-Level Lineage", "fit": "direct", "rationale": "Databricks Unity Catalog provides column-level lineage tracking that captures transformation operations at the finest granularity required by this control. For organizations using Databricks, Unity Catalog column lineage satisfies the transformation documentation requirements for Spark and SQL workloads.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "google_dataplex", "requirement_id": "Data Lineage API", "fit": "direct", "rationale": "Google Cloud Dataplex Data Lineage API captures transformation lineage for BigQuery and Dataflow workloads and supports custom lineage events via API for other transformation engines. Organizations on GCP can use Dataplex to satisfy transformation lineage documentation requirements for their AI data pipelines.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "owasp_ai_exchange", "requirement_id": "supplychainmanage", "fit": "supporting", "rationale": "DL-04 records every transformation/enrichment as a first-class lineage node with logic_hash and operator identity, tracking the data supply chain through processing steps.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "mitre", "requirement_id": "AML.M0023", "fit": "supporting", "direction": "control-supports-requirement", "rationale": "Control \"Every transformation, normalization, enrichment, join, aggregation, or derivation applied…\" enacts ATLAS mitigation AML.M0023 AI Bill of Materials; OpenCRE crosswalks this control’s OWASP AI Exchange concept (supplychainmanage) to this mitigation.", "source_version": "v2026.06", "reviewed_on": "2026-07-08", "source_status": "authoritative", "mapping_confidence": "medium", "legal_status": "not-applicable", "control_readiness": "approved", "implementation_maturity": "developing", "assurance_result": "not-assessed", "evidence_status": "no-evidence", "normative_force": "industry-framework", "basis": "asserted", "relation": "equivalent_to", "correction": "ap38-opencre-enhancement 2026-07-08" }, { "framework": "mitre", "requirement_id": "AML.M0014", "fit": "supporting", "direction": "control-supports-requirement", "rationale": "Control \"Every transformation, normalization, enrichment, join, aggregation, or derivation applied…\" enacts ATLAS mitigation AML.M0014 Verify AI Artifacts; OpenCRE crosswalks this control’s OWASP AI Exchange concept (supplychainmanage) to this mitigation.", "source_version": "v2026.06", "reviewed_on": "2026-07-08", "source_status": "authoritative", "mapping_confidence": "medium", "legal_status": "not-applicable", "control_readiness": "approved", "implementation_maturity": "developing", "assurance_result": "not-assessed", "evidence_status": "no-evidence", "normative_force": "industry-framework", "basis": "asserted", "relation": "equivalent_to", "correction": "ap38-opencre-enhancement 2026-07-08" }, { "framework": "nist_ai_100_2", "requirement_id": "NISTAML.05", "fit": "supporting", "rationale": "Control \"Every transformation, normalization, enrichment, join, aggregation, or derivation applied…\" defends against NIST AI 100-2 attack class NISTAML.05 \"Supply Chain Attacks\"; OpenCRE crosswalks this control’s OWASP AI Exchange concept (supplychainmanage) to this NIST class. (Anchor captures the attack taxonomy, so we cite the threat class as defends_against, not the NIST mitigation named by OpenCRE.)", "normative_force": "informative-reference", "source_version": "100-2e2025", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "defends_against", "mapping_confidence": "medium", "correction": "ap38-opencre-enhancement 2026-07-08" } ], "canonical_id": "apeiris://data/controls/DL-04", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every transformation, normalization, enrichment, join, aggregation, or derivation applied to data between its source and AI consumption must be present as a first-class lineage node in the lineage graph, containing transformation_type, logic_hash, applying_system_id, configuration_version, operator_identity, and input/output dataset timestamps — with no transformation path gaps for any AI-bound feature.", "evidence_required": [ "lineage_graph_transformation_path_export for a representative sample of AI-bound features, showing the complete transformation path from source to model input with all required metadata fields populated on each transformation node", "transformation_metadata_emission_rate_report confirming the percentage of pipeline runs that successfully emitted transformation metadata events with all required fields, with target of 100% for AI-bound pipelines", "semantic_change_detection_log showing that modifications to registered transformation logic triggered review workflows and were not deployed to production without operator identity attribution and version increment", "transformation_governance_approval_records for high-impact transformations (aggregations, enrichment joins, normalization rules) confirming governance review was completed and documented before production deployment" ], "machine_tests": [ "Deploy a data pipeline to a non-production environment without transformation metadata emission → assert the CI/CD linting check fails with error=missing_transformation_instrumentation before deployment proceeds", "Execute a transformation pipeline and query the lineage graph for the output dataset → assert all transformation nodes in the path have non-null operator_identity, logic_hash, and applying_system_id fields", "Modify a registered transformation logic file without committing a version increment → assert semantic change detection flags the modification and blocks deployment with status=unversioned_transformation_change" ], "human_review": [ "Manually trace the transformation lineage path for 5 AI-bound features from source to model input and verify no undocumented transformation steps exist between lineage nodes, with each node carrying complete metadata", "Review transformation governance approval records for high-impact transformations deployed in the past quarter to confirm operator identity is attributed and governance review was completed before production deployment", "Assess whether the transformation metadata schema enforces logic_hash computation at instrumentation time rather than allowing operators to self-report hashes, which would eliminate tamper detection for malicious transformation changes" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Documenting transformation logic in code comments or README files rather than as structured lineage node metadata with computed logic hashes, making tamper detection and automated completeness verification impossible", "Implementing transformation lineage only for SQL-based transformations (dbt, Spark SQL) while leaving Python script transformations, UDF-based enrichments, and third-party API enrichments entirely undocumented", "Using shared generic system account identities as operator_identity on transformation records instead of the individual who authored and deployed the transformation configuration, eliminating accountability for changes", "Capturing transformation lineage at pipeline completion rather than at each individual transformation step execution, creating coarse records that cannot differentiate between transformation steps within a multi-step pipeline", "Treating schema-level lineage (table-to-table) as sufficient transformation documentation without capturing logic_hash and configuration_version for each transformation, allowing logic changes to go undetected between schema versions" ], "update_status": "current", "layer_code": "DL" }, { "id": "DL-05", "layer": "DL", "plane": "control", "name": "Data Lineage Completeness Auditing", "plain": "The organization must continuously verify that lineage tracking is complete and accurate for all AI-bound data flows, including detection of untracked data sources, broken lineage chains, stale lineage records, and lineage metadata that does not match actual data flow behavior.", "threat": { "tags": [ "lineage-coverage-gap", "shadow-data-flow", "stale-lineage", "audit-blind-spot" ], "desc": "Lineage systems accumulate coverage gaps as new data sources are onboarded without instrumentation, pipelines are modified without updating lineage registrations, and data flows change without triggering lineage updates. Shadow data flows — pipelines that bypass the lineage-instrumented path — are a common source of undocumented AI training or inference data. When lineage appears complete but is actually inaccurate, it provides false assurance that undermines all downstream compliance and audit activity." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 12", "title": "Metadata Management — lineage quality and completeness" }, { "id": "dcam", "section": "Capability 5.1", "title": "The data quality management function is established" }, { "id": "cobit_2019", "section": "MEA02.04", "title": "Identify and report control deficiencies" }, { "id": "eu_ai_act", "section": "Art. 17(1)(f)", "title": "Quality management system for data governance" } ], "sources": [ { "id": "microsoft_purview_data_governance_2024", "title": "Microsoft Purview (Data Governance)", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/purview/purview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_purview_data_governance_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Purview (Data Governance) requirements informing the apeiris://data/controls/DL-05 Data Lineage Completeness Auditing control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Deploy a lineage completeness auditing process that combines automated graph analysis (detecting orphan nodes, broken chains, and coverage gaps) with periodic reconciliation between observed data flows and registered lineage records. Integrate completeness scores into data governance dashboards and trigger remediation workflows for gaps.", "steps": [ "Implement automated lineage graph analysis that runs daily, detecting: orphaned nodes (datasets with no upstream or downstream connections), broken chains (missing links between known pipeline stages), and coverage gaps (datasets appearing in AI pipelines without lineage records).", "Deploy network-level or catalog-level data flow discovery to identify data sources accessed by AI systems that are not registered in the lineage graph, surfacing shadow data flows.", "Establish a lineage completeness score metric (percentage of AI-bound data assets with complete, unbroken lineage from source to model) tracked in the data governance dashboard.", "Implement a remediation workflow that assigns lineage gaps to responsible data domain owners with SLA-bound closure deadlines and escalation to the data governance board for persistent gaps." ], "data_governance_officer": { "summary": "Lineage completeness auditing turns lineage from a passive metadata record into an active governance control. The completeness score is the primary indicator of whether the organization's lineage program is functioning.", "actions": [ "Set the lineage completeness target (100% for AI-bound assets in high-risk domains) and track it as a board-level data governance KPI.", "Chair the monthly lineage completeness review, escalating persistent gaps to data domain owners and senior leadership.", "Require a lineage completeness certification before any new AI system is approved for production deployment." ], "failure_signals": [ "Lineage completeness score below target for two consecutive monthly reviews.", "Shadow data flows discovered that have been in use for more than 30 days without lineage registration.", "AI systems deployed to production without lineage completeness certification." ] }, "data_engineer": { "summary": "Automated lineage completeness analysis must be integrated into the data platform operations cycle so gaps are detected and remediated continuously rather than discovered during audits.", "actions": [ "Build and deploy the lineage graph analysis job that detects orphans, broken chains, and coverage gaps on a daily schedule.", "Implement data flow discovery scanning that compares observed network or catalog access patterns against registered lineage.", "Create the lineage completeness dashboard and integrate it with the team's operational monitoring stack." ], "failure_signals": [ "Lineage graph analysis job failing to run for more than 48 hours.", "Data flow discovery not configured for AI serving infrastructure.", "Lineage completeness dashboard not updated within 24 hours of gap detection." ] }, "grc_auditor": { "summary": "Lineage completeness auditing is the internal control that validates all other lineage controls. Without it, lineage coverage cannot be independently verified and assurance relies entirely on implementer self-reporting.", "actions": [ "Review the lineage completeness score trend for the past 12 months and assess whether the program is improving or degrading.", "Test the shadow data flow detection capability by verifying that a known test data source not registered in lineage is flagged by the discovery scan.", "Verify that gap remediation SLAs are documented, tracked, and met for the most recent audit period." ], "metrics": [ "Lineage completeness score for AI-bound assets: target 100%.", "Mean time to remediate lineage gaps: target ≤ 10 business days.", "Percentage of gap remediation tasks closed within SLA: target 95%." ], "failure_signals": [ "Lineage completeness score trending downward for three consecutive months.", "Shadow data flow detection capability not tested in the past quarter.", "Gap remediation SLA breaches unescalated for more than 30 days." ] }, "it_operations": { "summary": "The automated lineage auditing jobs must be monitored as operational services with defined uptime requirements and failure alerting.", "actions": [ "Configure monitoring for the lineage graph analysis job with alerting on failure or missed schedule.", "Ensure the data flow discovery scanner has appropriate network access to observe AI serving and training infrastructure.", "Include lineage auditing job failures in the operations incident management process." ], "failure_signals": [ "Lineage graph analysis job missed more than 2 consecutive scheduled runs.", "Data flow discovery scanner lacking network visibility into AI infrastructure subnets.", "Lineage auditing job failures not generating operational incidents." ] }, "legal_counsel": { "summary": "A lineage program with unknown gaps gives false comfort. Counsel needs to know coverage boundaries before relying on lineage in filings, discovery responses, or DPIAs.", "actions": [ "Require coverage reporting for the data domains counsel relies on — personal data, training data, regulated records.", "Caveat legal reliance on lineage where audits show material gaps.", "Prioritize gap remediation on flows with active legal obligations." ], "failure_signals": [ "Legal responses built on lineage later shown to be incomplete.", "Coverage audits skipped for the systems most often cited in filings.", "Known gaps on regulated flows carried across multiple audit cycles." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Automated lineage completeness auditing with shadow flow detection is a practice that very few organizations have implemented. Most rely on manual reviews and self-reporting from pipeline teams." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "federated-enterprise", "eu-high-risk-ai" ], "implementers": [ "Data Governance Office", "Data Engineering", "GRC / Internal Audit" ], "frameworks": [ { "framework": "dama_dmbok", "requirement_id": "Ch. 12", "fit": "direct", "rationale": "DAMA-DMBOK2 Chapter 12 treats metadata quality — including lineage completeness and currency — as something to measure and manage like any other data quality problem. Lineage completeness auditing operationalizes that measurement.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "dcam", "requirement_id": "Capability 5.1", "fit": "partial", "rationale": "DCAM v2.2 capability 5.1 requires the data quality management function to be established with defined processes. Lineage completeness auditing applies that established DQ discipline to lineage metadata itself — measuring coverage and currency like any other quality dimension.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_2019", "requirement_id": "MEA02.04", "fit": "partial", "rationale": "COBIT 2019 practice MEA02.04 requires identifying and reporting control deficiencies found through continuous monitoring. Lineage completeness auditing is a continuous-monitoring control whose output — coverage gaps — must be reported and remediated through exactly this practice. MEA02 practices end at MEA02.04; the previously cited practice number does not exist in COBIT 2019.", "normative_force": "best-practice", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "eu_ai_act", "requirement_id": "Art. 17(1)(f)", "fit": "partial", "rationale": "EU AI Act Article 17(1)(f) requires providers of high-risk AI systems to establish a quality management system covering data management practices. Lineage completeness auditing is a component of the data management quality system required by this provision.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "microsoft_purview", "requirement_id": "Data Estate Insights", "fit": "direct", "rationale": "Microsoft Purview Data Estate Insights provided lineage coverage and asset-health reporting in the classic Purview portal; in the new unified Purview experience this reporting is folded into data estate health and data quality dashboards. Organizations should verify which reporting surface their tenant exposes before relying on it for completeness auditing.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://data/controls/DL-05", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "The lineage completeness auditing system must achieve and maintain a completeness score of 100% for all AI-bound data assets, with zero shadow data flows undetected for more than 48 hours. Gap detection must run on a daily automated schedule and produce a completeness report accessible in the governance dashboard within 24 hours of each scan cycle.", "evidence_required": [ "lineage_completeness_report with completeness_score, asset_count, coverage_gap_list, and report_generated_at timestamp confirming the report is no older than 30 days", "shadow_data_flow_detection_log showing network or catalog discovery scan results with identified_unregistered_flows list and scan_timestamp for the most recent scan cycle", "gap_remediation_record showing lineage_gap_id, assigned_owner, sla_deadline, and closure_date for every gap identified during the audit period", "lineage_completeness_dashboard export or screenshot showing current completeness score and 3-month trend with last_updated timestamp" ], "machine_tests": [ "Introduce a test dataset accessed by a mock AI pipeline that is not registered in the lineage graph → assert the discovery scan detects the orphan node within 24 hours and generates a gap record with dataset_id and detection_timestamp", "Remove a lineage link between two pipeline stages in the test environment → assert automated graph analysis detects the broken chain within one daily scan cycle and creates a remediation ticket with assigned_owner populated", "Query the lineage completeness score API → assert the returned report was generated within the past 24 hours, completeness_score field is present, and no gap records are in suppressed or hidden state" ], "human_review": [ "Review the monthly lineage completeness score trend for the past 12 months and assess whether the program shows sustained improvement, stagnation, or degradation relative to the 100% target", "Inspect three sampled gap remediation records and verify that assigned owners received timely notifications and closed gaps within the documented SLA deadline", "Confirm the shadow data flow detection scanner covers AI serving and training infrastructure subnets by reviewing scanner network access configuration" ], "blocking_effect": "advisory", "normative_status": "industry-framework", "anti_patterns": [ "Treating lineage completeness as a one-time implementation checklist rather than a continuous monitoring capability with daily automated scanning and gap remediation SLAs", "Accepting completeness scores below 100% for AI-bound assets as operationally acceptable without a documented exception and time-bound remediation plan approved by the data governance officer", "Relying solely on pipeline team self-reporting to identify lineage coverage gaps instead of deploying independent discovery scanning that compares observed data flows against registered lineage", "Calculating completeness scores only against assets already registered in the lineage catalog without reconciling against actual observed network or catalog access patterns that would surface shadow pipelines", "Allowing gap remediation records to age beyond SLA without escalation, normalizing a state of chronic incompleteness that provides false assurance to auditors reviewing the completeness score alone" ], "update_status": "current", "layer_code": "DL" }, { "id": "DL-06", "layer": "DL", "plane": "control", "name": "Data Lineage for Regulatory and Legal Purposes", "plain": "The organization must be able to export lineage records for any AI data flow in structured formats required by regulators, auditors, and courts, including timestamped provenance chains, transformation histories, and retention-compliant lineage archives that can be produced in response to legal process.", "threat": { "tags": [ "regulatory-production-failure", "legal-discovery-gap", "retention-noncompliance", "tampered-lineage-record" ], "desc": "Regulators under the EU AI Act, GDPR, and sector-specific AI rules may request production of AI data lineage records during conformity assessments or investigations. Organizations that cannot produce structured, verifiable lineage exports face enforcement action and fines. In litigation, inability to produce complete discovery records may result in adverse inferences or sanctions. Lineage records that lack tamper-evidence controls are subject to admissibility challenges." }, "standard": [ { "id": "eu_ai_act", "section": "Art. 18", "title": "Documentation and record-keeping obligations" }, { "id": "gdpr", "section": "Art. 5(2)", "title": "Accountability principle" }, { "id": "iso_27701", "section": "§7.2.8", "title": "Records related to processing of PII" }, { "id": "cobit_2019", "section": "MEA03.04", "title": "Ensure compliance with legal requirements" } ], "sources": [ { "id": "eu_ai_act", "title": "EU AI Act", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2024/1689", "published_on": "2024-07-12", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2024/1689/oj/eng", "license": "open-access", "status": "current", "flagship": false, "source_id": "eu_ai_act", "relationship": "normative_requirement", "rationale": "Establishes EU AI Act Art. 18 requirements informing the apeiris://data/controls/DL-06 Data Lineage for Regulatory and Legal Purposes control.", "reviewed_on": "2026-07-02" }, { "id": "gdpr", "title": "GDPR", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2016/679", "published_on": "2016-04-27", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj", "license": "open-access", "status": "current", "flagship": false, "source_id": "gdpr", "relationship": "normative_requirement", "rationale": "Establishes GDPR Art. 5(2) requirements informing the apeiris://data/controls/DL-06 Data Lineage for Regulatory and Legal Purposes control.", "reviewed_on": "2026-07-02" }, { "id": "iso_27701", "title": "ISO/IEC 27701:2019", "authority": "ISO/IEC", "source_type": "standard", "normative_force": "certification-standard", "version": "2019 (superseded by ISO/IEC 27701:2025, published 2025-10-14; 24-36 month certification transition)", "published_on": "2019-08-06", "retrieved_on": "2026-07-02", "canonical_url": "https://www.iso.org/standard/71670.html", "license": "proprietary", "status": "current", "flagship": false, "source_id": "iso_27701", "relationship": "implementation_pattern", "rationale": "Establishes ISO/IEC 27701:2019 §7.2.8 requirements informing the apeiris://data/controls/DL-06 Data Lineage for Regulatory and Legal Purposes control.", "reviewed_on": "2026-07-02" }, { "id": "cobit_2019", "title": "COBIT 2019", "authority": "ISACA", "source_type": "standard", "normative_force": "industry-framework", "version": "2019", "published_on": "2018-11-12", "retrieved_on": "2026-07-02", "canonical_url": "https://www.isaca.org/resources/cobit", "license": "proprietary", "status": "current", "flagship": false, "source_id": "cobit_2019", "relationship": "informative_reference", "rationale": "Establishes COBIT 2019 MEA03.04 requirements informing the apeiris://data/controls/DL-06 Data Lineage for Regulatory and Legal Purposes control.", "reviewed_on": "2026-07-02" }, { "id": "nist_pf", "title": "NIST Privacy Framework 1.0", "authority": "National Institute of Standards and Technology", "source_type": "standard", "normative_force": "voluntary-standard", "version": "1.0", "published_on": "2020-01-16", "retrieved_on": "2026-07-02", "canonical_url": "https://www.nist.gov/privacy-framework", "license": "open-access", "status": "current", "flagship": false, "source_id": "nist_pf", "relationship": "implementation_pattern", "rationale": "Establishes NIST Privacy Framework 1.0 GV.PO-P5 requirements informing the apeiris://data/controls/DL-06 Data Lineage for Regulatory and Legal Purposes control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Implement a lineage export service that generates structured, tamper-evident lineage packages on demand or on schedule. Support multiple export formats (JSON-LD, W3C PROV, CSV) for different regulatory audiences. Apply hash-chaining or digital signatures to exported packages to ensure integrity is verifiable by recipients. Maintain a retention-compliant lineage archive with documented retention schedules by data category and jurisdiction.", "steps": [ "Implement a lineage export API that accepts a dataset or model ID and returns a complete lineage package in the requested format (JSON-LD with W3C PROV vocabulary, structured CSV, or regulator-specified format), covering all upstream sources, transformations, and downstream AI uses.", "Apply SHA-256 hashing and optional digital signing to all lineage export packages so recipients can verify the package has not been altered since export.", "Establish and document lineage archive retention schedules by jurisdiction: minimum 10 years for EU AI Act high-risk AI documentation, aligned with applicable data retention laws.", "Conduct annual dry-run regulatory productions by generating lineage packages for a sample of AI systems and having Legal review them for completeness and format compliance before a real request arrives." ], "data_governance_officer": { "summary": "Regulatory-grade lineage export is the operational capability that converts internal lineage records into externally producible compliance evidence. This capability must be tested proactively, not discovered to be deficient during an actual regulatory inquiry.", "actions": [ "Define the standard lineage export formats for each regulatory jurisdiction where the organization operates AI systems.", "Coordinate with Legal to establish the process for responding to regulatory lineage production requests, including response timelines and authorized signatories.", "Ensure lineage archive retention schedules are formally adopted and implemented in the data management system." ], "failure_signals": [ "Lineage export API unable to produce complete packages for any AI system within 48 hours of a request.", "Lineage export packages failing tamper-evidence verification checks.", "Lineage archive retention schedules not formalized or not implemented in the data platform." ] }, "legal_counsel": { "summary": "Lineage records for regulatory and legal purposes are critical evidence artifacts. Legal must define the production requirements and review export packages for adequacy before they are needed in response to actual demands.", "actions": [ "Define the evidentiary requirements for lineage packages in each regulatory jurisdiction, including format, completeness, and authentication requirements.", "Review the lineage export package schema and confirm it meets the documentation standards required by the EU AI Act, GDPR accountability principle, and applicable sector regulations.", "Establish the legal hold and litigation-hold procedures for lineage archives when regulatory investigation or litigation is anticipated." ], "failure_signals": [ "No Legal review of lineage export package format and completeness in the past 12 months.", "Lineage archives subject to litigation hold being modified or deleted.", "No documented process for responding to regulatory lineage production demands within required timelines." ] }, "grc_auditor": { "summary": "The regulatory lineage export capability must be tested annually to confirm it can produce complete, tamper-evident packages for all AI systems within the required response timeline.", "actions": [ "Conduct an annual drill: request lineage export packages for 3 production AI systems and evaluate completeness, format compliance, and tamper-evidence integrity.", "Verify that retention schedules for lineage archives are documented, implemented, and enforced by the data platform.", "Review the process for responding to regulatory lineage demands and confirm response timelines are achievable with the current tooling." ], "metrics": [ "Time to produce a complete lineage package for any AI system: target ≤ 48 hours.", "Lineage export package integrity verification success rate: target 100%.", "Percentage of AI systems with lineage archives meeting documented retention schedules: target 100%." ], "failure_signals": [ "Annual drill producing incomplete lineage packages for any sampled AI system.", "Lineage export packages failing tamper-evidence verification.", "Lineage archives with retention gaps or premature deletion." ] }, "it_operations": { "summary": "The lineage archive must be maintained as a compliance-grade storage system with immutability controls, access logging, and a retention enforcement mechanism.", "actions": [ "Deploy the lineage archive on immutable or WORM storage with access logging for every read or export operation.", "Implement automated retention enforcement that prevents deletion of lineage records before the scheduled retention expiry, with override requiring documented approval.", "Monitor lineage archive storage capacity and access rates, escalating anomalies that could indicate unauthorized access or premature deletion attempts." ], "failure_signals": [ "Lineage archive storage not configured as immutable or WORM.", "Access to lineage archives not logged at the individual record level.", "Retention enforcement controls allowing deletion before scheduled expiry without documented approval." ] }, "data_engineer": { "summary": "Regulatory lineage production is an engineering deliverable: export tooling that turns internal lineage graphs into complete, regulator-consumable extracts within the response deadline.", "actions": [ "Build and maintain lineage export tooling that produces structured, self-contained extracts for a named dataset, decision or period.", "Test regulatory exports on a schedule — completeness, format validity and time-to-produce against the response SLA.", "Preserve lineage snapshots for periods under legal hold so exports remain reproducible later." ], "failure_signals": [ "Lineage exports assembled manually under deadline pressure.", "Export tests failing on completeness or taking longer than the response SLA.", "Lineage for held periods aged out before the hold was released." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most organizations have not yet been subject to formal regulatory lineage production requests for AI systems. Building the export and archive capability before it is demanded is a maturity gap that will become critical as AI regulation enforcement increases." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "high-risk-sector", "eu-high-risk-ai", "federated-enterprise", "universal-enterprise" ], "implementers": [ "Data Governance Office", "Legal / Compliance", "IT Operations", "Data Engineering" ], "frameworks": [ { "framework": "eu_ai_act", "requirement_id": "Art. 18", "fit": "direct", "rationale": "EU AI Act Article 18 requires providers of high-risk AI systems to keep technical documentation and logs for a period of at least 10 years after the system is placed on the market. Regulatory-grade lineage export directly implements this record-keeping obligation by producing structured, verifiable documentation on demand.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "gdpr", "requirement_id": "Art. 5(2)", "fit": "direct", "rationale": "GDPR Article 5(2) establishes the accountability principle requiring controllers to be able to demonstrate compliance with data protection obligations. Lineage records that can be produced in structured, verifiable formats are the technical mechanism for demonstrating accountability to supervisory authorities.", "normative_force": "binding-law", "source_version": "2016/679", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "iso_27701", "requirement_id": "§7.2.8", "fit": "direct", "rationale": "ISO/IEC 27701:2019 §7.2.8 requires organizations to determine and securely maintain the records necessary to demonstrate compliance with PII-processing obligations. Regulator- and litigation-ready lineage exports are precisely such demonstrable records for AI data flows. §7.4.7 governs retention periods, not record production.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_2019", "requirement_id": "MEA03.04", "fit": "partial", "rationale": "COBIT 2019 MEA03.04 requires processes to ensure the organization can demonstrate compliance with legal and regulatory requirements. The regulatory lineage export capability is a direct implementation of this requirement for AI-related legal and regulatory obligations.", "normative_force": "best-practice", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_pf", "requirement_id": "GV.PO-P5", "fit": "adjacent", "rationale": "NIST Privacy Framework GV.PO-P5 requires legal, regulatory, and contractual privacy requirements to be understood and managed. Regulator- and litigation-ready lineage is a managed response to those requirements: the organization knows which obligations demand lineage production and can meet them. GV.PO-P3 concerns workforce roles.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://data/controls/DL-06", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "The organization must be able to produce a complete, tamper-evident lineage export package for any AI system within 48 hours of a regulatory or legal request, in a structured format (W3C PROV or JSON-LD) verifiable by the recipient via SHA-256 hash. All lineage archives must be stored on immutable or WORM storage with retention periods meeting or exceeding the applicable regulatory minimum (10 years for EU AI Act high-risk systems).", "evidence_required": [ "lineage_export_package for a sampled AI system containing upstream source provenance chain, transformation history, and downstream AI uses in W3C PROV or JSON-LD format with computed SHA-256 package hash and export_timestamp", "export_integrity_verification_report confirming the package hash matches the hash recorded in the compliance registry at time of assembly", "lineage_archive_retention_schedule document showing retention_period by data_category and jurisdiction, with formal adoption date and implemented_in_platform=true confirmation", "annual_regulatory_drill_report listing 3 sampled AI systems, export completeness assessment per system, Legal sign-off on format compliance, and drill_completion_date" ], "machine_tests": [ "Request lineage export for a test AI system via the export API → assert the complete package is returned within 48 hours containing source-to-model provenance chain, all transformation records, and a valid SHA-256 package hash field", "Compute SHA-256 hash of the returned export package and compare against the hash recorded in the compliance registry → assert hashes match exactly with zero-byte divergence", "Query the lineage archive for a system onboarded more than 10 years ago → assert all lineage records for that system are present and the storage system returns retention_enforced=true with no early-deletion events in the access log" ], "human_review": [ "Review the most recent annual regulatory drill report and confirm Legal signed off on export package completeness and format compliance for all three sampled AI systems before the drill record was finalized", "Verify the lineage archive storage configuration is WORM or immutable by reviewing storage tier documentation and confirming access logs capture every read and export operation at the individual record level", "Confirm the documented regulatory production response process assigns named roles, target response timelines, and authorized signatories, and that it has been reviewed by Legal within the past 12 months" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Storing lineage archives in mutable object storage where records can be deleted or overwritten without generating an immutable audit trail, undermining the tamper-evidence claim required for regulatory admissibility", "Using proprietary lineage export formats not recognized by regulatory audiences, requiring manual translation during actual regulatory productions under time pressure", "Relying on ad-hoc lineage extraction from live operational systems at the time of a regulatory request rather than maintaining pre-assembled, hash-verified archives that can be produced within hours", "Setting lineage archive retention periods based on storage cost optimization rather than the most stringent applicable regulatory requirement, resulting in premature deletion of records needed for EU AI Act 10-year obligations", "Failing to conduct annual production drills, discovering export deficiencies in completeness, format, or integrity only when a real regulatory request arrives with a non-negotiable response deadline" ], "update_status": "current", "layer_code": "DL" }, { "id": "DL-07", "layer": "DL", "plane": "control", "name": "Data Lineage Tooling Governance", "plain": "The organization must select, configure, operate, and maintain the tooling that captures and exposes data lineage metadata, ensuring that lineage tools are evaluated against defined capability requirements, integrated with data platform governance, and kept current to prevent coverage gaps from tool limitations or version drift.", "threat": { "tags": [ "tool-coverage-gap", "vendor-lock-in-lineage", "lineage-schema-drift", "stale-tooling" ], "desc": "Lineage tools that are not actively governed accumulate coverage gaps as the data platform evolves and the tool's supported connectors fall behind new data services and pipeline frameworks. Organizations that rely on a single vendor's proprietary lineage format face lock-in that prevents migration and limits the ability to produce lineage records in formats required by other tools or regulators. Stale lineage tooling versions may have known vulnerabilities that expose metadata records to unauthorized access or manipulation." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 12", "title": "Metadata Management — metadata architecture and tooling" }, { "id": "dcam", "section": "Capability 6.1", "title": "The data governance function is established" }, { "id": "cobit_2019", "section": "APO04.04", "title": "Assess the potential of emerging technologies and innovation ideas" }, { "id": "iso_8000", "section": "ISO 8000-61", "title": "Data quality management — process reference model" } ], "sources": [ { "id": "snowflake_horizon_data_governance_2024", "title": "Snowflake Horizon (Data Governance)", "authority": "Snowflake Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.snowflake.com/en/data-cloud/workloads/data-governance/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "snowflake_horizon", "relationship": "informative_reference", "rationale": "Establishes Snowflake Horizon (Data Governance) requirements informing the apeiris://data/controls/DL-07 Data Lineage Tooling Governance control.", "reviewed_on": "2026-07-01" }, { "id": "databricks_unity_catalog_2024", "title": "Databricks Unity Catalog", "authority": "Databricks, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.databricks.com/en/data-governance/unity-catalog/index.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "databricks_unity_catalog_2024", "relationship": "informative_reference", "rationale": "Establishes Databricks Unity Catalog requirements informing the apeiris://data/controls/DL-07 Data Lineage Tooling Governance control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Establish a lineage tooling governance process covering: tool selection criteria, integration standards (OpenLineage API compliance required), coverage requirements by data platform component, patching and upgrade SLAs, and annual capability reviews. Maintain a lineage tool inventory with coverage mapping showing which data platform components are served by each tool.", "steps": [ "Define the lineage tooling selection criteria: OpenLineage API compliance, connector coverage for all data platform components, query performance for graph traversal, export format support (W3C PROV, JSON-LD), and vendor roadmap stability.", "Maintain a lineage tool inventory document mapping each data platform component (ingestion, storage, transformation, serving) to the lineage tool(s) covering it, with coverage gaps explicitly documented.", "Establish patching and upgrade SLAs for lineage tools: critical security patches within 30 days; minor version updates within 90 days. Include lineage tools in the enterprise vulnerability management program.", "Conduct an annual lineage tooling capability review assessing connector coverage gaps for new data services, export format adequacy for current regulatory requirements, and vendor roadmap alignment with the enterprise data platform direction." ], "data_governance_officer": { "summary": "Lineage tooling governance ensures that the technical infrastructure supporting the lineage program keeps pace with the data platform and regulatory requirements. Tool gaps translate directly into lineage coverage gaps that undermine compliance.", "actions": [ "Chair the annual lineage tooling capability review and approve the tooling roadmap based on coverage gap findings.", "Require a lineage tool coverage assessment before any new data platform component is approved for production use with AI workloads.", "Maintain the lineage tool inventory as a governed data governance artifact updated quarterly." ], "failure_signals": [ "New data platform components onboarded to AI pipelines without a lineage tool coverage assessment.", "Lineage tool inventory not updated in more than 90 days.", "Annual lineage tooling capability review not conducted." ] }, "data_engineer": { "summary": "Lineage tool integration is a platform engineering responsibility. Engineers must maintain working integrations between pipeline frameworks and lineage tools as both evolve.", "actions": [ "Maintain the lineage tool connector library for all pipeline frameworks in use, testing connector compatibility with each major version upgrade of pipeline and lineage tooling.", "Document lineage tool coverage gaps in the tool inventory and create remediation plans for any gaps affecting AI-bound data flows.", "Enforce OpenLineage API compliance in lineage tool selection to preserve interoperability and reduce vendor lock-in." ], "failure_signals": [ "Lineage tool connectors failing after pipeline framework upgrades without a remediation plan.", "OpenLineage compliance not verified for newly adopted lineage tooling.", "Coverage gaps in the tool inventory for AI-bound data platform components unresolved for more than 60 days." ] }, "grc_auditor": { "summary": "Lineage tooling governance is the control that ensures the technical foundation of the lineage program is sound. Without governed tooling, lineage coverage claims cannot be independently verified.", "actions": [ "Verify that a current lineage tool inventory exists, is accurate, and maps coverage to all AI-bound data platform components.", "Confirm that lineage tools are included in the enterprise vulnerability management and patching program.", "Review the most recent annual tooling capability review findings and assess whether identified gaps have been remediated." ], "metrics": [ "Percentage of AI-bound data platform components with documented lineage tool coverage: target 100%.", "Mean time to patch critical lineage tool vulnerabilities: target ≤ 30 days.", "Annual lineage tooling capability review completion: target 100% of years." ], "failure_signals": [ "Lineage tool inventory gaps for AI-bound data platform components.", "Critical lineage tool vulnerabilities unpatched beyond the 30-day SLA.", "Annual tooling capability review not completed in the past 12 months." ] }, "it_operations": { "summary": "Lineage tools must be operated, patched, and monitored with the same discipline as other data platform infrastructure components.", "actions": [ "Onboard all lineage tools into the enterprise patch management and vulnerability scanning program.", "Configure operational monitoring for lineage tool availability, performance, and event ingest rates.", "Include lineage tools in the disaster recovery and business continuity plan for data platform infrastructure." ], "failure_signals": [ "Lineage tools not enrolled in vulnerability scanning.", "Lineage tool availability below 99.5% monthly SLA without documented incident and remediation.", "Lineage tools excluded from disaster recovery testing." ] }, "legal_counsel": { "summary": "Lineage records are only as durable as the tooling that produces them. Counsel cares about evidentiary continuity: exports that survive vendor changes and records that meet retention duties.", "actions": [ "Require vendor-neutral export (e.g., OpenLineage) so lineage evidence survives tool migration.", "Set retention requirements for lineage records aligned to the obligations they evidence.", "Review tooling contracts for ownership and portability of lineage data." ], "failure_signals": [ "Lineage history lost or stranded in a decommissioned tool.", "Lineage retention shorter than the retention of the data it describes.", "Vendor contracts silent on export and ownership of lineage records." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Lineage tooling is often selected and deployed by individual data engineering teams without enterprise governance. The result is a fragmented, inconsistent lineage coverage landscape that cannot support organization-wide compliance claims." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "federated-enterprise" ], "implementers": [ "Data Engineering", "Data Governance Office", "IT Operations" ], "frameworks": [ { "framework": "dama_dmbok", "requirement_id": "Ch. 12", "fit": "direct", "rationale": "DAMA-DMBOK2 Chapter 12 covers metadata architecture and tooling — repositories, integration and delivery of metadata including lineage. Lineage tooling governance applies that architecture guidance when selecting and operating lineage tools.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "dcam", "requirement_id": "Capability 6.1", "fit": "partial", "rationale": "DCAM v2.2 capability 6.1 requires the data governance function to be established with defined accountability. Lineage tooling governance sits inside that function: tool selection, standards adoption and coverage accountability are governance-function decisions.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_2019", "requirement_id": "APO04.04", "fit": "partial", "rationale": "COBIT 2019 practice APO04.04 requires assessing the potential of emerging technologies and innovation ideas. Lineage tooling governance applies this assessment discipline to lineage capture technology — evaluating coverage, interoperability (e.g., OpenLineage) and lock-in before adoption.", "normative_force": "best-practice", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "databricks_unity", "requirement_id": "Unity Catalog", "fit": "adjacent", "rationale": "Unity Catalog is Databricks' governance layer; its lineage capture is a platform capability whose coverage boundaries and export options should be assessed under DL-07's tooling governance criteria.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "snowflake_horizon", "requirement_id": "Access History (object dependency lineage)", "fit": "adjacent", "rationale": "Snowflake's Access History exposes object- and column-level dependencies that function as lineage on the Snowflake platform. DL-07 tooling governance should assess its coverage and export path alongside dedicated lineage tools.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://data/controls/DL-07", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every AI-bound data platform component must be mapped in a current lineage tool inventory to at least one OpenLineage-compliant tool, with zero coverage gaps persisting beyond 60 days. All lineage tooling must be enrolled in enterprise vulnerability management with critical security patches applied within 30 days of availability.", "evidence_required": [ "lineage_tool_inventory document mapping each data platform component to its lineage tool(s) with last_updated date no older than 90 days, coverage_gaps[] list, and responsible_owner for each entry", "annual_tooling_capability_review_report documenting connector coverage assessment findings, export format adequacy determination, vendor roadmap review notes, and remediation decisions with assigned owners", "vulnerability_management_enrollment_record confirming each lineage tool is registered in the enterprise vulnerability scanning program with scan_frequency, last_scan_date, and open_critical_cves count", "openlineage_compliance_verification_record for each lineage tool in use showing API conformance test results, version_tested, and test_date" ], "machine_tests": [ "Trigger a pipeline event through each registered lineage tool connector and query the lineage graph → assert an OpenLineage-compliant event record is created within 5 minutes with valid job, run, dataset, and facet fields populated", "Query the lineage tool inventory for a data platform component onboarded in the past 60 days → assert the component has a tool_assignment entry and no open coverage gap record older than 60 days", "Submit a test CVE with severity=critical against a lineage tool in the vulnerability management system → assert a remediation task is created with a due_date no more than 30 days from submission_date" ], "human_review": [ "Review the annual tooling capability review report and confirm all coverage gaps identified have remediation plans with named owners and target closure dates approved by the data governance officer", "Assess the lineage tool inventory for completeness by cross-referencing against the enterprise data platform component registry and confirming no AI-serving or AI-training components are absent from the inventory", "Confirm OpenLineage compliance was verified for any lineage tool adopted in the past 12 months by reviewing test records or reviewing the vendor's published conformance certification" ], "blocking_effect": "advisory", "normative_status": "industry-framework", "anti_patterns": [ "Allowing individual data engineering teams to select and operate lineage tools independently without enterprise governance, resulting in a fragmented coverage landscape with no unified tool inventory and no accountability for gaps", "Adopting lineage tools without verifying OpenLineage API compliance, creating proprietary format lock-in that prevents interoperability with other governance tools and blocks regulatory export in standard formats", "Excluding lineage tools from the enterprise vulnerability management program on the assumption that metadata systems have no security exposure, leaving known CVEs unpatched and lineage records at risk of unauthorized access or tampering", "Conducting lineage tooling capability assessments only reactively when coverage gaps are reported by pipeline teams, rather than on a proactive annual cycle that anticipates data platform evolution", "Permitting coverage gap remediation plans to remain open without an escalation mechanism beyond 60 days, normalizing incomplete tooling coverage that invalidates organization-wide lineage completeness claims" ], "update_status": "current", "layer_code": "DL" }, { "id": "DL-08", "layer": "DL", "plane": "both", "name": "Data Lineage Evidence Package", "plain": "Compile a structured evidence package demonstrating that data lineage is tracked comprehensively for all AI data flows, drawing on artifacts from DL-01 through DL-07 to produce a complete attestation that lineage controls are implemented, operating, and effective.", "threat": { "tags": [ "evidence-incompleteness", "attestation-gap", "audit-unpreparedness", "control-assurance-failure" ], "desc": "Without a compiled evidence package, lineage controls exist as individual implementations that cannot be assessed holistically. Auditors and regulators who request evidence of AI data lineage governance receive ad hoc collections of artifacts rather than a structured demonstration of control effectiveness. Evidence gaps that would be surfaced by a compiled package review remain undetected until a regulatory inquiry or incident response exercise exposes them." }, "standard": [ { "id": "eu_ai_act", "section": "Art. 11 & Art. 18", "title": "Technical documentation and record-keeping for high-risk AI" }, { "id": "dama_dmbok", "section": "Ch. 12", "title": "Metadata Management — lineage program evidence" }, { "id": "cobit_2019", "section": "MEA04", "title": "Managed assurance" }, { "id": "dcam", "section": "Capability 5.2", "title": "Data quality is profiled and measured" } ], "sources": [ { "id": "eu_ai_act", "title": "EU AI Act", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2024/1689", "published_on": "2024-07-12", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2024/1689/oj/eng", "license": "open-access", "status": "current", "flagship": false, "source_id": "eu_ai_act", "relationship": "normative_requirement", "rationale": "Establishes EU AI Act Art. 11 & Art. 18 requirements informing the apeiris://data/controls/DL-08 Data Lineage Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "dama_dmbok", "title": "DAMA-DMBOK 2nd Ed", "authority": "DAMA International", "source_type": "standard", "normative_force": "best-practice", "version": "2 (2nd Edition, Revised 2024)", "published_on": "2017-07-05", "retrieved_on": "2026-07-02", "canonical_url": "https://www.dama.org/cpages/body-of-knowledge", "license": "proprietary", "status": "current", "flagship": false, "source_id": "dama_dmbok", "relationship": "informative_reference", "rationale": "Establishes DAMA-DMBOK 2nd Ed Ch. 12 requirements informing the apeiris://data/controls/DL-08 Data Lineage Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "cobit_2019", "title": "COBIT 2019", "authority": "ISACA", "source_type": "standard", "normative_force": "industry-framework", "version": "2019", "published_on": "2018-11-12", "retrieved_on": "2026-07-02", "canonical_url": "https://www.isaca.org/resources/cobit", "license": "proprietary", "status": "current", "flagship": false, "source_id": "cobit_2019", "relationship": "informative_reference", "rationale": "Establishes COBIT 2019 MEA04 requirements informing the apeiris://data/controls/DL-08 Data Lineage Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "dcam", "title": "DCAM v2.2", "authority": "EDM Council", "source_type": "standard", "normative_force": "industry-framework", "version": "2.2", "published_on": "2022-01-01", "retrieved_on": "2026-07-02", "canonical_url": "https://edmcouncil.org/frameworks/dcam/", "license": "proprietary", "status": "current", "flagship": false, "source_id": "dcam", "relationship": "informative_reference", "rationale": "Establishes DCAM v2.2 Capability 5.2 requirements informing the apeiris://data/controls/DL-08 Data Lineage Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "iso_27701", "title": "ISO/IEC 27701:2019", "authority": "ISO/IEC", "source_type": "standard", "normative_force": "certification-standard", "version": "2019 (superseded by ISO/IEC 27701:2025, published 2025-10-14; 24-36 month certification transition)", "published_on": "2019-08-06", "retrieved_on": "2026-07-02", "canonical_url": "https://www.iso.org/standard/71670.html", "license": "proprietary", "status": "current", "flagship": false, "source_id": "iso_27701", "relationship": "implementation_pattern", "rationale": "Establishes ISO/IEC 27701:2019 §7.2.8 requirements informing the apeiris://data/controls/DL-08 Data Lineage Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "gdpr", "title": "GDPR", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2016/679", "published_on": "2016-04-27", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj", "license": "open-access", "status": "current", "flagship": false, "source_id": "gdpr", "relationship": "normative_requirement", "rationale": "Establishes GDPR Art. 5(2) requirements informing the apeiris://data/controls/DL-08 Data Lineage Evidence Package control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Assemble a DL Evidence Package on a defined schedule (minimum annually, and before any AI system regulatory submission or conformity assessment). The package must include: lineage coverage reports (DL-01, DL-05), training data provenance artifacts (DL-02), inference provenance samples (DL-03), transformation documentation samples (DL-04), regulatory export test results (DL-06), and tooling governance records (DL-07). Apply a tamper-evident hash to the assembled package.", "steps": [ "Define the DL Evidence Package manifest: the specific artifact types required from each of DL-01 through DL-07, the currency requirements (lineage coverage report must be no older than 30 days), and the hash algorithm for package integrity.", "Automate extraction of evidence artifacts from the lineage platform (coverage reports, gap remediation records, export test results) on the defined schedule and assemble them into the package structure.", "Apply a SHA-256 hash to the assembled package and record the package hash in the compliance registry with the assembly date, assembler identity, and package version.", "Conduct a completeness review of the assembled package before it is finalized: verify that all required artifact types are present, currency requirements are met, and the package hash is recorded.", "Store the finalized DL Evidence Package in the compliance artifact repository with access logging and retention aligned to the lineage archive schedule (minimum 10 years for EU AI Act compliance)." ], "data_governance_officer": { "summary": "The DL Evidence Package is the definitive demonstration that the organization's lineage program is complete, operational, and audit-ready. It must be assembled and reviewed on a defined schedule, not reactively in response to a regulatory demand.", "actions": [ "Define the DL Evidence Package assembly schedule and assign ownership to a named data governance role.", "Review each assembled package for completeness before finalization and sign off as the authorizing data governance officer.", "Ensure the DL Evidence Package is referenced in the organization's AI system technical documentation submitted for EU AI Act conformity assessment." ], "failure_signals": [ "DL Evidence Package not assembled on schedule or more than 45 days overdue.", "Package assembled without data governance officer completeness review and sign-off.", "Package artifacts failing currency requirements for more than two required artifact types." ] }, "data_engineer": { "summary": "Evidence package assembly should be automated to the maximum extent possible to ensure currency and completeness of artifacts without manual extraction errors.", "actions": [ "Implement automated evidence extraction scripts that pull lineage coverage reports, gap metrics, and export test results from source systems on the defined assembly schedule.", "Build the package assembly pipeline to validate each artifact against the manifest requirements before finalizing the package hash.", "Test the evidence assembly automation annually to verify that all artifact sources are accessible and returning current data." ], "failure_signals": [ "Evidence extraction scripts failing silently and producing stale or empty artifacts.", "Package assembly pipeline finalizing packages with artifact validation failures.", "Annual automation test not conducted or producing unresolved failures after 30 days." ] }, "legal_counsel": { "summary": "The DL Evidence Package must be structured to satisfy the documentation requirements of applicable AI regulations and to serve as discoverable evidence in regulatory proceedings. Legal must review the package schema and retention arrangements.", "actions": [ "Review the DL Evidence Package manifest and confirm that all artifacts required by EU AI Act Articles 11 and 18 are included.", "Confirm that the package retention period and storage arrangement satisfy the 10-year documentation requirement for high-risk AI systems.", "Define the legal hold procedure for DL Evidence Packages when regulatory investigation or litigation is anticipated." ], "failure_signals": [ "DL Evidence Package schema missing artifact types required by EU AI Act Articles 11 or 18.", "Evidence packages stored with retention periods shorter than legally required.", "No legal hold procedure documented for DL Evidence Packages." ] }, "grc_auditor": { "summary": "The DL Evidence Package is the primary audit artifact for assessing the lineage control layer. It consolidates evidence from seven individual controls into a single reviewable package that supports holistic assurance conclusions.", "actions": [ "Review the most recently assembled DL Evidence Package and assess completeness against the defined manifest.", "Sample individual artifacts from the package (3 training provenance records, 5 inference provenance lookups, 1 regulatory export test result) and verify accuracy against source systems.", "Document assurance conclusions from the DL layer review, noting any gaps or exceptions identified during artifact sampling." ], "metrics": [ "DL Evidence Package assembly on schedule: target 100% of required periods.", "Package completeness rate (required artifacts present and current): target 100%.", "Artifact sampling accuracy rate (package artifacts matching source system records): target 100%." ], "failure_signals": [ "DL Evidence Package not assembled for the current audit period.", "Package missing required artifact types or artifacts failing currency requirements.", "Sampled artifacts not matching source system records." ] }, "it_operations": { "summary": "The DL evidence package draws on lineage systems operations runs. Reliable exports, retention alignment, and integrity of the packaged artifacts are operational deliverables.", "actions": [ "Automate lineage-coverage and provenance exports into the evidence assembly pipeline.", "Align lineage-store retention with evidence-package retention requirements.", "Hash and verify packaged lineage artifacts at assembly time." ], "failure_signals": [ "Evidence packages assembled from stale lineage exports.", "Lineage records expiring before the evidence period closes.", "Package verification failures at audit time." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Compiled, tamper-evident evidence packages for AI data lineage are a new practice driven by EU AI Act documentation requirements. Most organizations are at the initial stage of defining what such a package must contain." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "Data Governance Office", "GRC / Internal Audit", "Data Engineering", "Legal / Compliance" ], "frameworks": [ { "framework": "eu_ai_act", "requirement_id": "Art. 11 & Art. 18", "fit": "direct", "rationale": "EU AI Act Articles 11 and 18 together require providers of high-risk AI systems to compile and maintain technical documentation demonstrating compliance with data governance requirements. The DL Evidence Package is the structured compilation mechanism that satisfies this dual documentation and record-keeping obligation.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 12", "fit": "direct", "rationale": "DAMA-DMBOK2 Chapter 12 defines the metadata program whose outputs — lineage records, coverage measures and tooling documentation — constitute the evidence the DL package assembles.", "normative_force": "industry-framework", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "cobit_2019", "requirement_id": "MEA04", "fit": "partial", "rationale": "COBIT 2019 objective MEA04 (Managed Assurance) covers risk-based planning of assurance initiatives and the evidence those initiatives consume. The DL Evidence Package is the assurance-ready artifact lineage assurance initiatives draw on. The previously cited MEA02 practice number does not exist in COBIT 2019, whose MEA02 practices end at MEA02.04.", "normative_force": "best-practice", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "dcam", "requirement_id": "Capability 5.2", "fit": "direct", "rationale": "DCAM v2.2 capability 5.2 (data quality is profiled and measured) produces the measurement outputs — coverage scores, completeness metrics — that the DL evidence package packages as demonstrable evidence.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_27701", "requirement_id": "§7.2.8", "fit": "adjacent", "rationale": "ISO/IEC 27701:2019 §7.2.8 requires maintainable, demonstrable records of PII processing. The lineage evidence package assembles those records into an audit-ready artifact. §5.4 addresses PIMS planning, not evidence records.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "gdpr", "requirement_id": "Art. 5(2)", "fit": "partial", "rationale": "GDPR Article 5(2) requires controllers to be responsible for and be able to demonstrate compliance with data protection principles. The DL Evidence Package provides the documentary basis for demonstrating that data lineage governance practices comply with the accountability principle for AI data flows involving personal data.", "normative_force": "binding-law", "source_version": "2016/679", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" } ], "canonical_id": "apeiris://data/controls/DL-08", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "A complete, tamper-evident DL Evidence Package must be assembled and finalized on the defined schedule (minimum annually and before any regulatory submission), containing current artifacts from all seven DL controls with no artifact exceeding its currency requirement. The package SHA-256 hash must be recorded in the compliance registry, and the package must receive a completeness sign-off from the data governance officer before finalization.", "evidence_required": [ "assembled_dl_evidence_package manifest listing artifact_type, source_control_id, artifact_currency, and file_reference for each of DL-01 through DL-07, with package_hash (SHA-256), assembly_date, and assembler_identity", "package_completeness_review_sign_off record showing data_governance_officer name, review_date, completeness_verdict, and any exceptions noted before the hash was finalized", "compliance_registry_entry recording package_hash, assembly_date, package_version, and storage_location for audit retrieval", "artifact_currency_check_report confirming each required artifact type meets its defined currency requirement (e.g., lineage coverage report no older than 30 days, regulatory export test no older than 12 months)" ], "machine_tests": [ "Run the automated evidence extraction pipeline → assert all required artifact types from DL-01 through DL-07 are present in the output manifest with no missing_artifact or currency_violation entries", "Compute SHA-256 hash of the assembled evidence package and compare against the hash recorded in the compliance registry → assert hashes match exactly, confirming no post-assembly modification", "Query the compliance registry for the most recent DL Evidence Package entry → assert the assembly_date falls within the defined assembly interval (no more than 13 months from the current date)" ], "human_review": [ "Sample 3 artifacts from the assembled package (one training provenance record, one inference provenance lookup, one regulatory export test result) and cross-verify each against the corresponding source system record to confirm accuracy", "Review the completeness sign-off record and confirm the data governance officer completed the review before the package hash was finalized, not after", "Confirm the finalized package is stored in the compliance artifact repository with a retention configuration of at least 10 years and access logging enabled at the individual record level" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Assembling the DL Evidence Package reactively in response to a regulatory demand rather than on a proactive schedule, resulting in stale or missing artifacts under time pressure that cannot be remediated before the production deadline", "Finalizing the package SHA-256 hash before the data governance officer completeness review is complete, making the signed hash untrustworthy as an integrity anchor and invalidating its use as a tamper-evidence mechanism", "Including placeholder or stub artifacts that reference lineage controls planned but not yet implemented, misrepresenting the organization's actual lineage capability to auditors who rely on the package as a holistic evidence source", "Storing the evidence package in mutable storage without WORM or immutability protection, allowing post-assembly modification that the package hash alone cannot detect if the hash record is also mutable", "Treating the DL Evidence Package as a one-time compliance artifact assembled at initial certification rather than a periodically renewed governance record, leaving a multi-year evidence gap between the initial package and the next model release cycle" ], "update_status": "current", "layer_code": "DL" }, { "id": "DA-01", "layer": "DA", "plane": "control", "name": "AI Data Access Authorization Framework", "plain": "Every AI system must access training and inference data only through an authorization framework that applies role-based and attribute-based access controls, enforces least-privilege bindings, and documents the access policy for each data asset as machine-readable policy-as-code.", "threat": { "tags": [ "unauthorized-data-access", "privilege-escalation", "access-scope-creep", "policy-bypass" ], "desc": "Without a structured authorization framework, AI systems and the engineers operating them accumulate broad data access that exceeds operational necessity. Ad-hoc access grants are rarely reviewed or revoked, enabling scope creep over model iteration cycles. Adversaries who compromise a service account inherit unbounded access to training corpora, undermining both data confidentiality and model integrity." }, "standard": [ { "id": "gdpr", "section": "Art. 5(1)(f)", "title": "Integrity and confidentiality principle" }, { "id": "cobit_2019", "section": "DSS06.03", "title": "Manage roles, responsibilities, access privileges and levels of authority" }, { "id": "nist_pf", "section": "PR.AC-P4", "title": "Access permissions managed with least privilege and separation of duties" }, { "id": "dama_dmbok", "section": "Ch. 7", "title": "Data Security management" } ], "sources": [ { "id": "eu_data_gov_act_2022_868", "title": "EU Data Governance Act — Regulation (EU) 2022/868", "authority": "European Parliament and Council of the European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2022/868", "published_on": "2022-06-03", "retrieved_on": "2026-06-29", "canonical_url": "https://eur-lex.europa.eu/legal-content/EN/TXT/?uri=CELEX%3A32022R0868", "license": "open-access", "status": "current", "flagship": false, "source_id": "eu_data_gov_act_2022_868", "relationship": "normative_requirement", "rationale": "Establishes EU Data Governance Act — Regulation (EU) 2022/868 requirements informing the apeiris://data/controls/DA-01 AI Data Access Authorization Framework control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Define a data classification taxonomy; bind each AI data asset to a classification tag; implement RBAC policies at the storage layer reinforced by attribute-based controls; require policy-as-code representation of every access grant stored in version control; audit all access decisions at runtime against an immutable audit sink.", "steps": [ "Classify all AI data assets (training sets, feature stores, inference logs) using a four-tier sensitivity taxonomy (public, internal, confidential, restricted) and enter every asset into the data catalog with its classification tag.", "Implement RBAC groups mapped to AI use-case roles (model-trainer, inference-service, data-scientist, auditor) and bind each group to the minimum set of data assets required for its function.", "Deploy attribute-based access control policies that enforce context conditions such as data residency, accessor jurisdiction, and purpose code before granting access; express all policies as policy-as-code artifacts (e.g., OPA Rego, Cedar) stored in version control.", "Route all access decision events to an immutable audit sink and configure alerts on out-of-policy access attempts; target a detection-to-alert latency of under 60 seconds.", "Reconcile runtime IAM state against policy-as-code definitions daily and surface any drift to the data governance officer within one business day." ], "data_governance_officer": { "summary": "The authorization framework is the primary governance artifact proving that data access is controlled, justified, and documented for every AI use case.", "actions": [ "Approve the data classification taxonomy and ensure all AI assets are classified before access grants are issued.", "Review policy-as-code artifacts quarterly to confirm alignment with business purpose and regulatory obligations.", "Track authorization framework coverage as a KRI; escalate when coverage drops below 100% of active AI data assets." ], "failure_signals": [ "AI data assets with no associated access policy found during inventory reconciliation.", "Access grants not linked to a documented business purpose or use-case owner.", "Policy-as-code repository diverges from runtime enforcement state without a logged reconciliation action." ] }, "data_engineer": { "summary": "Implement storage-layer access controls enforced at the platform level so no pathway exists to reach AI data without passing through the authorization framework.", "actions": [ "Configure IAM bindings on all AI data stores (S3 bucket policies, BigQuery IAM, Unity Catalog grants) to align with approved RBAC groups.", "Integrate ABAC policy engine into data pipeline ingestion paths so purpose codes are validated before data is loaded into training jobs.", "Build daily reconciliation scripts that compare runtime access state against policy-as-code definitions and surface drift to the data governance officer." ], "failure_signals": [ "Direct database credentials or storage access keys found outside the approved secrets management vault.", "Pipeline jobs accessing AI data stores without passing through the ABAC evaluation layer.", "Drift detected between policy-as-code definitions and applied IAM bindings persisting beyond one business day." ] }, "grc_auditor": { "summary": "The authorization framework is the foundational evidence layer for all data access controls. Audit coverage, completeness, and policy-as-code fidelity.", "actions": [ "Request the data asset inventory and cross-reference each asset against the policy-as-code repository to confirm every asset has an associated access policy.", "Sample 15% of active access grants and verify each is linked to a named business purpose, a data owner approval, and a valid expiry date.", "Run a reconciliation report confirming no access exists in runtime state that is absent from the policy-as-code repository." ], "metrics": [ "Authorization framework coverage: 100% of AI data assets must have a documented access policy.", "Policy-as-code drift rate: zero unexplained divergences per quarterly review.", "Out-of-policy access alert resolution SLA: 100% resolved within 24 hours." ], "failure_signals": [ "Coverage below 100% for more than one reporting cycle.", "Unresolved policy drift items older than 24 hours.", "Access grants with no linked business purpose or missing expiry date." ] }, "it_operations": { "summary": "Operate the access control infrastructure reliably and ensure all access decision logs are delivered to the immutable audit sink without gaps.", "actions": [ "Maintain availability of the policy engine and IAM services at 99.9% uptime; access must fail-closed when the policy engine is unavailable.", "Monitor the access decision log pipeline for lag or dropped events and alert when delivery SLA is breached.", "Coordinate emergency access procedures (see DA-07) when the authorization framework must be bypassed for incident response." ], "failure_signals": [ "Policy engine unavailability exceeding 0.1% in any 30-day window.", "Access decision log gaps exceeding 5 minutes detected in the audit sink.", "Fail-open behavior observed during a policy engine degraded state." ] }, "legal_counsel": { "summary": "Access authorization is where 'appropriate security' and purpose limitation become operational. Counsel defines which data classes carry legal access conditions and reviews the policy basis for AI systems' standing access.", "actions": [ "Define legal access conditions per data class — consent-bound, jurisdiction-bound, privilege-bound.", "Review standing access grants to AI pipelines against purpose limitation.", "Approve the policy basis for service-account access to special-category data." ], "failure_signals": [ "AI pipelines holding standing access to special-category data with no documented basis.", "Access policies that ignore consent or jurisdiction conditions attached to the data.", "Legal access conditions existing only in contracts, never in enforced policy." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most enterprises have ad-hoc IAM bindings for AI data assets without a unified authorization framework or policy-as-code representation." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "eu-high-risk-ai" ], "implementers": [ "Data Governance Team", "IAM Team", "Platform Engineering", "Data Engineering" ], "frameworks": [ { "framework": "gdpr", "requirement_id": "Art. 5(1)(f)", "fit": "direct", "rationale": "GDPR Article 5(1)(f) requires personal data to be processed in a manner that ensures appropriate security, including protection against unauthorised access. An RBAC/ABAC authorization framework is the primary technical control for satisfying this integrity-and-confidentiality principle for AI systems that process personal data.", "normative_force": "binding-law", "source_version": "2016/679", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "cobit_2019", "requirement_id": "DSS06.03", "fit": "direct", "rationale": "COBIT 2019 DSS06.03 prescribes management of roles, responsibilities, and access privilege levels for data assets. The RBAC/ABAC framework directly implements these requirements for AI data assets by binding roles to datasets with documented authorization and enforcing access decisions at runtime.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_pf", "requirement_id": "PR.AC-P4", "fit": "direct", "rationale": "NIST Privacy Framework PR.AC-P4 requires access permissions and authorizations to be managed, incorporating the principles of least privilege and separation of duties. The AI data access authorization framework is the direct implementation of PR.AC-P4 for data consumed and produced by AI systems.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 7", "fit": "direct", "rationale": "DAMA DMBOK2 Chapter 7 (Data Security) establishes that data security management must include access control frameworks governing who can access what data under which conditions. DA-01 operationalises this chapter's requirements specifically for AI-bearing data assets with a classification-driven, policy-as-code approach.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "eu_data_gov_act", "requirement_id": "Art. 5-6 (Conditions for re-use)", "fit": "direct", "rationale": "EU Data Governance Act Articles 5 and 6 set the conditions under which protected public-sector data may be re-used, including technical and access conditions imposed by public bodies. An access authorization framework is how an AI operator demonstrates it can honor such conditions. Article 12 concerns providers of data intermediation services, not re-use conditions.", "normative_force": "binding-law", "source_version": "2022/868", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "ms_ifc_agents", "requirement_id": "Confidentiality label read-access list ({Alice, Bob, Charlie}); Check before acting enforces reader authorization", "fit": "partial", "rationale": "IFC confidentiality labels can carry an explicit read-access list, and the check-before-acting step enforces that only authorized readers receive the data — an access-authorization control. The article frames this as data-flow confidentiality rather than a full access-authorization framework, so the fit is partial.", "normative_force": "best-practice", "source_version": "2026", "reviewed_on": "2026-07-03", "basis": "anchored", "relation": "informs" }, { "framework": "owasp_ai_exchange", "requirement_id": "discrete", "fit": "supporting", "rationale": "DA-01 enforces policy-as-code access to every AI data asset with runtime IAM matching the policy, keeping AI data on a documented need-to-know basis.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "mitre", "requirement_id": "AML.M0001", "fit": "supporting", "direction": "control-supports-requirement", "rationale": "Control \"Every active AI data asset must have a documented access policy expressed as…\" enacts ATLAS mitigation AML.M0001 Limit Model Artifact Release; OpenCRE crosswalks this control’s OWASP AI Exchange concept (discrete) to this mitigation.", "source_version": "v2026.06", "reviewed_on": "2026-07-08", "source_status": "authoritative", "mapping_confidence": "medium", "legal_status": "not-applicable", "control_readiness": "approved", "implementation_maturity": "developing", "assurance_result": "not-assessed", "evidence_status": "no-evidence", "normative_force": "industry-framework", "basis": "asserted", "relation": "equivalent_to", "correction": "ap38-opencre-enhancement 2026-07-08" }, { "framework": "mitre", "requirement_id": "AML.M0000", "fit": "supporting", "direction": "control-supports-requirement", "rationale": "Control \"Every active AI data asset must have a documented access policy expressed as…\" enacts ATLAS mitigation AML.M0000 Limit Public Release of Information; OpenCRE crosswalks this control’s OWASP AI Exchange concept (discrete) to this mitigation.", "source_version": "v2026.06", "reviewed_on": "2026-07-08", "source_status": "authoritative", "mapping_confidence": "medium", "legal_status": "not-applicable", "control_readiness": "approved", "implementation_maturity": "developing", "assurance_result": "not-assessed", "evidence_status": "no-evidence", "normative_force": "industry-framework", "basis": "asserted", "relation": "equivalent_to", "correction": "ap38-opencre-enhancement 2026-07-08" } ], "canonical_id": "apeiris://data/controls/DA-01", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every active AI data asset must have a documented access policy expressed as policy-as-code in version control, with runtime IAM state matching the policy definition and zero unexplained divergences per quarterly review. All out-of-policy access attempts must generate alerts delivered to the immutable audit sink within 60 seconds and be resolved within 24 hours.", "evidence_required": [ "policy_as_code_repository_snapshot showing access policy definitions for all classified AI data assets with version history, commit timestamps, and approving reviewer for each policy file", "iam_reconciliation_report confirming runtime access state matches policy-as-code definitions with drift_items=[] or all drift items linked to approved change records with resolution timestamps", "access_decision_audit_log extract showing access decision events against AI data assets with decision_result, policy_evaluated, accessor_principal, and event_timestamp delivered to immutable sink within 60 seconds", "data_asset_classification_inventory listing all AI data assets with sensitivity_tier, assigned_rbac_group, abac_policy_id, and last_reviewed_date for each asset" ], "machine_tests": [ "Attempt to access an AI data store with a principal not in any approved RBAC group → assert access is denied with error_code=unauthorized and an audit event is written to the immutable sink within 60 seconds containing principal_id and attempted_resource", "Modify an IAM binding in the test environment to grant access not present in the policy-as-code repository → assert the daily reconciliation job detects the drift and surfaces a drift_item to the data governance officer within one business day", "Submit a data access request with a purpose_code that does not match any ABAC policy entry for the requested asset → assert the ABAC engine denies the request and logs the denial with reason=purpose_code_mismatch and the evaluated purpose_code value" ], "human_review": [ "Sample 15% of active access grants and verify each has a named business purpose, a documented data owner approval, and a valid expiry date that matches the corresponding policy-as-code definition", "Review the policy-as-code repository change history for the past quarter and confirm every policy modification has an approval record from the data governance officer before the change was merged", "Assess the ABAC policy configuration for at least three AI use cases and confirm that context conditions for data residency, jurisdiction, and purpose code are correctly expressed in the policy engine and enforced at query time" ], "blocking_effect": "blocks-runtime-action", "normative_status": "binding-law", "anti_patterns": [ "Managing AI data access through ad-hoc IAM bindings applied directly in the cloud console without policy-as-code representation, making access grants invisible to governance review, drift detection, and audit", "Assigning a single shared service account to multiple AI systems and granting it broad read access to all training data stores rather than per-system, per-purpose RBAC groups with least-privilege bindings", "Implementing ABAC policies only as written governance rules rather than as executable policy-as-code in a policy engine (e.g., OPA Rego, Cedar), allowing runtime bypass without generating a policy violation event", "Configuring the authorization policy engine to fail-open when unavailable or degraded, permitting unrestricted AI data access during infrastructure incidents without any access decision audit record", "Issuing broad data access grants at AI project onboarding and never reviewing, scoping down, or revoking them as the model evolves across training iterations where data requirements narrow" ], "update_status": "current", "layer_code": "DA" }, { "id": "DA-02", "layer": "DA", "plane": "control", "name": "Data Minimization Implementation for AI", "plain": "AI systems must be architected to access only the minimum data necessary for each specific use case; access to columns, records, or time ranges beyond what the model or pipeline requires must be technically prevented at the data platform layer, not merely discouraged by policy.", "threat": { "tags": [ "data-over-collection", "purpose-creep", "inference-data-leakage", "training-data-exfiltration" ], "desc": "AI pipelines frequently request broad read access to entire datasets when only a subset of fields or records is needed for the task. Over-collection becomes entrenched as engineers design models against a broad schema rather than the minimal one. A breach of an over-provisioned AI data store exposes far more personal or sensitive data than the model's actual information need ever justified, and regulators treat over-collection as an independent violation regardless of whether a breach occurs." }, "standard": [ { "id": "gdpr", "section": "Art. 5(1)(c)", "title": "Data minimisation principle" }, { "id": "nist_pf", "section": "CT.DP-P1", "title": "Data processed to limit observability and linkability" }, { "id": "iso_27701", "section": "§7.4.4", "title": "Limiting processing" } ], "sources": [ { "id": "gdpr", "title": "GDPR", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2016/679", "published_on": "2016-04-27", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj", "license": "open-access", "status": "current", "flagship": false, "source_id": "gdpr", "relationship": "normative_requirement", "rationale": "Establishes GDPR Art. 5(1)(c) requirements informing the apeiris://data/controls/DA-02 Data Minimization Implementation for AI control.", "reviewed_on": "2026-07-02" }, { "id": "nist_pf", "title": "NIST Privacy Framework 1.0", "authority": "National Institute of Standards and Technology", "source_type": "standard", "normative_force": "voluntary-standard", "version": "1.0", "published_on": "2020-01-16", "retrieved_on": "2026-07-02", "canonical_url": "https://www.nist.gov/privacy-framework", "license": "open-access", "status": "current", "flagship": false, "source_id": "nist_pf", "relationship": "implementation_pattern", "rationale": "Establishes NIST Privacy Framework 1.0 CT.DP-P1 requirements informing the apeiris://data/controls/DA-02 Data Minimization Implementation for AI control.", "reviewed_on": "2026-07-02" }, { "id": "iso_27701", "title": "ISO/IEC 27701:2019", "authority": "ISO/IEC", "source_type": "standard", "normative_force": "certification-standard", "version": "2019 (superseded by ISO/IEC 27701:2025, published 2025-10-14; 24-36 month certification transition)", "published_on": "2019-08-06", "retrieved_on": "2026-07-02", "canonical_url": "https://www.iso.org/standard/71670.html", "license": "proprietary", "status": "current", "flagship": false, "source_id": "iso_27701", "relationship": "implementation_pattern", "rationale": "Establishes ISO/IEC 27701:2019 §7.4.4 requirements informing the apeiris://data/controls/DA-02 Data Minimization Implementation for AI control.", "reviewed_on": "2026-07-02" }, { "id": "dama_dmbok", "title": "DAMA-DMBOK 2nd Ed", "authority": "DAMA International", "source_type": "standard", "normative_force": "best-practice", "version": "2 (2nd Edition, Revised 2024)", "published_on": "2017-07-05", "retrieved_on": "2026-07-02", "canonical_url": "https://www.dama.org/cpages/body-of-knowledge", "license": "proprietary", "status": "current", "flagship": false, "source_id": "dama_dmbok", "relationship": "informative_reference", "rationale": "Establishes DAMA-DMBOK 2nd Ed Ch. 7 requirements informing the apeiris://data/controls/DA-02 Data Minimization Implementation for AI control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "For each AI use case, produce a data necessity justification document enumerating required fields, record filters, and time windows; enforce minimization technically via column-level access controls, row-level security predicates, and time-bounded access tokens; block access to columns or records not listed in the justification document; revoke unused field access when telemetry shows it was never read.", "steps": [ "Require a data necessity justification document for each AI use case, signed by the use-case owner and data governance officer, listing every data field, record population filter, and historical time window required.", "Implement column-level security at the data platform layer (e.g., BigQuery column-level access, Unity Catalog column masks, Snowflake column masking policies) to enforce field-level minimization against the approved justification.", "Apply row-level security predicates and time-bounded access tokens so AI pipelines cannot read records outside the approved population or historical window.", "Instrument training pipeline data readers with field-usage telemetry that logs every field accessed and compares against the necessity justification; alert on any field accessed that is not listed.", "Review necessity justifications at each model version increment; revoke access to fields that usage telemetry shows were never read in the prior training run." ], "data_governance_officer": { "summary": "Data minimization is a regulatory obligation and a data quality discipline. Every AI use case must have an approved necessity justification document before data access is provisioned.", "actions": [ "Establish the necessity justification document template and approval workflow; require sign-off before data access is provisioned for any new AI use case.", "Maintain a register of all active necessity justifications and trigger review whenever a model version increments or business purpose changes.", "Track field-usage telemetry anomalies as a governance KRI; unused field access must be revoked within one sprint cycle of detection." ], "failure_signals": [ "AI use cases with data access provisioned but no associated necessity justification document.", "Necessity justification documents older than 12 months with no review stamp.", "Field-usage telemetry showing access to fields not listed in the approved justification." ] }, "data_engineer": { "summary": "Minimization must be enforced at the platform layer, not just documented in policy. Build column masks, row filters, and time-window constraints directly into the data platform configuration.", "actions": [ "Implement column-level security policies in the data platform for every table exposed to AI pipelines, blocking access to columns absent from the necessity justification.", "Configure row-level security predicates that filter records to the approved population segment (e.g., only records from the past 24 months, only records from consenting users).", "Instrument training job data readers to emit field-access telemetry to the governance log sink for automated comparison against necessity justifications." ], "failure_signals": [ "Column security policies absent on any table flagged as containing sensitive data and used by an AI pipeline.", "Training job telemetry logs showing field reads not present in the necessity justification.", "Row-level predicates not applied on tables with population-restricted data." ] }, "legal_counsel": { "summary": "Data minimization is a binding legal obligation under GDPR and equivalent laws. Ensure necessity justifications are legally grounded and that purpose limitation is respected across the AI use case lifecycle.", "actions": [ "Review necessity justification documents to confirm the stated AI purpose is compatible with the original data collection purpose under GDPR Art. 5(1)(b) and purpose limitation requirements.", "Advise on whether statistical or pseudonymized data can substitute for identifiable records in the justification to reduce legal exposure.", "Flag use cases where the minimization scope has expanded beyond what was disclosed in the original privacy notice or data subject consent." ], "failure_signals": [ "AI use cases processing data for purposes incompatible with the original collection basis.", "Necessity justification documents approved without legal review for use cases involving special-category data.", "Absence of purpose limitation clauses in AI vendor data processing agreements." ] }, "grc_auditor": { "summary": "Audit the completeness of necessity justifications, the fidelity of technical enforcement, and the timeliness of access revocation when fields are found unused.", "actions": [ "Sample 20% of active AI use cases and verify each has a current, approved necessity justification document with named data owner and use-case owner signatures.", "Compare field-usage telemetry for sampled use cases against their necessity justifications and report any field accessed but not listed in the justification.", "Verify that revocation of unused field access is completed within the required sprint-cycle SLA." ], "metrics": [ "Necessity justification coverage: 100% of active AI use cases must have a current approved document.", "Field access compliance rate: zero fields accessed outside the approved justification per quarterly audit.", "Unused field revocation SLA: 100% revoked within one sprint cycle of detection." ], "failure_signals": [ "Coverage below 100% for active use cases in any audit cycle.", "Fields accessed outside the justification not remediated within the sprint-cycle SLA.", "Justification documents with no review stamp older than 12 months." ] }, "it_operations": { "summary": "Maintain the platform-layer enforcement mechanisms and ensure field-usage telemetry is delivered reliably to the governance sink.", "actions": [ "Monitor column-level security policy application and alert when schema changes introduce new columns not yet covered by a column security policy.", "Ensure the field-access telemetry pipeline delivers events to the governance log within 15 minutes of occurrence with no gaps.", "Coordinate with data engineers when schema evolution requires necessity justification updates before new columns are made accessible to AI pipelines." ], "failure_signals": [ "Schema changes deploying new columns without associated column security policies.", "Telemetry pipeline lag exceeding 15 minutes for field-access events without an alert.", "Access to new columns granted before the necessity justification is updated." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Most organisations have data minimization as a written policy but lack technical enforcement at the column and row level specifically for AI pipelines." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "eu-high-risk-ai", "high-risk-sector", "multi-tenant" ], "implementers": [ "Data Engineering", "Data Governance Team", "Legal / Privacy" ], "frameworks": [ { "framework": "gdpr", "requirement_id": "Art. 5(1)(c)", "fit": "direct", "rationale": "GDPR Article 5(1)(c) mandates that personal data be adequate, relevant, and limited to what is necessary for the specified purpose. DA-02 directly operationalises this requirement for AI pipelines by requiring necessity justification documents and enforcing field and record minimization technically at the data platform layer.", "normative_force": "binding-law", "source_version": "2016/679", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_pf", "requirement_id": "CT.DP-P1", "fit": "direct", "rationale": "NIST Privacy Framework CT.DP-P1 requires data to be processed so as to limit observability and linkability. Data minimization for AI — restricting fields, rows and granularity to what the AI purpose requires — is a primary technique for reducing observability and linkability of individuals in AI pipelines.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_27701", "requirement_id": "§7.4.4", "fit": "direct", "rationale": "ISO/IEC 27701 §7.4.4 requires that personal data processing be limited to what is necessary for the processing purpose. DA-02 implements this requirement via column-level access controls and row-level security predicates that technically enforce the documented processing scope, making limiting of processing verifiable.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 7", "fit": "partial", "rationale": "DAMA DMBOK2 Chapter 7 includes data security principles that encompass access restriction to necessary data elements. DA-02 implements the data minimization dimension of data security management for AI pipelines, complementing the broader access control framework established in DA-01.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "owasp_ai_exchange", "requirement_id": "dataminimize", "fit": "direct", "rationale": "DA-02 requires an approved data-necessity justification and enforces column/row-level restrictions so no field outside the justification is accessed, directly minimizing sensitive data used.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://data/controls/DA-02", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every active AI use case must have a current, approved data necessity justification document, with column-level security policies and row-level predicates technically enforced at the data platform layer that match the justification exactly. Field-usage telemetry must show zero fields accessed outside the approved justification across any audit period, and unused fields must be revoked within one sprint cycle of detection.", "evidence_required": [ "data_necessity_justification_document for each AI use case listing approved_fields[], record_population_filter, time_window, use_case_owner signature, and data_governance_officer approval_date (no older than 12 months or last model version increment)", "column_security_policy_configuration export from the data platform showing active column masks or access restrictions for each table exposed to AI pipelines with policy_last_updated date", "field_usage_telemetry_report for the audit period showing every field_name accessed per use_case_id, comparison_result against the necessity justification, and any out_of_justification_access events with timestamps", "unused_field_revocation_log showing field_name, use_case_id, detection_date, revocation_action, and completion_date for fields flagged as unread in prior training runs" ], "machine_tests": [ "Submit a training pipeline read query requesting a column not listed in the necessity justification for that use case → assert the column security policy blocks the query and generates a telemetry event with reason=field_not_in_justification and the blocked_column_name", "Attempt a row-level query against a table with a population predicate using a filter expression that bypasses the predicate → assert the row-level security policy prevents predicate bypass and the result set contains only records from the approved population", "Run the field-usage telemetry comparison job for a use case with a known necessity justification → assert no fields appear in the telemetry output that are absent from the justification's approved_fields list" ], "human_review": [ "Sample 20% of active AI use cases and verify each necessity justification document is current (reviewed within 12 months or at the last model version increment), has both required signatures, and was approved before data access was provisioned", "Confirm column-level security policies are applied to all tables containing sensitive data used by AI pipelines by cross-referencing the data platform policy export against the data asset classification inventory for completeness", "Review the unused field revocation log and confirm all flagged fields were revoked within one sprint cycle of detection; examine any exceptions for documented justification and approver sign-off" ], "blocking_effect": "blocks-runtime-action", "normative_status": "binding-law", "anti_patterns": [ "Relying on written data minimization policies without implementing column-level security controls at the data platform layer, leaving the minimization boundary unenforced and unverifiable by auditors or regulators", "Granting AI pipelines read access to entire tables or schemas on the grounds of operational convenience rather than provisioning access only to the specific columns and row populations required by the necessity justification", "Approving necessity justification documents for use cases involving special-category personal data without legal review of purpose compatibility, creating purpose limitation violations invisible until regulatory investigation", "Instrumenting field-usage telemetry without building an automated comparison process against the necessity justification, producing monitoring data that accumulates without generating revocation actions or governance alerts", "Retaining field access across model version increments for continuity even after telemetry confirms the fields were never read in the prior training run, allowing the justified minimization boundary to decay silently over time" ], "update_status": "current", "layer_code": "DA" }, { "id": "DA-03", "layer": "DA", "plane": "control", "name": "Consent-Based Data Access Controls", "plain": "AI systems that process personal data must link every data access to a valid, current consent record; access must be automatically revoked or suppressed when consent is withdrawn, and AI models trained on data for which consent has since been withdrawn must be flagged in a model impact register for output suppression review.", "threat": { "tags": [ "consent-bypass", "withdrawn-consent-persistence", "unlawful-processing", "stale-training-data" ], "desc": "AI training pipelines frequently decouple data ingestion from consent lifecycle management. Data ingested under a valid consent basis may remain in training corpora long after the individual has withdrawn consent or the consent has expired. Models trained on this data continue to encode the withdrawn data subject's information indefinitely, creating ongoing legal liability. A breach of the consent link between data and the AI system constitutes unlawful processing under GDPR regardless of whether any harm to the individual is demonstrable." }, "standard": [ { "id": "gdpr", "section": "Art. 5(1)(b)", "title": "Purpose limitation and lawful basis" }, { "id": "iso_27701", "section": "§7.2.3", "title": "Lawful basis for processing personally identifiable information" }, { "id": "nist_pf", "section": "CT.PO-P1", "title": "Policies for authorizing data processing, including individual consent" }, { "id": "eu_ai_act", "section": "Art. 10(5)", "title": "Special category data in training sets" } ], "sources": [ { "id": "anthropic_privacy_policy_2024", "title": "Anthropic Data Handling & Privacy Policy", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-09-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/privacy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_privacy_policy_2024", "relationship": "informative_reference", "rationale": "Establishes Anthropic Data Handling & Privacy Policy requirements informing the apeiris://data/controls/DA-03 Consent-Based Data Access Controls control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Maintain a consent ledger linked to each personal data record; gate AI data access on consent validity checks at query time via a consent gateway; subscribe to consent-withdrawal events and propagate suppression signals to all downstream AI data stores within a 24-hour SLA; maintain a model impact register for models trained on data with subsequent consent withdrawal.", "steps": [ "Build or integrate a consent ledger that stores consent status per data subject, per processing purpose, and links consent records to individual data asset identifiers used by AI pipelines.", "Implement a consent gateway that wraps all AI data access queries, checks the consent ledger before returning records, and filters out records for data subjects who have withdrawn consent for the relevant purpose code.", "Subscribe all AI data pipeline consent gateways to a consent-withdrawal event stream; propagate suppression signals to training data stores and feature stores within a 24-hour SLA.", "Maintain a model impact register recording which trained models were built using data from subjects who have since withdrawn consent; trigger a model retraining or output suppression review within 48 hours of a withdrawal event affecting a deployed model.", "Conduct quarterly consent ledger integrity checks comparing consent records to data subject exercise request logs to confirm no gaps in consent-withdrawal propagation." ], "data_governance_officer": { "summary": "Consent lifecycle management is a legal and ethical obligation. The consent ledger and withdrawal propagation system are core governance infrastructure for all AI use cases involving personal data.", "actions": [ "Approve the consent ledger schema and ensure it captures processing purpose codes that map to each AI use case before access is provisioned.", "Establish the consent-withdrawal propagation SLA (target: 24 hours) and report on SLA compliance monthly.", "Own the model impact register and escalate to legal counsel when a deployed model is identified as containing withdrawn-consent data." ], "failure_signals": [ "Consent ledger not linked to AI data stores for any use case involving personal data.", "Consent-withdrawal events not propagated to feature stores or training sets within the 24-hour SLA.", "Model impact register not maintained or not reviewed following consent-withdrawal events affecting deployed models." ] }, "data_engineer": { "summary": "The consent gateway must be a technical enforcement layer in the data pipeline, not a documentation artefact. Build it as a query-time filter that is impossible to bypass without generating an audit event.", "actions": [ "Implement the consent gateway as a middleware layer in all AI data access paths that queries the consent ledger and filters results before returning data to the requesting pipeline.", "Build a consent-withdrawal event consumer that subscribes to the withdrawal event stream and updates suppression flags in training data stores within the 24-hour SLA.", "Instrument the consent gateway with metrics for consent check latency, filter rate, and bypass attempts; alert on any bypass attempt that does not generate a corresponding audit event." ], "failure_signals": [ "AI data access paths that bypass the consent gateway without generating an audit event.", "Consent-withdrawal event consumer lag exceeding the 24-hour SLA without an alert.", "Consent gateway not deployed on any data store containing personal data used for AI training." ] }, "legal_counsel": { "summary": "Consent withdrawal must trigger immediate data access suppression. Assess legal exposure from AI models trained on data for which consent has been subsequently withdrawn and advise on remediation obligations.", "actions": [ "Confirm that the consent legal basis documentation covers each AI processing purpose and is compatible with data subject rights under GDPR Art. 17 (right to erasure).", "Advise on the legal obligation to retrain or suppress outputs from models trained on withdrawn-consent data, distinguishing between erasure obligations and model output suppression.", "Review the consent withdrawal SLA and confirm it satisfies the organisation's obligations under applicable data protection law and any sector-specific requirements." ], "failure_signals": [ "AI use cases relying on consent as a legal basis without a documented withdrawal propagation procedure.", "Trained models identified in the impact register not reviewed by legal counsel within 30 days.", "Consent withdrawal SLA exceeding what is legally permissible under the applicable regulatory regime." ] }, "grc_auditor": { "summary": "Audit the completeness of the consent ledger, the timeliness of withdrawal propagation, and the maintenance of the model impact register.", "actions": [ "Sample 20 consent withdrawal events from the prior quarter and verify each was propagated to all linked AI data stores within the 24-hour SLA.", "Cross-reference the model impact register against consent withdrawal events to confirm no deployed models are missing from the register.", "Verify the consent gateway is deployed on all data stores flagged as containing personal data used for AI training or inference." ], "metrics": [ "Consent withdrawal propagation SLA compliance: 100% within 24 hours.", "Model impact register completeness: 100% of affected deployed models identified within 48 hours of a withdrawal event.", "Consent gateway deployment coverage: 100% of personal data stores used for AI training or inference." ], "failure_signals": [ "Withdrawal events not propagated within the SLA in any sampled case.", "Deployed models absent from the impact register following withdrawal events.", "Personal data stores without consent gateway deployment discovered during sampling." ] }, "it_operations": { "summary": "Operate the consent ledger and event stream infrastructure reliably and ensure suppression signals are delivered to all downstream systems within the SLA.", "actions": [ "Monitor consent ledger availability and alert if uptime falls below 99.9%; configure the consent gateway to fail-closed if the ledger is unavailable.", "Monitor the consent-withdrawal event stream for consumer lag and alert if propagation to any downstream system is at risk of breaching the 24-hour SLA.", "Maintain a tested failover procedure for the consent ledger to ensure continuity of consent enforcement during infrastructure incidents." ], "failure_signals": [ "Consent ledger downtime causing the consent gateway to fail-open and allow unchecked data access.", "Event stream consumer lag exceeding 6 hours without an alert being raised.", "No tested failover procedure for the consent ledger in the operational runbook." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most AI data pipelines do not have a consent gateway; consent withdrawal is handled manually with long propagation delays and no model impact register." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "eu-high-risk-ai", "high-risk-sector", "multi-tenant" ], "implementers": [ "Data Engineering", "Data Governance Team", "Legal / Privacy" ], "frameworks": [ { "framework": "gdpr", "requirement_id": "Art. 5(1)(b)", "fit": "direct", "rationale": "GDPR Article 5(1)(b) requires personal data to be collected for specified, explicit, and legitimate purposes and not further processed in a manner incompatible with those purposes. DA-03's consent gateway and purpose-code linking directly enforce this requirement for AI data access by gating access on a valid, current consent record tied to the AI processing purpose.", "normative_force": "binding-law", "source_version": "2016/679", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "iso_27701", "requirement_id": "§7.2.3", "fit": "direct", "rationale": "ISO/IEC 27701 §7.2.3 requires organisations to identify and document the lawful basis for each processing activity and to verify that basis at the point of processing. DA-03 operationalises this by maintaining a consent ledger queried at access time, ensuring the lawful basis is verified before data is returned to the AI pipeline.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_pf", "requirement_id": "CT.PO-P1", "fit": "direct", "rationale": "NIST Privacy Framework CT.PO-P1 requires policies, processes, and procedures for authorizing data processing — explicitly including individual consent — together with revoking and maintaining those authorizations. Consent-based access controls implement exactly this authorization-and-revocation lifecycle for AI data use.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "eu_ai_act", "requirement_id": "Art. 10(5)", "fit": "direct", "rationale": "EU AI Act Article 10(5) permits use of special-category data in training sets only under strict necessity with appropriate safeguards. DA-03's consent-based access controls are a required safeguard for such processing, ensuring only data with a valid, current consent basis is accessible to AI training pipelines.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "anthropic_privacy", "requirement_id": "Privacy Policy", "fit": "adjacent", "rationale": "Anthropic's Privacy Policy describes lawful bases and user choices for data use, including opt-out and deletion mechanisms. It is a document-level example of the consent and preference disclosures that consent-based access control must honor across vendors.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "owasp_ai_exchange", "requirement_id": "alloweddata", "fit": "supporting", "rationale": "DA-03 gates every personal-data access through a consent gateway verifying a valid consent record for the processing purpose, restricting AI use to consented data.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://data/controls/DA-03", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every personal data access by an AI system must be gated by a consent gateway that verifies a valid, current consent record linked to the applicable processing purpose before returning data. Consent withdrawal events must propagate to all linked AI data stores within 24 hours, and the model impact register must identify all affected deployed models within 48 hours of any withdrawal event.", "evidence_required": [ "consent_gateway_deployment_record listing each personal data store used for AI training or inference with gateway_deployed=true, gateway_version, and last_tested_date for each store", "consent_withdrawal_propagation_log for sampled withdrawal events showing event_id, withdrawal_timestamp, affected_data_stores[], propagation_completed_at per store, and sla_compliance=true/false", "model_impact_register showing model_id, training_dataset_ids, withdrawal_event_id, register_entry_created_at, and review_status for each entry created in the audit period", "consent_ledger_integrity_check_report from the quarterly reconciliation showing consent_records_audited, gaps_found, and resolution_actions taken" ], "machine_tests": [ "Submit a data access query for a record belonging to a data subject with consent_status=withdrawn in the consent ledger for the requested purpose_code → assert the consent gateway returns 0 records for that subject and logs a consent_check_result=withdrawn audit event with data_subject_id and purpose_code", "Emit a consent-withdrawal event for a test data subject on the event stream → assert all subscribed AI data stores receive the suppression signal and update their filter state within 24 hours, confirmed by querying each store's suppression_log for the test subject", "Query the model impact register API 48 hours after emitting a test withdrawal event affecting a deployed model's training dataset → assert the affected model_id appears in the register with withdrawal_event_id linked and review_status=pending" ], "human_review": [ "Sample 20 consent withdrawal events from the prior quarter and verify each was propagated to all linked AI data stores within the 24-hour SLA, with no unlogged exceptions or manual workarounds", "Review the model impact register for the past quarter and confirm every entry has a named reviewer assigned within 48 hours of creation, with legal counsel engaged for entries involving deployed models serving live users", "Confirm the consent gateway is configured to fail-closed when the consent ledger is unavailable by reviewing the gateway failsafe configuration documentation and the most recent infrastructure failover test record" ], "blocking_effect": "blocks-runtime-action", "normative_status": "binding-law", "anti_patterns": [ "Managing consent withdrawal through manual processes with no automated event stream, relying on data engineers to manually remove records from training sets after receiving withdrawal notifications via email or ticket", "Deploying a consent gateway on only some personal data stores used by AI pipelines, leaving other stores accessible without consent verification and creating exploitable consent bypass paths", "Maintaining the model impact register as a manually updated spreadsheet with no automated linkage to withdrawal events, failing to identify deployed models containing withdrawn-consent data within the required 48-hour window", "Treating AI model retraining as always required after consent withdrawal without a documented legal analysis distinguishing cases where output suppression is a legally sufficient and proportionate alternative", "Configuring the consent gateway to cache consent decisions for extended periods (e.g., 7 days) without a TTL invalidation mechanism triggered by withdrawal events, permitting continued access to withdrawn-consent subject records during the cache window" ], "update_status": "current", "layer_code": "DA" }, { "id": "DA-04", "layer": "DA", "plane": "control", "name": "Privileged Data Access Monitoring", "plain": "All access to sensitive AI data assets by privileged users (DBAs, data engineers, ML engineers, platform admins) and privileged processes (ETL service accounts, training orchestrators) must be monitored, logged immutably, and subject to real-time alerting when access patterns deviate from established behavioral baselines.", "threat": { "tags": [ "insider-threat", "privileged-access-abuse", "audit-log-tampering", "training-data-exfiltration" ], "desc": "Privileged users and service accounts with broad data access represent the highest-risk vector for AI training data compromise. Unlike external attackers, insiders already hold valid credentials and can exfiltrate large volumes of data without triggering perimeter controls. Service accounts used by training pipelines are frequently over-provisioned and can be weaponised by attackers who compromise the orchestration layer. Without behavioral baselines, anomalous access patterns go undetected until post-incident review." }, "standard": [ { "id": "cobit_2019", "section": "DSS06.03", "title": "Manage roles, responsibilities, access privileges" }, { "id": "dama_dmbok", "section": "Ch. 7", "title": "Data security monitoring requirements" }, { "id": "nist_pf", "section": "CT.DM-P8", "title": "Audit/log records determined, documented, implemented and reviewed" }, { "id": "iso_27701", "section": "§6.9.4", "title": "Logging and monitoring — event logs for PII access" } ], "sources": [ { "id": "microsoft_purview_data_governance_2024", "title": "Microsoft Purview (Data Governance)", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/purview/purview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_purview_data_governance_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Purview (Data Governance) requirements informing the apeiris://data/controls/DA-04 Privileged Data Access Monitoring control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Enumerate all privileged user accounts and service accounts with access to sensitive AI data; enforce just-in-time (JIT) privileged access for human users; route all privileged access events to an immutable audit log delivered to a SIEM within 60 seconds; define behavioral baselines per privileged identity and alert on deviations; require dual-control approval for bulk export operations.", "steps": [ "Enumerate all privileged user accounts and service accounts with access to AI training data, inference data stores, and feature stores; classify each identity as privileged in the access grant register.", "Implement just-in-time access provisioning for human privileged users so standing access to sensitive AI data is eliminated and each session requires an approval workflow with a declared scope and duration.", "Route all privileged access events (reads, writes, exports, schema changes) to an immutable audit log with tamper-evident hashing; deliver logs to the SIEM within 60 seconds of event occurrence.", "Define behavioral baselines per privileged identity covering typical query volume, data volume accessed, access hours, and tables accessed; configure SIEM alerts when any dimension deviates by more than two standard deviations from the baseline.", "Require dual-control approval from two authorised persons for bulk data export operations exceeding a defined volume threshold from AI data stores classified as confidential or restricted." ], "data_governance_officer": { "summary": "Privileged access monitoring is the governance safety net ensuring that even authorised actors cannot abuse their access without detection. Approve behavioral baselines and the dual-control export policy.", "actions": [ "Approve the enumeration of privileged identities and confirm each has a documented business justification for privileged access to AI data.", "Review SIEM anomalous access alert summaries monthly and confirm all alerts have been investigated and resolved within SLA.", "Sponsor the dual-control policy for bulk exports and ensure it is enforced without exception for AI data assets classified as restricted." ], "failure_signals": [ "Privileged identities with standing access to restricted AI data stores not subject to JIT provisioning.", "SIEM alert backlog exceeding five unresolved anomalous access events at any point.", "Bulk export approvals completed by a single approver without dual-control documentation." ] }, "data_engineer": { "summary": "Instrument all data access paths used by privileged service accounts with comprehensive telemetry; ensure no privileged access path exists that bypasses the audit logging layer.", "actions": [ "Audit all service account IAM bindings on AI data stores and reduce to least-privilege; document each remaining privileged binding with a named justification.", "Deploy query-level logging on all data platforms (BigQuery audit logs, Snowflake query history, Databricks cluster access logs) and verify delivery to the SIEM within the 60-second SLA.", "Implement volume-based trip wires on data export APIs that trigger dual-control approval workflows when thresholds are exceeded." ], "failure_signals": [ "Service accounts with owner-level IAM bindings on AI data stores without documented justification.", "Query-level logging gaps on any data platform containing sensitive AI data.", "Bulk export volume thresholds not configured or not triggering dual-control workflows." ] }, "grc_auditor": { "summary": "Audit the completeness of the privileged identity enumeration, the coverage and timeliness of immutable logging, and the effectiveness of anomaly alerting.", "actions": [ "Request the privileged identity register and verify each identity has a documented justification, a named owner, and JIT access controls applied for human users.", "Sample 30 privileged access events from the prior quarter and verify each was logged in the immutable audit log with a tamper-evident hash within the 60-second delivery SLA.", "Review the prior quarter's SIEM alert log and confirm 100% of anomalous access alerts were investigated and resolved within the defined 4-hour investigation SLA." ], "metrics": [ "Privileged standing access elimination: 100% of human privileged users on JIT provisioning.", "Audit log delivery SLA: 100% of privileged access events delivered to SIEM within 60 seconds.", "SIEM alert investigation SLA: 100% of anomalous access alerts investigated and resolved within 4 hours." ], "failure_signals": [ "Human privileged users with standing access to restricted AI data stores discovered during sampling.", "Audit log delivery gaps or events arriving outside the 60-second SLA.", "Unresolved SIEM anomalous access alerts older than 4 hours." ] }, "it_operations": { "summary": "Operate the JIT access provisioning system and SIEM log pipeline reliably; validate tamper-evident hashing integrity on a weekly schedule.", "actions": [ "Maintain JIT access provisioning system availability at 99.9%; document fallback procedures for incident response access that do not eliminate audit logging.", "Monitor SIEM log ingestion pipeline for lag or dropped events; alert on any log delivery gap exceeding 5 minutes for privileged access sources.", "Run weekly tamper-evident hash validation on the immutable audit log to confirm no events have been modified or deleted." ], "failure_signals": [ "JIT provisioning system unavailability causing manual access provisioning without audit events.", "Log delivery gap exceeding 5 minutes without an alert being raised.", "Tamper-evident hash validation failure on any segment of the immutable audit log." ] }, "legal_counsel": { "summary": "Privileged users can bypass every other data control, so their access to personal and regulated data carries heightened legal risk — including employment-law limits on monitoring the monitors.", "actions": [ "Confirm privileged-access monitoring of AI data stores is itself lawful under employment and privacy law in each jurisdiction.", "Define the privileged-access events that require legal notification — bulk export of personal data, access under legal hold.", "Review investigation procedures so privileged-abuse cases preserve evidence lawfully." ], "failure_signals": [ "Employee-monitoring objections or works-council disputes over unreviewed monitoring scope.", "Bulk exports of regulated data by administrators with no legal escalation.", "Privileged-abuse investigations that mishandle evidence or employee rights." ] } }, "maturity": { "current": "repeatable", "target": "managed", "notes": "Many organisations have basic audit logging but lack behavioral baselines and JIT access provisioning specifically for AI data assets." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "federated-enterprise" ], "implementers": [ "IAM Team", "Security Operations", "Data Engineering", "Platform Engineering" ], "frameworks": [ { "framework": "cobit_2019", "requirement_id": "DSS06.03", "fit": "direct", "rationale": "COBIT 2019 DSS06.03 requires management of access privileges and monitoring of privileged user activity to detect misuse. DA-04 directly implements this requirement for AI data assets through JIT provisioning, immutable audit logging, and behavioral anomaly detection configured specifically for AI data access patterns.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 7", "fit": "direct", "rationale": "DAMA DMBOK2 Chapter 7 establishes monitoring of privileged access to sensitive data as a data security management requirement. DA-04 implements these requirements specifically for AI training and inference data assets, which represent a high-value target class not always addressed in general-purpose data security programs.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "nist_pf", "requirement_id": "CT.DM-P8", "fit": "direct", "rationale": "NIST Privacy Framework CT.DM-P8 requires audit/log records to be determined, documented, implemented, and reviewed in accordance with policy. Privileged access monitoring implements this for the highest-risk class of AI data access — administrator and service-principal activity.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "iso_27701", "requirement_id": "§6.9.4", "fit": "direct", "rationale": "ISO/IEC 27701:2019 §6.9.4 extends ISO/IEC 27002 logging-and-monitoring guidance to PII: event logs recording access to personal data should be produced, protected and regularly reviewed. Privileged access monitoring implements this for administrator and break-glass access to AI data stores. §8.2.6 concerns processors' records of processing, not event logging.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "microsoft_purview", "requirement_id": "Purview Insider Risk Management", "fit": "adjacent", "rationale": "Microsoft Purview Insider Risk Management provides behavioral analytics over user activity signals to flag risky data handling by privileged users. It is a reference implementation model for DA-04's monitoring of privileged access to AI data on Microsoft platforms.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://data/controls/DA-04", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "All privileged user accounts and service accounts with access to AI data assets must have every access event captured in an immutable audit log delivered to the SIEM within 60 seconds, with behavioral baselines active per identity and anomaly alerts generated within two standard deviations of baseline — and every bulk export operation must require documented dual-control approval from two authorised persons before data is returned.", "evidence_required": [ "privileged_identity_register listing each privileged human account and service account with asset, access level, JIT enrollment status, and named business justification approved by a data owner", "immutable_audit_log_delivery_report confirming 100% of privileged access events delivered to SIEM within 60 seconds with tamper-evident hash for a representative 30-day sampling period", "behavioral_baseline_configuration_export per privileged identity showing monitored dimensions (query volume, data volume, access hours, tables accessed) and deviation thresholds", "siem_alert_resolution_log documenting each anomalous access alert from the prior quarter with alert timestamp, investigator identity, resolution action, and closure within the 4-hour SLA", "dual_control_approval_record for each bulk export operation showing two named approvers, declared scope, volume, and approval timestamps" ], "machine_tests": [ "Execute a privileged read on an AI training data store as a registered service account → assert the event appears in the SIEM within 60 seconds with entity_id, resource_id, query_volume, and tamper-evident hash present", "Trigger a bulk export exceeding the configured volume threshold from a confidential AI data store → assert the dual-control approval workflow is invoked and data is not returned until both approvers have confirmed", "Inject an out-of-sequence event into the immutable audit log segment → assert tamper-evident hash validation fails and a tamper alert is raised within the detection window", "Execute a privileged query outside the behavioral baseline access hours for a registered identity → assert a SIEM anomaly alert is generated within the detection window with the identity, deviation dimension, and threshold cited" ], "human_review": [ "Review the privileged identity register for completeness: verify every JIT-enrolled human and service account has a documented business justification approved by a named data owner, with no standing access granted to human users", "Assess SIEM behavioral baseline configurations for adequacy: confirm baselines cover query volume, data volume, access hours, and table scope per individual privileged identity rather than at role or team level", "Review the dual-control bulk export policy to confirm volume thresholds are calibrated to actual legitimate business patterns and are not set so high as to be never triggered in practice" ], "blocking_effect": "advisory", "normative_status": "certification-standard", "anti_patterns": [ "Relying on data platform native query logs without forwarding to a centralized immutable SIEM, allowing privileged users with platform-admin rights to delete or modify their own access records", "Defining behavioral baselines at the team or role level rather than per privileged identity, masking individual anomalous activity within aggregate team patterns and preventing attribution", "Setting bulk export volume thresholds at levels that are never triggered in practice (e.g., 100 GB) instead of calibrating to actual legitimate business export volumes", "Using shared service account credentials across multiple data engineers so SIEM alerts cannot be attributed to a specific human actor responsible for the anomalous access", "Implementing JIT access provisioning as a manual approval ticket without automated session expiry, leaving credentials active indefinitely after the initial approval window has passed" ], "update_status": "current", "layer_code": "DA" }, { "id": "DA-05", "layer": "DA", "plane": "lifecycle", "name": "AI Data Access Review and Recertification", "plain": "All access grants to AI systems and their supporting roles must be reviewed on a defined periodic schedule, with excess permissions revoked through automated enforcement and access grants that cannot be justified by a current business purpose automatically expired at the close of each recertification campaign.", "threat": { "tags": [ "access-creep", "orphaned-grants", "stale-permissions", "ex-employee-residual-access" ], "desc": "AI data access grants accumulate over model iteration cycles as projects are prototyped, pivoted, and deprecated without corresponding access revocation. Departed employees' accounts and decommissioned model service accounts retain access to training data stores indefinitely. Periodic recertification with automated expiry is the primary control for preventing access creep from silently expanding the attack surface over time." }, "standard": [ { "id": "cobit_2019", "section": "DSS06.03", "title": "Periodic access review requirements" }, { "id": "gdpr", "section": "Art. 5(1)(f) + Art. 32", "title": "Integrity, confidentiality and security of processing" }, { "id": "dama_dmbok", "section": "Ch. 7", "title": "Data security access review" } ], "sources": [ { "id": "cobit_2019", "title": "COBIT 2019", "authority": "ISACA", "source_type": "standard", "normative_force": "industry-framework", "version": "2019", "published_on": "2018-11-12", "retrieved_on": "2026-07-02", "canonical_url": "https://www.isaca.org/resources/cobit", "license": "proprietary", "status": "current", "flagship": false, "source_id": "cobit_2019", "relationship": "informative_reference", "rationale": "Establishes COBIT 2019 DSS06.03 requirements informing the apeiris://data/controls/DA-05 AI Data Access Review and Recertification control.", "reviewed_on": "2026-07-02" }, { "id": "gdpr", "title": "GDPR", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2016/679", "published_on": "2016-04-27", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj", "license": "open-access", "status": "current", "flagship": false, "source_id": "gdpr", "relationship": "normative_requirement", "rationale": "Establishes GDPR Art. 5(1)(f) + Art. 32 requirements informing the apeiris://data/controls/DA-05 AI Data Access Review and Recertification control.", "reviewed_on": "2026-07-02" }, { "id": "dama_dmbok", "title": "DAMA-DMBOK 2nd Ed", "authority": "DAMA International", "source_type": "standard", "normative_force": "best-practice", "version": "2 (2nd Edition, Revised 2024)", "published_on": "2017-07-05", "retrieved_on": "2026-07-02", "canonical_url": "https://www.dama.org/cpages/body-of-knowledge", "license": "proprietary", "status": "current", "flagship": false, "source_id": "dama_dmbok", "relationship": "informative_reference", "rationale": "Establishes DAMA-DMBOK 2nd Ed Ch. 7 requirements informing the apeiris://data/controls/DA-05 AI Data Access Review and Recertification control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Maintain a living access grant register for all AI data assets; schedule automated recertification campaigns every 90 days; distribute recertification tasks to named data owners; automatically expire grants not recertified within the campaign window; produce a recertification summary report for governance review.", "steps": [ "Build a living access grant register enumerating every access grant to AI data assets with: grantee identity, grantee type (human/service account), data asset, access level, grant date, last recertification date, and granting justification.", "Schedule automated recertification campaigns every 90 days; distribute recertification tasks to named data owners for each asset in scope and set a 14-day campaign completion window.", "Automatically expire any access grant not recertified within the campaign window; generate an expiry notice to the grantee and their manager and enforce revocation in the data platform within 24 hours of expiry.", "Require data owners to provide a brief written justification for each grant they recertify; flag grants recertified without justification for follow-up review within 5 business days.", "Produce a quarterly recertification report summarising total grants reviewed, grants revoked, grants recertified with justification gaps, and any campaigns with completion below 100%." ], "data_governance_officer": { "summary": "Recertification is the primary control for maintaining least privilege over the AI data access lifecycle. Own the recertification schedule and review the quarterly summary report.", "actions": [ "Publish and maintain the recertification campaign schedule; ensure all data owners are notified at least 30 days before each campaign opens.", "Review the quarterly recertification report and escalate any data asset with a campaign completion rate below 100% to the responsible owner.", "Maintain the access grant register as a live artifact; require all new access grants to be entered within 24 hours of provisioning." ], "failure_signals": [ "Recertification campaign completion rate below 100% for any data asset without a documented exception.", "Access grants in the register with no recertification date within the prior 90 days.", "New access grants not entered into the register within 24 hours of provisioning." ] }, "data_engineer": { "summary": "Automate grant discovery and expiry enforcement so the recertification process is complete and accurate without relying on manual inventory.", "actions": [ "Build automated discovery scripts that scan all AI data stores and enumerate current access grants; reconcile against the access grant register daily and surface any gaps.", "Implement automated expiry enforcement that revokes grants in the data platform (IAM bindings, SQL grants) when the recertification system flags them as expired, within the 24-hour SLA.", "Surface a self-service dashboard to data owners showing their pending recertification tasks, grant history, and last-active-date for each AI data asset they own." ], "failure_signals": [ "Automated discovery scripts not running or producing results older than 24 hours.", "Expired grants in the recertification system not revoked in the data platform within 24 hours.", "Gaps between the access grant register and runtime IAM state for any AI data store." ] }, "grc_auditor": { "summary": "Audit recertification completeness, accuracy, and timeliness; verify automated expiry enforcement is functioning and the access grant register accurately reflects runtime state.", "actions": [ "Review the quarterly recertification report and verify campaign completion rates, revocation counts, and justification gap rates across all AI data assets.", "Sample 25 access grants from the register and cross-reference each against runtime IAM state to confirm register accuracy.", "Verify that automated expiry enforcement revoked all flagged grants within the 24-hour SLA by sampling expiry events from the prior quarter." ], "metrics": [ "Recertification campaign completion rate: 100% within the 14-day campaign window.", "Access grant register accuracy: zero divergences between register and runtime IAM state in sampled grants.", "Automated expiry enforcement SLA: 100% of expired grants revoked within 24 hours." ], "failure_signals": [ "Campaign completion below 100% in any quarter without a documented exception.", "Divergences between register and runtime IAM state discovered during sampling.", "Expired grants persisting beyond the 24-hour revocation SLA." ] }, "it_operations": { "summary": "Operate the automated discovery and expiry enforcement infrastructure and ensure the access grant register is populated accurately and in near-real time.", "actions": [ "Monitor automated grant discovery jobs for failures and alert when a job has not completed successfully within its scheduled window.", "Ensure IAM revocation commands executed by the expiry enforcement system are applied successfully; log each revocation event to the immutable audit sink.", "Coordinate with data engineers on schema changes or platform migrations that require access grant register updates outside the normal 90-day campaign cycle." ], "failure_signals": [ "Discovery jobs failing silently without generating alerts.", "IAM revocation commands failing without a retry and alert mechanism.", "Platform migrations not triggering an out-of-cycle access grant register reconciliation." ] }, "legal_counsel": { "summary": "Stale access is indefensible: recertification is the evidence that access remains necessary and proportionate. Counsel relies on review records when demonstrating GDPR Art. 32-appropriate access management.", "actions": [ "Set recertification frequency for access to special-category and regulated data.", "Use recertification records as evidence in audits and breach investigations.", "Require immediate review triggers on role change, offboarding and vendor termination." ], "failure_signals": [ "Ex-employees or terminated vendors retaining access to AI data stores.", "Recertification campaigns rubber-stamped with no evidence of actual review.", "Breach investigations finding access that should have been removed cycles earlier." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most enterprises rely on manual access reviews with low completion rates; automated recertification with expiry enforcement for AI data assets specifically is uncommon." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "Data Governance Team", "IAM Team", "Data Engineering" ], "frameworks": [ { "framework": "cobit_2019", "requirement_id": "DSS06.03", "fit": "direct", "rationale": "COBIT 2019 DSS06.03 explicitly requires periodic review of access privileges to ensure they remain appropriate and are revoked when no longer needed. DA-05's 90-day recertification cycle and automated expiry enforcement directly implement this requirement for AI data access grants with measurable completion metrics.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "gdpr", "requirement_id": "Art. 5(1)(f) + Art. 32", "fit": "partial", "rationale": "GDPR Article 5(1)(f) requires processing with appropriate security (integrity and confidentiality), and Article 32 requires technical and organisational measures appropriate to the risk — which supervisory practice reads to include periodic review of access rights. Access review and recertification for AI data stores directly evidences these obligations; Art. 5(1)(e) concerns storage limitation, not access lifecycle.", "normative_force": "binding-law", "source_version": "2016/679", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 7", "fit": "direct", "rationale": "DAMA DMBOK2 Chapter 7 includes periodic access review as a core data security management practice. DA-05 operationalises this practice for AI data assets with a structured campaign process, automated expiry enforcement, and governance reporting designed to achieve and demonstrate 100% completion.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://data/controls/DA-05", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every access grant to an AI data asset must have been reviewed and recertified by the named data owner within the prior 90 days, with all grants not recertified within the 14-day campaign window automatically revoked in the data platform within 24 hours of expiry — such that the runtime IAM state contains zero AI data grants older than 90 days without a current recertification record.", "evidence_required": [ "access_grant_register export showing all current grants with grantee identity, grantee type (human/service account), asset, access level, grant date, last recertification date, and recertification justification as of the most recent campaign close", "recertification_campaign_report for the most recent 90-day cycle showing 100% completion rate, total grants reviewed, grants revoked, and justification-gap exceptions requiring follow-up", "automated_expiry_enforcement_log confirming all flagged-expired grants were revoked in the data platform IAM layer within 24 hours of campaign close, with revocation timestamps", "runtime_iam_reconciliation_report confirming zero divergence between the access grant register and the runtime IAM state across a sample of AI data stores from the prior quarter" ], "machine_tests": [ "Mark a test access grant as recertification-expired in the recertification system → assert the IAM binding is revoked in the data platform within 24 hours and subsequent access attempts return 403", "Advance a test campaign window past the 14-day deadline for an unreviewed grant → assert the grant is automatically expired, the grantee and manager receive notification, and revocation is enforced within 24 hours", "Compare the access grant register against runtime IAM bindings across all enrolled AI data stores → assert zero undocumented runtime grants are present that are absent from the register" ], "human_review": [ "Review a sample of recertified grants to verify data owners provided specific written justifications for each individual grant rather than bulk-approving all grants without per-grant review", "Assess recertification campaign coverage to confirm all AI data stores including shadow stores, inference endpoints, and batch-processing service accounts are enrolled in the 90-day cycle", "Evaluate exception handling for campaigns with completion below 100%: confirm exceptions are escalated to the data governance officer within the campaign window rather than resolved after close with grandfathered access" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Running recertification campaigns as a manual email-based bulk-approval process where data owners approve all grants in a single click without reviewing individual justifications, treating the campaign as a compliance checkbox", "Scoping recertification campaigns only to explicitly provisioned IAM grants while omitting service accounts, shared credentials, and database-level SQL grants, which then accumulate without review", "Defaulting unreviewed grants to 'continue' when the campaign closes below 100% completion, allowing stale access to persist indefinitely under the guise of pending review", "Applying the same 90-day recertification cadence to high-sensitivity AI training data assets as to low-sensitivity reference data without risk-tiered review frequency", "Failing to reconcile the access grant register against runtime IAM state after each campaign, so revocations performed outside the campaign process leave the register inaccurate" ], "update_status": "current", "layer_code": "DA" }, { "id": "DA-06", "layer": "DA", "plane": "control", "name": "Cross-Border Data Access Jurisdiction Controls", "plain": "Access to AI data assets subject to cross-border transfer restrictions must be filtered based on the jurisdictional context of the accessor; access requests from restricted jurisdictions must be blocked at the data platform layer or routed through a legally approved transfer mechanism before data is returned.", "threat": { "tags": [ "cross-border-transfer-violation", "jurisdiction-bypass", "data-sovereignty-breach", "regulatory-noncompliance" ], "desc": "Distributed AI teams and cloud-based inference infrastructure routinely result in data being accessed from jurisdictions where transfer is restricted or prohibited. An engineer in a non-adequate country querying a European personal data training set, or a model inference endpoint serving training data to a restricted-jurisdiction subsidiary, can constitute an illegal cross-border transfer without any malicious intent. The legal and reputational consequences can be severe, with regulatory fines and enforcement orders capable of disrupting AI operations entirely." }, "standard": [ { "id": "gdpr", "section": "Chapter V (Art. 44-49)", "title": "Transfers of personal data to third countries" }, { "id": "iso_27701", "section": "§7.5.1", "title": "Basis for transfer of personally identifiable information between jurisdictions" }, { "id": "nist_pf", "section": "GV.PO-P1", "title": "Organizational privacy values and policies established and communicated" } ], "sources": [ { "id": "gdpr", "title": "GDPR", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2016/679", "published_on": "2016-04-27", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj", "license": "open-access", "status": "current", "flagship": false, "source_id": "gdpr", "relationship": "normative_requirement", "rationale": "Establishes GDPR Chapter V (Art. 44-49) requirements informing the apeiris://data/controls/DA-06 Cross-Border Data Access Jurisdiction Controls control.", "reviewed_on": "2026-07-02" }, { "id": "iso_27701", "title": "ISO/IEC 27701:2019", "authority": "ISO/IEC", "source_type": "standard", "normative_force": "certification-standard", "version": "2019 (superseded by ISO/IEC 27701:2025, published 2025-10-14; 24-36 month certification transition)", "published_on": "2019-08-06", "retrieved_on": "2026-07-02", "canonical_url": "https://www.iso.org/standard/71670.html", "license": "proprietary", "status": "current", "flagship": false, "source_id": "iso_27701", "relationship": "implementation_pattern", "rationale": "Establishes ISO/IEC 27701:2019 §7.5.1 requirements informing the apeiris://data/controls/DA-06 Cross-Border Data Access Jurisdiction Controls control.", "reviewed_on": "2026-07-02" }, { "id": "nist_pf", "title": "NIST Privacy Framework 1.0", "authority": "National Institute of Standards and Technology", "source_type": "standard", "normative_force": "voluntary-standard", "version": "1.0", "published_on": "2020-01-16", "retrieved_on": "2026-07-02", "canonical_url": "https://www.nist.gov/privacy-framework", "license": "open-access", "status": "current", "flagship": false, "source_id": "nist_pf", "relationship": "implementation_pattern", "rationale": "Establishes NIST Privacy Framework 1.0 GV.PO-P1 requirements informing the apeiris://data/controls/DA-06 Cross-Border Data Access Jurisdiction Controls control.", "reviewed_on": "2026-07-02" }, { "id": "cobit_2019", "title": "COBIT 2019", "authority": "ISACA", "source_type": "standard", "normative_force": "industry-framework", "version": "2019", "published_on": "2018-11-12", "retrieved_on": "2026-07-02", "canonical_url": "https://www.isaca.org/resources/cobit", "license": "proprietary", "status": "current", "flagship": false, "source_id": "cobit_2019", "relationship": "informative_reference", "rationale": "Establishes COBIT 2019 DSS06.03 requirements informing the apeiris://data/controls/DA-06 Cross-Border Data Access Jurisdiction Controls control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Tag each AI data asset with a jurisdiction-restriction label specifying permitted accessor jurisdictions; implement an accessor jurisdiction detection mechanism at the data platform access layer; apply restriction rules as platform-layer access policies; maintain a transfer mechanism register linking any permitted cross-jurisdiction access to an approved legal basis.", "steps": [ "Classify all AI data assets with a jurisdiction-restriction tag specifying permitted accessor jurisdiction tiers (e.g., EEA-only, EU-adequate-countries, unrestricted) and maintain this classification in the data asset catalog.", "Implement accessor jurisdiction detection at the data platform access layer using requestor IP geolocation, cloud region context, and directory-linked country attributes to determine the jurisdictional context of each access request.", "Configure access control policies at the data platform layer that evaluate the accessor jurisdiction against the data asset's restriction tag and block requests from non-permitted jurisdictions at query time.", "Maintain a transfer mechanism register documenting the legal basis (adequacy decision, SCCs, BCRs, or Article 49 derogation) for any permitted access from jurisdictions not covered by the default restriction; require legal counsel sign-off for each entry.", "Generate a monthly cross-border access compliance report summarising access attempts by accessor jurisdiction, blocking actions taken, and any access permitted via documented transfer mechanisms." ], "data_governance_officer": { "summary": "Cross-border data access restrictions are a legal obligation tied to the organisation's data residency and transfer commitments. Maintain the jurisdiction-restriction classification and ensure the transfer mechanism register is current.", "actions": [ "Own the jurisdiction-restriction classification schema and ensure all new AI data assets are classified before access is provisioned.", "Review the transfer mechanism register quarterly with legal counsel to confirm all permitted cross-jurisdiction access routes have a current, valid legal basis.", "Review the monthly compliance report and escalate any pattern of blocked access attempts suggesting systematic attempts to circumvent jurisdiction controls." ], "failure_signals": [ "AI data assets with no jurisdiction-restriction classification tag in the data asset catalog.", "Cross-border access permitted without a corresponding entry in the transfer mechanism register.", "Transfer mechanism register entries not reviewed with legal counsel within the prior 12 months." ] }, "data_engineer": { "summary": "Implement jurisdiction detection and restriction enforcement at the data platform layer so cross-border access controls are enforced technically, not just by contract.", "actions": [ "Integrate an accessor jurisdiction detection service into the data platform access layer that resolves each requestor's jurisdictional context using IP geolocation and directory attributes.", "Configure data platform access policies (e.g., BigQuery VPC Service Controls, Snowflake network policies, Databricks IP access lists) to enforce jurisdiction-based restrictions for each classified data asset.", "Build reporting queries against access logs to produce the monthly cross-border compliance report, segmented by accessor jurisdiction and data asset." ], "failure_signals": [ "Accessor jurisdiction detection service not integrated into the data platform access layer.", "Data platform policies not enforcing jurisdiction restrictions for assets tagged as jurisdiction-restricted.", "Monthly compliance report generation failing or producing incomplete jurisdiction data." ] }, "legal_counsel": { "summary": "Jurisdiction controls are a direct regulatory compliance requirement. Advise on the adequacy determination for each permitted accessor jurisdiction and maintain the legal accuracy of the transfer mechanism register.", "actions": [ "Maintain the list of approved accessor jurisdictions for each data asset classification tier; update when adequacy decisions change or new regulatory developments occur.", "Review and sign off on each entry in the transfer mechanism register; confirm the cited legal basis (adequacy, SCCs, BCRs, or derogation) is current and applicable to the specific data processing.", "Assess detected jurisdiction restriction bypass attempts to determine whether they constitute reportable data incidents under applicable data protection law." ], "failure_signals": [ "Permitted access to restricted data assets from jurisdictions where no valid transfer mechanism exists in the register.", "Transfer mechanism register entries referencing SCCs or adequacy decisions that have since been invalidated.", "Detected jurisdiction bypass attempts not assessed for reportability within 72 hours." ] }, "grc_auditor": { "summary": "Audit the completeness of jurisdiction-restriction classification, the accuracy of transfer mechanism register entries, and the effectiveness of technical enforcement.", "actions": [ "Sample 20 AI data assets from the catalog and verify each has a current jurisdiction-restriction classification tag.", "Review the transfer mechanism register and confirm each entry has a valid signed legal basis and was reviewed with legal counsel within the past 12 months.", "Execute a test access attempt from a simulated restricted-jurisdiction context and confirm the access is blocked at the data platform layer." ], "metrics": [ "Jurisdiction-restriction classification coverage: 100% of AI data assets classified in the catalog.", "Transfer mechanism register currency: 100% of entries reviewed with legal counsel within 12 months.", "Restriction enforcement effectiveness: 100% of test access attempts from restricted jurisdictions blocked by platform controls." ], "failure_signals": [ "Unclassified AI data assets discovered during catalog sampling.", "Transfer mechanism entries not reviewed within the 12-month window.", "Test access from a restricted jurisdiction not blocked by data platform controls." ] }, "it_operations": { "summary": "Operate the accessor jurisdiction detection service and ensure data platform restriction policies are applied consistently across all environments including non-production.", "actions": [ "Monitor the jurisdiction detection service for accuracy and latency; alert if detection failure causes access decisions to default to permissive rather than restrictive behavior.", "Ensure data platform policies enforcing jurisdiction restrictions are applied consistently across production, staging, and development environments containing copies of production AI data.", "Maintain an incident response playbook for scenarios where an accessor circumvents jurisdiction controls through VPN, proxy, or cloud region switching." ], "failure_signals": [ "Jurisdiction detection service degradation causing access decisions to fail open.", "Jurisdiction restriction policies absent in non-production environments containing copies of production AI data.", "No incident response playbook documented for jurisdiction bypass scenarios." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Cross-border access controls for AI data are rarely implemented technically; most organisations rely on contractual controls in data processing agreements which are unenforceable at query time." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "eu-high-risk-ai", "federated-enterprise", "high-risk-sector" ], "implementers": [ "Data Governance Team", "Legal / Privacy", "Data Engineering", "Platform Engineering" ], "frameworks": [ { "framework": "gdpr", "requirement_id": "Chapter V (Art. 44-49)", "fit": "direct", "rationale": "GDPR Chapter V restricts transfers of personal data to third countries lacking an adequacy decision unless a valid transfer mechanism is in place. DA-06 directly implements these restrictions for AI data access by classifying data assets with jurisdiction tags and enforcing accessor jurisdiction checks at query time before any data is returned.", "normative_force": "binding-law", "source_version": "2016/679", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "iso_27701", "requirement_id": "§7.5.1", "fit": "direct", "rationale": "ISO/IEC 27701 §7.5.1 requires organisations to identify and document the basis for transferring PII between jurisdictions and to ensure transfers comply with applicable requirements. DA-06's transfer mechanism register and technical restriction enforcement directly operationalise this requirement for AI data access patterns.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "nist_pf", "requirement_id": "GV.PO-P1", "fit": "adjacent", "rationale": "NIST Privacy Framework GV.PO-P1 requires organizational privacy values and policies — including conditions on data processing — to be established and communicated. Jurisdiction-scoped access conditions are such policy conditions, applied to cross-border AI data access.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "cobit_2019", "requirement_id": "DSS06.03", "fit": "partial", "rationale": "COBIT 2019 DSS06.03 includes management of access privileges based on contextual attributes. Jurisdiction-based access restriction is an instance of attribute-based access control that extends COBIT's access management requirements to geographic and legal-context dimensions for AI data assets.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "owasp_ai_exchange", "requirement_id": "alloweddata", "fit": "supporting", "rationale": "DA-06 evaluates jurisdictional context and blocks restricted-jurisdiction access unless a valid GDPR Chapter V mechanism exists, permitting only lawful cross-border data access.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://data/controls/DA-06", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every access request to an AI data asset tagged with cross-border transfer restrictions must have the accessor's jurisdictional context evaluated at the data platform layer before any data is returned — with requests from restricted jurisdictions blocked unless routed through a documented valid GDPR Chapter V transfer mechanism — such that zero data returns occur to restricted jurisdictions without a recorded legal transfer basis.", "evidence_required": [ "data_asset_jurisdiction_tag_inventory showing each AI data asset's transfer restriction classification, restricted jurisdiction list, applicable legal transfer bases, and last tag review date", "access_request_jurisdiction_evaluation_log showing accessor identity, resolved jurisdiction, transfer basis applied (or block reason), and allow/block decision for each cross-border access event over a representative 30-day period", "transfer_mechanism_registry documenting each active adequacy decision, standard contractual clause set, and binding corporate rule applicable to each restricted AI data asset, with validity dates", "cross_border_block_event_log listing all blocked cross-border access attempts from the prior 90 days with accessor identity, resolved jurisdiction, requested asset, and block timestamp" ], "machine_tests": [ "Submit a query to a GDPR-restricted AI training dataset from an IP address in a non-adequate third country with no transfer mechanism configured → assert 403 response with error=jurisdiction_restricted before any data payload is returned", "Submit the same query from a non-adequate jurisdiction with a valid adequacy-basis token present → assert the request is routed through the approved transfer mechanism and the response includes transfer_basis metadata in the audit log", "Tag a test AI data asset as jurisdiction-restricted and query it from a restricted jurisdiction IP via a service-account access path → assert the platform-layer jurisdiction filter intercepts the request before it reaches the underlying data store" ], "human_review": [ "Review the data asset jurisdiction tag inventory for completeness and currency: verify all AI training datasets containing personal data of EU/UK subjects are tagged with appropriate transfer restrictions and that tags reflect the current adequacy decision landscape", "Assess each active transfer mechanism in the registry for legal validity: confirm SCCs use the 2021 EDPB-approved version, BCRs have current supervisory authority approval, and cited adequacy decisions remain in force", "Evaluate the platform-layer jurisdiction filtering implementation for bypass risk: confirm no API endpoints, maintenance interfaces, or service-account-accessible data paths bypass the jurisdiction check" ], "blocking_effect": "blocks-runtime-action", "normative_status": "binding-law", "anti_patterns": [ "Relying on contractual terms and accessor self-certification of jurisdiction rather than platform-layer enforcement, so jurisdiction violations require voluntary disclosure and are discovered only after transfer has occurred", "Applying jurisdiction controls only to direct user queries while exempting service account and API-based access used by training pipelines that may process data from restricted jurisdictions", "Using IP geolocation as the sole jurisdiction indicator without handling VPN, cloud egress, and proxy addresses that can mask the true accessor jurisdiction", "Tagging only datasets explicitly labeled as containing PII as jurisdiction-restricted while omitting AI training corpora derived from EU personal data sources where re-identification risk persists", "Failing to refresh jurisdiction restriction tags when adequacy decisions change, leaving controls calibrated to a legal landscape that no longer exists after a Schrems-style invalidation" ], "update_status": "current", "layer_code": "DA" }, { "id": "DA-07", "layer": "DA", "plane": "both", "name": "Emergency AI Data Access Procedures", "plain": "Break-glass access procedures must exist for emergency situations requiring access to AI data outside normal authorization pathways; every invocation must use time-limited scoped credentials, trigger mandatory real-time notification, generate enhanced telemetry, and be followed by a post-access review audit completed within 48 hours of access closure.", "threat": { "tags": [ "break-glass-abuse", "emergency-access-persistence", "post-incident-audit-gap", "credential-sharing" ], "desc": "Emergency access procedures, if poorly governed, become a standing bypass mechanism for normal access controls. Engineers who invoke break-glass access for a legitimate incident may leave credentials active or share them informally. Without mandatory post-access audit requirements and automatic credential expiry, emergency access events are not reviewed, creating a permanent blind spot that adversaries can exploit by fabricating emergency conditions to obtain unconstrained data access." }, "standard": [ { "id": "cobit_2019", "section": "DSS06.03", "title": "Emergency access and privileged access management" }, { "id": "dama_dmbok", "section": "Ch. 7", "title": "Emergency data access procedures" }, { "id": "nist_pf", "section": "CT.DM-P8", "title": "Audit/log records determined, documented, implemented and reviewed" }, { "id": "iso_27701", "section": "§6.9.4", "title": "Logging and monitoring — event logs for PII access" } ], "sources": [ { "id": "cobit_2019", "title": "COBIT 2019", "authority": "ISACA", "source_type": "standard", "normative_force": "industry-framework", "version": "2019", "published_on": "2018-11-12", "retrieved_on": "2026-07-02", "canonical_url": "https://www.isaca.org/resources/cobit", "license": "proprietary", "status": "current", "flagship": false, "source_id": "cobit_2019", "relationship": "informative_reference", "rationale": "Establishes COBIT 2019 DSS06.03 requirements informing the apeiris://data/controls/DA-07 Emergency AI Data Access Procedures control.", "reviewed_on": "2026-07-02" }, { "id": "dama_dmbok", "title": "DAMA-DMBOK 2nd Ed", "authority": "DAMA International", "source_type": "standard", "normative_force": "best-practice", "version": "2 (2nd Edition, Revised 2024)", "published_on": "2017-07-05", "retrieved_on": "2026-07-02", "canonical_url": "https://www.dama.org/cpages/body-of-knowledge", "license": "proprietary", "status": "current", "flagship": false, "source_id": "dama_dmbok", "relationship": "informative_reference", "rationale": "Establishes DAMA-DMBOK 2nd Ed Ch. 7 requirements informing the apeiris://data/controls/DA-07 Emergency AI Data Access Procedures control.", "reviewed_on": "2026-07-02" }, { "id": "iso_27701", "title": "ISO/IEC 27701:2019", "authority": "ISO/IEC", "source_type": "standard", "normative_force": "certification-standard", "version": "2019 (superseded by ISO/IEC 27701:2025, published 2025-10-14; 24-36 month certification transition)", "published_on": "2019-08-06", "retrieved_on": "2026-07-02", "canonical_url": "https://www.iso.org/standard/71670.html", "license": "proprietary", "status": "current", "flagship": false, "source_id": "iso_27701", "relationship": "implementation_pattern", "rationale": "Establishes ISO/IEC 27701:2019 §6.9.4 requirements informing the apeiris://data/controls/DA-07 Emergency AI Data Access Procedures control.", "reviewed_on": "2026-07-02" }, { "id": "gdpr", "title": "GDPR", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2016/679", "published_on": "2016-04-27", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj", "license": "open-access", "status": "current", "flagship": false, "source_id": "gdpr", "relationship": "normative_requirement", "rationale": "Establishes GDPR Art. 5(1)(f) requirements informing the apeiris://data/controls/DA-07 Emergency AI Data Access Procedures control.", "reviewed_on": "2026-07-02" }, { "id": "nist_pf", "title": "NIST Privacy Framework 1.0", "authority": "National Institute of Standards and Technology", "source_type": "standard", "normative_force": "voluntary-standard", "version": "1.0", "published_on": "2020-01-16", "retrieved_on": "2026-07-02", "canonical_url": "https://www.nist.gov/privacy-framework", "license": "open-access", "status": "current", "flagship": false, "source_id": "nist_pf", "relationship": "implementation_pattern", "rationale": "Establishes NIST Privacy Framework 1.0 CT.DM-P8 requirements informing the apeiris://data/controls/DA-07 Emergency AI Data Access Procedures control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Define a break-glass access runbook with time-limited credentials, dual-person authorisation, mandatory real-time notification to the data governance officer and CISO, automatic access expiry after the declared emergency window, and a mandatory post-access review audit completed within 48 hours of access closure.", "steps": [ "Define the break-glass access procedure in a runbook specifying trigger conditions, required approvers (minimum two), notification recipients (data governance officer, CISO, legal counsel for personal data assets), and maximum access window duration (default: 4 hours, extendable once with additional senior approval).", "Implement a break-glass credential issuance system that generates short-lived, scoped IAM tokens or temporary database credentials covering only the declared data asset and access type; credentials must expire automatically at the declared window end.", "Configure mandatory real-time notification to the data governance officer and CISO at the moment break-glass access is invoked, including the requestor identity, declared emergency type, data asset accessed, and approved window duration.", "Enable enhanced telemetry for all break-glass sessions that logs query text, record counts accessed, and data volume to a dedicated emergency access audit log separate from the standard audit sink.", "Mandate a post-access review audit completed within 48 hours of break-glass access closure, documenting what data was accessed, confirming it was necessary for the declared emergency, and recording any residual risk for escalation." ], "data_governance_officer": { "summary": "Emergency access is the highest-risk data access event. Receive real-time notification of every break-glass invocation and review post-access audits within SLA.", "actions": [ "Maintain the break-glass access runbook; review and update it at least annually or after any break-glass event that reveals a procedural gap.", "Review and approve every post-access audit within 48 hours of its submission; escalate to CISO if the audit reveals data accessed beyond the declared scope.", "Track break-glass access frequency as a governance KRI; escalate if frequency exceeds three invocations per quarter per data asset as a signal of normalised bypass behaviour." ], "failure_signals": [ "Break-glass access invoked without real-time notification reaching the data governance officer within 15 minutes.", "Post-access audits not completed within the 48-hour SLA.", "Break-glass frequency exceeding the governance threshold without escalation or documented exception." ] }, "data_engineer": { "summary": "Implement the break-glass credential issuance system so emergency access is scoped, time-bounded, and comprehensively logged with query-level telemetry.", "actions": [ "Build the break-glass credential issuance system using short-lived IAM tokens or temporary database credentials with scope limited to the declared data asset and declared access type.", "Implement automatic credential expiry and access revocation at the window end; alert the data governance officer and CISO if revocation does not complete within 5 minutes of expiry.", "Configure enhanced telemetry for break-glass credentials that logs every query executed, including query text and record count, to the dedicated emergency access audit log." ], "failure_signals": [ "Break-glass credentials not expiring automatically at the declared window end.", "Enhanced telemetry not capturing query text for break-glass sessions.", "Break-glass credential issuance system not integrated with the real-time notification pipeline." ] }, "legal_counsel": { "summary": "Emergency access to personal data may constitute a data processing incident requiring regulatory notification assessment. Ensure the runbook includes a legal review trigger for all break-glass access to personal data assets.", "actions": [ "Review the break-glass runbook to confirm it includes a legal notification trigger for access to data assets containing personal data, particularly special-category data.", "Assess post-access audits involving personal data to determine whether the access scope, combined with the emergency context, constitutes a reportable incident under applicable data protection law.", "Advise on the retention period for emergency access audit logs to satisfy both legal defensibility requirements and data minimization obligations." ], "failure_signals": [ "Break-glass runbook without a legal notification trigger for personal data access.", "Post-access audits involving personal data not reviewed by legal counsel within 48 hours.", "Emergency access audit logs retained for a period inconsistent with legal counsel advice." ] }, "grc_auditor": { "summary": "Audit every break-glass access event: verify dual authorisation, notification delivery, access scope, enhanced telemetry completeness, and post-access audit timeliness.", "actions": [ "Review the register of all break-glass access events in the prior quarter and verify each was invoked with at least two approvers and that real-time notifications were delivered to all required recipients.", "Sample post-access audit reports and verify each documents the accessed data, confirms access necessity, and was completed within the 48-hour SLA.", "Verify the break-glass credential issuance system automatically expired credentials at the declared window end for every event in the sample." ], "metrics": [ "Break-glass dual-person authorisation compliance: 100% of events with at least two approvers.", "Post-access audit completion rate: 100% completed within 48 hours of access closure.", "Automatic credential expiry compliance: 100% of credentials expired within 5 minutes of the declared window end." ], "failure_signals": [ "Break-glass events with single-person authorisation discovered in quarterly review.", "Post-access audits not completed within the 48-hour SLA in any sampled event.", "Credentials persisting beyond the declared window end without a documented extension approval." ] }, "it_operations": { "summary": "Operate the break-glass credential issuance system, notification pipeline, and enhanced telemetry infrastructure reliably under the high-stress conditions of genuine emergency response.", "actions": [ "Conduct quarterly break-glass procedure tests with simulated emergency scenarios to verify credential issuance, notification delivery, and telemetry pipeline function correctly under load.", "Maintain a tested fallback procedure for the scenario where the break-glass issuance system itself is unavailable during a genuine emergency, ensuring fallback procedures also generate audit events.", "Monitor the break-glass notification pipeline for delivery failures and alert the data governance officer immediately if a notification fails to reach any required recipient." ], "failure_signals": [ "Quarterly break-glass procedure test not completed within the scheduled quarter.", "Notification delivery failures not triggering immediate alerts.", "No documented and tested fallback procedure for break-glass system unavailability." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Emergency access procedures for AI data assets are rarely documented; break-glass access typically occurs through shared privileged credentials with no post-access review requirement or automatic expiry." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "cloud-native", "federated-enterprise" ], "implementers": [ "IAM Team", "Security Operations", "Data Governance Team", "Platform Engineering" ], "frameworks": [ { "framework": "cobit_2019", "requirement_id": "DSS06.03", "fit": "direct", "rationale": "COBIT 2019 DSS06.03 requires management of emergency access and privileged access procedures, including time-limited credentials and post-use review. DA-07's break-glass procedure with dual authorisation, time-bounded credential issuance, and mandatory post-access audit directly implements these requirements for AI data assets.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 7", "fit": "direct", "rationale": "DAMA DMBOK2 Chapter 7 includes emergency access procedures as a data security management requirement. DA-07 operationalises these procedures with a formal runbook, time-limited credential issuance, and mandatory post-access audit specific to AI data assets.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "iso_27701", "requirement_id": "§6.9.4", "fit": "direct", "rationale": "ISO/IEC 27701:2019 §6.9.4 requires event logging and log review extended to PII access. Emergency access is exactly the scenario where complete event logs and mandatory review carry the control weight; §8.2.6 concerns processors' records of processing.", "normative_force": "certification-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "gdpr", "requirement_id": "Art. 5(1)(f)", "fit": "partial", "rationale": "GDPR Article 5(1)(f) requires appropriate security for personal data including protection against unauthorised access. Emergency access, if not tightly controlled, represents a pathway for disproportionate or unauthorised access to personal data. DA-07's dual authorisation and time-limited credential controls satisfy this integrity-and-confidentiality requirement for emergency scenarios.", "normative_force": "binding-law", "source_version": "2016/679", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "nist_pf", "requirement_id": "CT.DM-P8", "fit": "adjacent", "rationale": "NIST Privacy Framework CT.DM-P8 requires audit/log records determined, documented, implemented and reviewed per policy. Break-glass access is the case where complete logging and mandatory after-the-fact review are the primary compensating control.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://data/controls/DA-07", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every invocation of the emergency AI data access procedure must use time-limited scoped credentials that automatically expire no later than 4 hours after issuance, trigger mandatory real-time notification to the data governance officer and security operations within 5 minutes of invocation, and be followed by a documented post-access review audit completed within 48 hours of access closure — with zero break-glass sessions persisting beyond credential expiry without a new approved invocation.", "evidence_required": [ "break_glass_invocation_log for the prior 12 months showing each invocation with invoker identity, second authoriser identity, asset accessed, emergency justification code, credential scope, issuance timestamp, and credential expiry timestamp", "real_time_notification_delivery_record confirming governance officer and security operations notifications were sent within 5 minutes of each break-glass invocation, with delivery acknowledgement", "post_access_review_report for each invocation completed within 48 hours of access closure, documenting what data was accessed, necessity and proportionality assessment, and any remediation actions", "credential_expiry_enforcement_log confirming each break-glass credential was deactivated at or before the configured expiry time with no sessions persisting beyond expiry" ], "machine_tests": [ "Issue a break-glass credential and allow the configured expiry window to elapse without renewal → assert all active sessions using that credential are terminated and subsequent queries return 401 with error=credential_expired", "Invoke the break-glass procedure on a test AI data asset → assert notifications are delivered to the governance officer and security ops distribution list within 5 minutes of invocation, confirmed by delivery receipt in the notification log", "Attempt to invoke break-glass access without supplying a mandatory emergency justification code → assert the request is rejected with error=justification_required before any credential is issued" ], "human_review": [ "Review post-access audit reports for the prior quarter to verify every break-glass invocation was followed by a completed review within 48 hours, and assess whether invocation frequency indicates the normal access provisioning process is inadequately slow", "Evaluate credential scope configurations to confirm emergency credentials are scoped to the minimum specific data assets required for the declared emergency type, not broad dataset or schema-level access", "Assess the break-glass approval workflow to confirm a second named authoriser distinct from the invoker is required for every invocation, with no self-authorised emergency access permitted" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Issuing break-glass credentials with indefinite expiry or manual-only revocation requirements, so emergency access persists long after the incident is resolved due to administrative oversight", "Scoping break-glass credentials to entire data lakes or schema-level permissions rather than to the specific table, partition, or dataset required for the declared emergency type", "Permitting individual engineers to self-approve break-glass invocations without a second authoriser, eliminating the separation of duties that detects abuse", "Treating the 48-hour post-access review as optional or low-priority so reviews accumulate as a backlog and are eventually completed in bulk weeks later, losing investigative value and violating the documented SLA", "Using break-glass procedures so frequently that they become a routine data access path, effectively bypassing normal access provisioning because the emergency channel is faster" ], "update_status": "current", "layer_code": "DA" }, { "id": "DA-08", "layer": "DA", "plane": "control", "name": "Data Access Evidence Package", "plain": "At each governance checkpoint a data access evidence package must be compiled from DA-01 through DA-07 artifacts to demonstrate that AI data access is authorized, minimized, consent-linked, privileged-access-monitored, recertified, jurisdiction-compliant, and properly governed for emergency scenarios; the package is the primary DA-layer input to the DataGovernanceAttestation (DV-08) compilation.", "threat": { "tags": [ "evidence-gap", "compliance-assertion-failure", "audit-readiness-deficit", "fragmented-control-attestation" ], "desc": "Even when individual data access controls are operative, the absence of a compiled evidence package means no single artifact demonstrates the combined assurance posture across the DA layer. Regulators, auditors, and downstream AI governance consumers cannot make an evidence-based compliance determination from disparate control outputs. Gaps discovered during a regulatory audit after an incident are significantly more costly than gaps identified and remediated during routine evidence package compilation." }, "standard": [ { "id": "eu_ai_act", "section": "Art. 11, Art. 17", "title": "Technical documentation and quality management obligations" }, { "id": "gdpr", "section": "Art. 5(2)", "title": "Accountability principle — demonstrating compliance" }, { "id": "dama_dmbok", "section": "Ch. 7", "title": "Data security audit and assurance" }, { "id": "cobit_2019", "section": "MEA02", "title": "Monitor, evaluate, and assess the system of internal control" } ], "sources": [ { "id": "eu_ai_act", "title": "EU AI Act", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2024/1689", "published_on": "2024-07-12", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2024/1689/oj/eng", "license": "open-access", "status": "current", "flagship": false, "source_id": "eu_ai_act", "relationship": "normative_requirement", "rationale": "Establishes EU AI Act Art. 11, Art. 17 requirements informing the apeiris://data/controls/DA-08 Data Access Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "gdpr", "title": "GDPR", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2016/679", "published_on": "2016-04-27", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj", "license": "open-access", "status": "current", "flagship": false, "source_id": "gdpr", "relationship": "normative_requirement", "rationale": "Establishes GDPR Art. 5(2) requirements informing the apeiris://data/controls/DA-08 Data Access Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "dama_dmbok", "title": "DAMA-DMBOK 2nd Ed", "authority": "DAMA International", "source_type": "standard", "normative_force": "best-practice", "version": "2 (2nd Edition, Revised 2024)", "published_on": "2017-07-05", "retrieved_on": "2026-07-02", "canonical_url": "https://www.dama.org/cpages/body-of-knowledge", "license": "proprietary", "status": "current", "flagship": false, "source_id": "dama_dmbok", "relationship": "informative_reference", "rationale": "Establishes DAMA-DMBOK 2nd Ed Ch. 7 requirements informing the apeiris://data/controls/DA-08 Data Access Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "cobit_2019", "title": "COBIT 2019", "authority": "ISACA", "source_type": "standard", "normative_force": "industry-framework", "version": "2019", "published_on": "2018-11-12", "retrieved_on": "2026-07-02", "canonical_url": "https://www.isaca.org/resources/cobit", "license": "proprietary", "status": "current", "flagship": false, "source_id": "cobit_2019", "relationship": "informative_reference", "rationale": "Establishes COBIT 2019 MEA02 requirements informing the apeiris://data/controls/DA-08 Data Access Evidence Package control.", "reviewed_on": "2026-07-02" }, { "id": "nist_pf", "title": "NIST Privacy Framework 1.0", "authority": "National Institute of Standards and Technology", "source_type": "standard", "normative_force": "voluntary-standard", "version": "1.0", "published_on": "2020-01-16", "retrieved_on": "2026-07-02", "canonical_url": "https://www.nist.gov/privacy-framework", "license": "open-access", "status": "current", "flagship": false, "source_id": "nist_pf", "relationship": "implementation_pattern", "rationale": "Establishes NIST Privacy Framework 1.0 GV.MT-P1 requirements informing the apeiris://data/controls/DA-08 Data Access Evidence Package control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Define a DA evidence package schema referencing required artifacts from DA-01 through DA-07; build an automated evidence collector on a weekly cadence; produce a quarterly DA-layer attestation record summarising control status, coverage metrics, and open findings with remediation owners; link the package to the Apeiris DataGovernanceAttestation (DV-08) compilation.", "steps": [ "Define the DA evidence package schema listing for each of DA-01 through DA-07: the required artifact type, responsible control, collection method (automated or manual), freshness SLA, and completeness threshold.", "Build an automated evidence collector that queries each DA control's output on a weekly cadence: authorization framework coverage report (DA-01), necessity justification register (DA-02), consent propagation SLA report (DA-03), privileged access anomaly summary (DA-04), recertification completion report (DA-05), cross-border compliance report (DA-06), and break-glass audit register (DA-07).", "Produce a DA-layer evidence summary record aggregating control status (compliant / finding / not-assessed) for each of DA-01 through DA-07, coverage metrics, open findings with named remediation owners and target dates, and an overall DA-layer assurance verdict.", "Store the compiled evidence package in the organisation's evidence management system with a tamper-evident hash; link it to the relevant AI system's technical documentation required under EU AI Act Art. 11.", "Deliver the DA evidence package to the DV-08 DataGovernanceAttestation compilation process and to relevant GRC workflows on a quarterly cadence or on demand for regulatory inquiries." ], "data_governance_officer": { "summary": "The evidence package is the primary accountability artifact for the DA layer. Own the package schema, approve the collection cadence, and sign off on the DA-layer summary record before it is submitted to DV-08.", "actions": [ "Approve the DA evidence package schema and review it annually or when any DA control changes substantially.", "Review the quarterly DA-layer evidence summary record; sign off on the overall DA-layer assurance verdict and ensure all open findings have named owners and documented target remediation dates before submission to DV-08.", "Track evidence package completeness as a governance KRI; escalate when any required artifact is missing or stale beyond its freshness SLA." ], "failure_signals": [ "DA evidence package not compiled on the defined quarterly cadence.", "Required artifacts from any DA control missing from the compiled package.", "DA-layer summary record submitted to DV-08 without data governance officer sign-off." ] }, "data_engineer": { "summary": "Build and maintain the automated evidence collector so DA-layer artifacts are gathered reliably in machine-readable form compatible with the Apeiris evidence schema.", "actions": [ "Implement the automated evidence collector as a scheduled job that queries each DA control's output system and compiles artifacts into the defined package schema in JSON format.", "Ensure all evidence artifacts are output in machine-readable format compatible with the Apeiris evidence schema (evidence_id, verdict, confidence, collected_at, valid_until fields required at minimum).", "Instrument the evidence collector with health monitoring that alerts the data governance officer when any artifact collection job fails or produces a stale result." ], "failure_signals": [ "Evidence collector jobs failing without generating alerts to the data governance officer.", "Artifacts produced in non-machine-readable formats incompatible with the Apeiris evidence schema.", "Evidence package not generated within 48 hours of the scheduled compilation date." ] }, "legal_counsel": { "summary": "The evidence package is the primary document demonstrating accountability under GDPR Art. 5(2) and satisfying technical documentation requirements under EU AI Act Art. 11. Confirm it meets regulatory documentation obligations.", "actions": [ "Review the DA evidence package structure to confirm it captures the information required to demonstrate GDPR accountability and EU AI Act Art. 11 technical documentation requirements.", "Confirm the evidence package is retained for the period required by applicable law and is accessible for regulatory inspection on request with no more than 24-hour retrieval time.", "Advise on whether the DA evidence package structure should be adapted when the organisation is subject to sector-specific data governance regulations beyond GDPR." ], "failure_signals": [ "DA evidence package structure not reviewed by legal counsel against regulatory documentation requirements in the prior 12 months.", "Evidence package not retained for the legally required period.", "Package not retrievable for regulatory inspection within the required timeframe." ] }, "grc_auditor": { "summary": "Use the DA evidence package as the primary audit artifact for the data access layer. Verify completeness, artifact freshness, and the accuracy of the DA-layer assurance verdict.", "actions": [ "Request the current DA evidence package and verify each required artifact from DA-01 through DA-07 is present and within its freshness SLA.", "Cross-reference the DA-layer assurance verdict against sampled source artifacts from individual controls to confirm the verdict accurately reflects the underlying control status.", "Review the open findings register within the package and confirm each finding has a documented remediation owner and a target date within an acceptable remediation window." ], "metrics": [ "DA evidence package completeness: 100% of required artifacts present and within freshness SLA at each quarterly compilation.", "Assurance verdict accuracy: zero discrepancies between the verdict and sampled source artifacts.", "Open findings remediation tracking: 100% of open findings with a named owner and documented target remediation date." ], "failure_signals": [ "Package completeness below 100% at any quarterly compilation.", "Discrepancies found between the assurance verdict and sampled source artifacts from individual controls.", "Open findings without a named remediation owner or documented target date." ] }, "it_operations": { "summary": "Operate the evidence management system with tamper-evident storage and ensure packages are retrievable on demand with verified integrity.", "actions": [ "Maintain the evidence management system at 99.9% availability and ensure each compiled package is stored with a tamper-evident hash recorded at time of write.", "Implement automated retrieval and hash-verification tests monthly to confirm stored evidence packages can be retrieved and verified within 30 minutes.", "Ensure the evidence management system is backed up with a recovery time objective of 4 hours and a recovery point objective of 24 hours." ], "failure_signals": [ "Evidence management system availability falling below 99.9% in any calendar month.", "Monthly retrieval test failing to complete within 30 minutes.", "Backup recovery time or recovery point objectives not tested or not met in the most recent test." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Most organisations lack a compiled DA-layer evidence package; compliance assertions are typically made from informal control outputs without a structured compilation, assurance verdict, or attestation workflow." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "eu-high-risk-ai", "high-risk-sector", "federated-enterprise" ], "implementers": [ "Data Governance Team", "GRC / Audit", "Data Engineering" ], "frameworks": [ { "framework": "eu_ai_act", "requirement_id": "Art. 11, Art. 17", "fit": "direct", "rationale": "EU AI Act Articles 11 and 17 require high-risk AI providers to maintain technical documentation and a quality management system covering data governance. DA-08 directly satisfies these requirements by compiling and maintaining a structured evidence package that documents the data access control posture and serves as a component of the required technical documentation.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "gdpr", "requirement_id": "Art. 5(2)", "fit": "direct", "rationale": "GDPR Article 5(2) establishes the accountability principle: the controller must be able to demonstrate compliance with Article 5(1). DA-08's evidence package is the primary accountability artifact for data access controls, providing a compiled, auditable record of control status across the DA layer that is retained and accessible for regulatory inspection.", "normative_force": "binding-law", "source_version": "2016/679", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 7", "fit": "direct", "rationale": "DAMA DMBOK2 Chapter 7 includes data security audit and assurance as a core data management practice. DA-08 operationalises the assurance dimension of data security management by compiling control outputs into a structured evidence package with a layer-level assurance verdict and open findings register.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "cobit_2019", "requirement_id": "MEA02", "fit": "direct", "rationale": "COBIT 2019 MEA02 (Monitor, Evaluate and Assess the System of Internal Control) requires organisations to compile and review evidence of internal control effectiveness. DA-08's quarterly evidence package compilation and DA-layer assurance verdict directly implement the monitoring and assessment activities prescribed in MEA02.", "normative_force": "industry-framework", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "nist_pf", "requirement_id": "GV.MT-P1", "fit": "partial", "rationale": "NIST Privacy Framework GV.MT-P1 requires privacy risk to be re-evaluated on an ongoing basis. The data access evidence package is the reviewable artifact stream that makes ongoing re-evaluation of data-access risk possible and auditable.", "normative_force": "voluntary-standard", "source_version": "1.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://data/controls/DA-08", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "At each governance checkpoint, a complete DA-layer evidence package must be compiled from DA-01 through DA-07 control artifacts with documented status (pass/fail/conditional/not-applicable-with-waiver) for every control — with no control presenting 'not available' without a documented waiver — and the package must be accepted as valid input by the DV-08 DataGovernanceAttestation compilation process.", "evidence_required": [ "data_access_evidence_package artifact with documented status for each of DA-01 through DA-07, compilation timestamp, package version, and named compiler identity", "evidence_artifact_manifest listing the specific artifact file references for each DA-layer control with SHA-256 content hashes to enable integrity verification", "governance_checkpoint_acceptance_record showing the evidence package was reviewed and accepted at the designated checkpoint, with reviewer identity and acceptance timestamp", "dv_08_package_ingestion_log confirming the DA evidence package was successfully ingested as input to the DataGovernanceAttestation compilation with package_id and attestation_status=accepted" ], "machine_tests": [ "Submit a DA evidence package with the DA-03 status field absent → assert DV-08 compilation rejects the package with error=incomplete_control_coverage and names the missing control in the error response", "Submit a DA evidence package where a referenced control artifact SHA-256 hash does not match the file content → assert integrity check fails with error=artifact_hash_mismatch before the package is accepted", "Generate a DA evidence package with all 7 DA controls at pass status and valid artifact hashes → assert DV-08 ingestion log records successful acceptance with package_id, compilation_timestamp, and attestation_status=accepted" ], "human_review": [ "Review DA controls with 'conditional' or 'not-applicable-with-waiver' status in the evidence package to confirm each waiver has a named approver, a residual risk statement, and an expiry date", "Assess the evidence artifact manifest for freshness: confirm no contributing artifact was produced more than 90 days before the governance checkpoint date, ensuring the package reflects current control state rather than stale evidence", "Verify the evidence package compilation process is documented and repeatable, with a named owner responsible for each DA-layer artifact and an independent reviewer who validates the package before submission to DV-08" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Compiling the DA evidence package only at formal audit time rather than at each governance checkpoint, producing a point-in-time snapshot that does not reflect the continuous governance posture", "Including narrative descriptions of control operations in the evidence package rather than references to specific verifiable artifacts with content hashes, making compliance assertions unverifiable", "Marking DA controls as 'pass' in the evidence package without artifact references, asserting compliance without any supporting evidence that can be independently validated", "Allowing a single person to both operate DA controls and compile and sign off the evidence package without independent review, eliminating the oversight function of the attestation process", "Treating the evidence package as a one-time deliverable that is not refreshed when control artifacts expire or control configurations change, causing the package to misrepresent the current control state" ], "update_status": "current", "layer_code": "DA" }, { "id": "DV-01", "layer": "DV", "plane": "control", "name": "Schema Validation for AI Inputs", "plain": "Every data payload ingested at an AI system boundary must be validated against a versioned schema that enforces expected field names, data types, value ranges, and structural constraints before processing begins.", "threat": { "tags": [ "schema-bypass", "malformed-input", "type-confusion", "injection-via-data" ], "desc": "Unvalidated inputs allow malformed or adversarially crafted data to reach AI inference and training pipelines. Type confusion and range violations can corrupt model behavior, cause silent processing errors, or enable prompt-injection-style attacks embedded in structured data fields. Without schema enforcement at the ingestion boundary, downstream quality guarantees collapse and root-cause analysis becomes intractable." }, "standard": [ { "id": "eu_ai_act", "section": "Art. 10(3)", "title": "Data quality criteria for high-risk AI system inputs" }, { "id": "dama_dmbok", "section": "Ch. 13", "title": "Data Quality — completeness and validity dimensions" }, { "id": "iso_8000", "section": "ISO 8000-8", "title": "Syntactic data quality — concepts and measuring" }, { "id": "microsoft_purview", "section": "Data Quality (data type and format rules)", "title": "Data type and format rule evaluation over governed assets" } ], "sources": [ { "id": "databricks_unity_catalog_2024", "title": "Databricks Unity Catalog", "authority": "Databricks, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.databricks.com/en/data-governance/unity-catalog/index.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "databricks_unity_catalog_2024", "relationship": "informative_reference", "rationale": "Establishes Databricks Unity Catalog requirements informing the apeiris://data/controls/DV-01 Schema Validation for AI Inputs control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_purview_data_governance_2024", "title": "Microsoft Purview (Data Governance)", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/purview/purview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_purview_data_governance_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Purview (Data Governance) requirements informing the apeiris://data/controls/DV-01 Schema Validation for AI Inputs control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "A schema registry stores versioned JSON Schema, Avro, or Protobuf definitions keyed by pipeline ID and schema version. A validation sidecar or gateway intercepts all AI ingestion requests, deserializes the payload, validates it against the registered schema version, and returns structured rejection messages on failure. Invalid payloads are quarantined before reaching the model or feature store.", "steps": [ "Establish a schema registry (e.g., Confluent Schema Registry, AWS Glue Schema Registry, or Apicurio) with version control, compatibility enforcement, and an approval workflow for all AI input schemas.", "Deploy schema validation at every AI ingestion boundary — REST endpoints, Kafka topics, batch file ingestion, and streaming pipelines — as a mandatory pre-processing step before data reaches the AI model or feature store.", "Define rejection behavior: invalid payloads must be quarantined, logged with structured schema violation details (field name, expected type, received value), and never silently dropped or forwarded.", "Integrate schema version pinning into CI/CD so pipeline deployments are blocked without a registered and approved schema, and schema changes require backward-compatibility checks before promotion." ], "data_governance_officer": { "summary": "Schema validation is the first line of defense ensuring AI systems only process data that meets the organization's structural and quality standards. The schema registry is the authoritative record of what inputs AI systems are permitted to accept.", "actions": [ "Maintain the schema registry policy specifying the approval workflow for schema creation and changes.", "Review schema validation coverage metrics monthly to confirm all AI ingestion points are registered and protected.", "Ensure schema definitions are versioned and linked to the relevant data asset catalog entries." ], "failure_signals": [ "Any AI ingestion pathway operating without a registered and approved schema.", "Schema registry containing unreviewed schema versions outstanding for more than 30 days.", "Validation rejection logs showing recurring same-field failures not remediated within SLA." ] }, "data_engineer": { "summary": "Schema validation must be implemented as a first-class pipeline component at every AI ingestion boundary, with rejection events wired to the quarantine store and alert pipeline.", "actions": [ "Deploy schema validation middleware or sidecar for all AI ingestion endpoints, integrated with the central schema registry.", "Wire validation rejection events to the quarantine store and monitoring alert pipeline with structured failure metadata.", "Automate schema compatibility checks in the CI/CD pipeline for any schema change prior to deployment." ], "failure_signals": [ "Validation sidecar unavailable without automatic rejection fallback active.", "Schema registry not reachable causing pipelines to bypass validation silently.", "Recurring schema mismatches between producer and consumer not resolved within 48 hours." ] }, "grc_auditor": { "summary": "Schema validation evidence demonstrates that AI systems are protected from structurally non-conformant data at the ingestion boundary, directly satisfying EU AI Act Art. 10 data quality requirements for high-risk systems.", "actions": [ "Request schema registry exports and confirm all active AI pipelines reference a registered, approved schema version.", "Review validation rejection logs for the audit period and verify quarantine disposition records exist for all rejected payloads.", "Confirm schema change approval records exist for any schema modifications deployed during the audit window." ], "metrics": [ "Schema coverage rate: target 100% of AI ingestion endpoints with a registered schema.", "Validation rejection quarantine compliance: target 100% of rejected payloads captured in quarantine store.", "Schema change approval compliance: target 100% of changes approved before deployment." ], "failure_signals": [ "Any active AI pipeline with no associated schema registration.", "Validation rejections not quarantined or structured logs absent.", "Schema changes deployed without documented approval records." ] }, "legal_counsel": { "summary": "Malformed inputs are a quiet legal risk: they corrupt the data that decisions and representations rest on. Counsel's stake is that inputs to consequential AI systems are validated before they can produce legally significant errors.", "actions": [ "Identify AI systems whose outputs carry legal consequence and confirm their inputs pass schema validation.", "Require validation-failure records to be retained where they may bear on disputed decisions.", "Advise on duties arising when invalid data is found to have influenced past decisions." ], "failure_signals": [ "Consequential decisions traced to inputs that bypassed validation.", "Validation logs unavailable for the period of a disputed decision.", "Known-invalid data reprocessed without assessing affected decisions." ] }, "it_operations": { "summary": "Schema validation sits inline in ingestion paths, so operations owns its performance, availability and fail-closed behavior. A validator outage must not become a validation bypass.", "actions": [ "Operate schema validation as a highly available inline service with defined latency budgets.", "Configure fail-closed or quarantine behavior on validator outage — never silent pass-through.", "Track validation coverage as pipelines are added or changed." ], "failure_signals": [ "Pipelines routing around the validator during incidents.", "Validator outages defaulting to pass-through.", "New ingestion paths deployed without validation wiring." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most enterprises apply ad-hoc input validation rather than formal schema registry governance for AI pipelines; schema registries are common for event streaming but rarely extended to all AI ingestion boundaries." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "eu-high-risk-ai" ], "implementers": [ "Data Engineering", "Platform Engineering", "MLOps" ], "frameworks": [ { "framework": "eu_ai_act", "requirement_id": "Art. 10(3)", "fit": "direct", "rationale": "EU AI Act Article 10(3) requires that training, validation, and testing data for high-risk AI systems meet quality criteria including freedom from errors and completeness. Schema validation is the foundational technical mechanism for ensuring structural correctness at the ingestion boundary. Without schema enforcement, downstream data quality claims for high-risk AI cannot be substantiated.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 13 — Validity", "fit": "direct", "rationale": "DAMA-DMBOK2 identifies validity as a core data quality dimension, requiring data values to comply with defined formats, domains, and structural rules. Schema validation is the primary technical implementation of validity enforcement at data movement boundaries. DMBOK2 further recommends automated validation gates rather than manual review as the enforcement mechanism for this dimension.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "iso_8000", "requirement_id": "ISO 8000-8", "fit": "direct", "rationale": "ISO 8000-8 defines syntactic quality — conformance of data to its specified syntax and schema — as a measurable quality dimension. Schema validation for AI inputs is the runtime enforcement of syntactic quality.", "normative_force": "certification-standard", "source_version": "2015", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "microsoft_purview", "requirement_id": "Data Quality (data type and format rules)", "fit": "direct", "rationale": "Microsoft Purview Data Quality provides rule types such as data type match and format match that check column-level conformance of governed assets. These rule types implement schema-adjacent validation, though Purview DQ operates as scheduled scans over governed assets rather than as an inline gate.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "databricks_unity", "requirement_id": "Lakeflow Declarative Pipelines (DLT) expectations", "fit": "partial", "rationale": "Expectations in Lakeflow Declarative Pipelines (formerly Delta Live Tables) declare row-level quality constraints with warn, drop or fail actions, providing inline schema and quality enforcement inside Databricks pipelines rather than in Unity Catalog itself.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "owasp_ai_exchange", "requirement_id": "dataqualitycontrol", "fit": "supporting", "rationale": "DV-01 rejects payloads with unexpected fields, type mismatches, or out-of-range values at the AI ingestion boundary, enforcing structural training/inference-data quality.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "mitre", "requirement_id": "AML.M0007", "fit": "supporting", "direction": "control-supports-requirement", "rationale": "Control \"Every data payload presented at an AI system ingestion boundary must pass syntactic…\" enacts ATLAS mitigation AML.M0007 Sanitize Training Data; OpenCRE crosswalks this control’s OWASP AI Exchange concept (dataqualitycontrol) to this mitigation.", "source_version": "v2026.06", "reviewed_on": "2026-07-08", "source_status": "authoritative", "mapping_confidence": "medium", "legal_status": "not-applicable", "control_readiness": "approved", "implementation_maturity": "developing", "assurance_result": "not-assessed", "evidence_status": "no-evidence", "normative_force": "industry-framework", "basis": "asserted", "relation": "equivalent_to", "correction": "ap38-opencre-enhancement 2026-07-08" } ], "canonical_id": "apeiris://data/controls/DV-01", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every data payload presented at an AI system ingestion boundary must pass syntactic schema validation against the registered versioned schema for that endpoint before any processing begins — with no payload entering training or inference pipelines if it contains unexpected field names, type mismatches, out-of-range values, or structural violations — and each validation result must be logged with the schema version applied and the outcome.", "evidence_required": [ "schema_registry_export listing each AI ingestion endpoint's registered versioned schema with schema_id, version, effective_date, field names, declared types, and value range constraints", "schema_validation_execution_log showing each inbound payload's validation result (pass/fail), schema_version applied, validation timestamp, and failure reason with field name for a representative 30-day period", "schema_rejection_disposition_log documenting how each failed payload was handled (quarantined, returned to sender with error code, escalated for manual review) with disposition timestamp", "schema_version_change_record documenting each schema update with change description, approver, effective date, and handling of in-flight payloads during the version transition" ], "machine_tests": [ "Submit a payload with a required field absent to an AI ingestion endpoint → assert validation fails with error=missing_required_field and the payload does not appear in the downstream processing pipeline", "Submit a payload with a string value in a field declared as integer with range 0-100 → assert validation fails with error=type_mismatch, logged with schema_version, field_name, and received_type, and payload is blocked", "Submit a structurally valid payload with all required fields present and within declared ranges → assert validation passes, payload proceeds to downstream processing, and a validation_result=pass log entry is written", "Submit a payload referencing a deprecated schema version → assert rejection with error=schema_version_deprecated specifying the current required version before any processing occurs" ], "human_review": [ "Review the schema registry for completeness: verify every AI ingestion endpoint is registered with a current versioned schema and no endpoints accept payloads without schema validation", "Assess schema constraint adequacy for model-critical fields: review type, range, and pattern constraints for fields that feed directly into model decision logic to confirm they are specific enough to block adversarial manipulation", "Evaluate schema rejection handling to ensure quarantined payloads are investigated rather than silently dropped, and that rejection volume anomalies are monitored as an upstream data quality signal" ], "blocking_effect": "blocks-runtime-action", "normative_status": "binding-law", "anti_patterns": [ "Implementing schema validation only at the application or middleware layer without enforcement at the ingestion boundary, allowing payloads to reach AI pipelines if the validation layer is bypassed or misconfigured", "Using a single unversioned schema across all AI ingestion endpoints so schema changes propagate instantly and break validation consistency for in-flight batch jobs without a transition window", "Defining schemas with overly permissive constraints (any string, any integer) for model-critical fields, providing no effective protection against adversarially crafted inputs that remain syntactically valid", "Logging only validation failures without logging successful validations, preventing detection of anomalous patterns in the validated payload stream such as sudden distribution shifts", "Treating schema definitions as static configuration that is never reviewed when the AI model's expected input specification changes, causing schema drift where the registered schema no longer matches actual model input requirements" ], "update_status": "current", "layer_code": "DV" }, { "id": "DV-02", "layer": "DV", "plane": "control", "name": "Data Quality Gate Enforcement", "plain": "Data that fails defined quality thresholds for completeness, accuracy, or consistency must be blocked from entering AI training and inference pipelines, with failed batches routed to a quarantine process that tracks disposition through resolution.", "threat": { "tags": [ "low-quality-training-data", "pipeline-contamination", "garbage-in-garbage-out", "silent-quality-degradation" ], "desc": "Training and inference pipelines that accept low-quality data produce AI outputs with degraded reliability that is difficult to distinguish from normal variance. Incomplete records corrupt feature engineering. Inaccurate reference data poisons model training in ways that manifest as subtle behavioral errors rather than system failures. Without hard quality gates, data quality problems propagate invisibly into production AI behavior and are attributed to model problems rather than data problems." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 13", "title": "Data Quality — completeness, accuracy, and consistency dimensions" }, { "id": "eu_ai_act", "section": "Art. 10(2)", "title": "Data governance practices for high-risk AI systems" }, { "id": "iso_8000", "section": "ISO 8000-8", "title": "Semantic data quality — concepts and measuring" }, { "id": "dcam", "section": "Capability 5.1", "title": "The data quality management program is established" } ], "sources": [ { "id": "google_dataplex_bigquery_2024", "title": "Google Cloud Dataplex & BigQuery", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://cloud.google.com/dataplex/docs/introduction", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "google_dataplex_bigquery_2024", "relationship": "informative_reference", "rationale": "Establishes Google Cloud Dataplex & BigQuery requirements informing the apeiris://data/controls/DV-02 Data Quality Gate Enforcement control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "Quality gate rule sets are defined per dataset and pipeline stage, specifying minimum thresholds for completeness, uniqueness, validity, accuracy, and consistency. A quality gate engine evaluates each data batch against these rules before promotion to the next pipeline stage. Failed batches are quarantined and routed to a remediation workflow rather than being advanced or silently dropped.", "steps": [ "Define quality gate rule sets for each AI training dataset and inference input, specifying threshold values for completeness (e.g., ≥95% non-null for required fields), uniqueness, format validity, and referential integrity, with thresholds justified by the business risk profile of the AI use case.", "Integrate quality gate evaluation into the pipeline orchestration layer (e.g., Apache Airflow sensors, Databricks DLT expectations, dbt tests) as a mandatory pre-promotion step that blocks data advancement on rule failure.", "Route all quality gate failures to a quarantine store with structured failure metadata including the rule violated, affected record count, dataset version, batch ID, and timestamp, ensuring no failed batch is silently advanced.", "Establish a remediation SLA for quarantined data and surface quality gate pass rates and quarantine volumes in the data governance dashboard for trend monitoring." ], "data_governance_officer": { "summary": "Quality gates are the enforcement mechanism for the organization's data quality policy, ensuring AI pipelines only consume data that meets defined fitness-for-purpose standards. The gate configuration represents a documented policy decision about acceptable data quality for each AI use case.", "actions": [ "Approve quality gate rule sets for each AI pipeline, ensuring thresholds align with the business risk profile of the AI use case and regulatory requirements.", "Review quarantine disposition reports monthly to track remediation SLA compliance and identify systemic quality issues.", "Ensure quality gate definitions are versioned, linked to dataset contracts in the data catalog, and reviewed following model retraining events." ], "failure_signals": [ "AI pipeline advancing data without a quality gate rule set defined and approved.", "Quarantine store growing without remediation actions being taken within the defined SLA.", "Quality gate thresholds set permissively (e.g., ≥0% completeness) for required fields without documented justification." ] }, "data_engineer": { "summary": "Quality gates must be built as first-class pipeline components that treat quality failures as pipeline failures, not advisory warnings, with quarantine routing and structured failure metadata.", "actions": [ "Implement quality gate checks using the organization's approved DQ framework at every pipeline promotion point, configured to block rather than warn on threshold violation.", "Configure quarantine routing for failed batches with structured failure metadata and wire quarantine events to the exception register.", "Automate quality gate rule deployment and version control alongside pipeline code so rule changes are tracked in source control." ], "failure_signals": [ "Quality gate checks configured in warn-only mode for critical AI pipelines.", "Quarantine records not populated or purged without remediation actions.", "Quality gate rule sets diverging from approved policy without change management records." ] }, "grc_auditor": { "summary": "Quality gate evidence demonstrates that AI systems are technically prevented from consuming low-quality data, satisfying EU AI Act Art. 10 data governance requirements and DAMA DMBOK2 quality management practices.", "actions": [ "Request quality gate rule set exports for all active AI pipelines and verify thresholds are not trivially permissive for fields identified as critical in the data quality policy.", "Review quarantine records for the audit period and confirm remediation was completed within the defined SLA for each batch.", "Sample pipeline execution logs to verify that quality gates were active and enforced in blocking mode during the audit window." ], "metrics": [ "Quality gate coverage: target 100% of AI training and inference pipelines with an active gate configuration.", "Quarantine remediation SLA compliance: target ≥95% of quarantined batches resolved within the defined SLA.", "Quality gate pass rate trend: track 30-day rolling average per pipeline to detect gradual degradation." ], "failure_signals": [ "Any AI pipeline lacking an active blocking quality gate configuration.", "Quarantine records without remediation disposition older than the defined SLA.", "Quality gate pass rate declining more than 5 percentage points over 30 days without documented investigation." ] }, "legal_counsel": { "summary": "Quality gate enforcement provides documented evidence that the organization applies appropriate data governance practices to AI systems processing personal data, supporting defensibility under EU AI Act Art. 10 and GDPR accuracy requirements.", "actions": [ "Confirm that quality gate policies address GDPR Article 5(1)(d) accuracy requirements for personal data used in AI systems.", "Review quarantine and remediation records to ensure data subjects' rights are not impaired by quality failures involving personal data.", "Verify that quality gate rule sets for high-risk AI use cases are documented at a level sufficient for production in regulatory proceedings." ], "failure_signals": [ "Personal data quarantined without privacy team notification procedures in place.", "Quality gate documentation insufficient to demonstrate Art. 10 compliance to a regulatory body." ] }, "it_operations": { "summary": "Quality gates are enforcement points operations must keep both effective and available: gate failures should stop bad data, and gate infrastructure failures should not stop good data silently.", "actions": [ "Operate quality-gate execution with monitoring on both gate verdicts and gate health.", "Implement quarantine flows for gated data with capacity and aging alerts.", "Ensure gate configuration changes deploy through change control with rollback." ], "failure_signals": [ "Gates disabled 'temporarily' under load and never re-enabled.", "Quarantine stores overflowing or aging without triage.", "Gate configuration drift between environments." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Many enterprises run DQ checks as advisory warnings rather than hard blocking gates; converting to blocking enforcement requires pipeline redesign and stakeholder alignment on acceptable failure modes." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "cloud-native", "eu-high-risk-ai" ], "implementers": [ "Data Engineering", "Data Governance Office", "MLOps" ], "frameworks": [ { "framework": "eu_ai_act", "requirement_id": "Art. 10(3)", "fit": "direct", "rationale": "EU AI Act Article 10(2)(f) requires bias examination of training data, and Article 10(3) sets the dataset quality criteria (relevant, sufficiently representative, and to the best extent possible free of errors and complete). Quality gate enforcement blocks datasets that fail these criteria from reaching training or inference — the enforcement mechanism the Art. 10(2) governance practices presuppose.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-07-08", "basis": "anchored", "correction": "ai-exchange-verify 2026-07-08", "relation": "satisfies" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 13 — Data Quality Management", "fit": "direct", "rationale": "DAMA DMBOK2 Chapter 13 defines data quality management as a discipline requiring defined rules, automated measurement, and remediation workflows applied at data movement boundaries. Quality gates are the enforcement realization of DMBOK2's quality rule and threshold concepts. DMBOK2 explicitly recommends gates at data movement boundaries to prevent quality degradation from propagating across systems.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "iso_8000", "requirement_id": "ISO 8000-8", "fit": "direct", "rationale": "ISO 8000-8 defines semantic quality — the correspondence of data values to the real-world entities they describe — alongside syntactic quality. Quality gate enforcement extends validation beyond syntax to these semantic dimensions before data reaches AI systems.", "normative_force": "certification-standard", "source_version": "2015", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "dcam", "requirement_id": "Capability 5.1", "fit": "direct", "rationale": "DCAM v2.2 capability 5.1 requires the data quality management program to be established with defined standards and thresholds. Quality gate enforcement is the program's enforcement arm: the defined standards become blocking checks at pipeline boundaries.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "google_dataplex", "requirement_id": "Dataplex Data Quality — Quality Rules", "fit": "partial", "rationale": "Google Cloud Dataplex provides data quality scanning with rule-based validation and result publishing to BigQuery for monitoring and reporting. Dataplex quality rules partially implement quality gate requirements but do not natively block pipeline execution; additional orchestration (e.g., Cloud Composer conditions or Dataform assertions) is required to create hard blocking gates. Dataplex scan results provide durable evidence artifacts for attestation reporting.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "owasp_ai_exchange", "requirement_id": "dataqualitycontrol", "fit": "direct", "rationale": "DV-02 runs automated completeness/accuracy/consistency quality gates on every ingested batch and quarantines failures before the pipeline proceeds, directly controlling data quality.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "mitre", "requirement_id": "AML.M0007", "fit": "supporting", "direction": "control-supports-requirement", "rationale": "Control \"Every data batch ingested into an AI training or inference pipeline must pass automated…\" enacts ATLAS mitigation AML.M0007 Sanitize Training Data; OpenCRE crosswalks this control’s OWASP AI Exchange concept (dataqualitycontrol) to this mitigation.", "source_version": "v2026.06", "reviewed_on": "2026-07-08", "source_status": "authoritative", "mapping_confidence": "medium", "legal_status": "not-applicable", "control_readiness": "approved", "implementation_maturity": "developing", "assurance_result": "not-assessed", "evidence_status": "no-evidence", "normative_force": "industry-framework", "basis": "asserted", "relation": "equivalent_to", "correction": "ap38-opencre-enhancement 2026-07-08" } ], "canonical_id": "apeiris://data/controls/DV-02", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every data batch ingested into an AI training or inference pipeline must pass automated quality gate checks for completeness, accuracy, and consistency against defined thresholds before processing proceeds — with any batch failing one or more quality rules blocked from the pipeline and routed to a quarantine process that tracks each failed batch through to documented resolution.", "evidence_required": [ "quality_rule_registry listing each defined quality rule per dataset and AI pipeline with rule_id, quality dimension (completeness/accuracy/consistency), threshold value, blocking_action, and last_reviewed_date", "quality_gate_execution_log showing each batch evaluation result (pass/fail), rules evaluated, threshold values applied, and failure details for a representative 30-day period", "quarantine_batch_register showing all batches blocked in the prior 90 days with quarantine_entry_timestamp, failure_reason, assigned_investigator, resolution_action, and resolution_timestamp", "quality_threshold_calibration_record documenting how thresholds were established, with reference to training data quality requirements in the model card or dataset specification" ], "machine_tests": [ "Submit a test batch with 40% null values in a field with a completeness threshold of 95% → assert the batch is blocked before entering the training pipeline and a quarantine_entry is created with rule_id, threshold, and observed_value", "Submit a test batch where 8% of records contain reference values absent from the approved lookup table against a consistency threshold of 99% → assert batch is rejected with failure logged including sample of inconsistent values", "Submit a batch passing all quality rules at defined thresholds → assert the batch enters the pipeline with quality_gate_result=pass logged and no quarantine record created" ], "human_review": [ "Review quality rule thresholds for adequacy: assess whether completeness, accuracy, and consistency thresholds are calibrated to the specific sensitivity and model-criticality of each dataset rather than applying uniform thresholds across all pipelines", "Audit the quarantine register resolution log to verify all quarantined batches have been resolved through documented actions (reprocessed with corrections, permanently rejected, escalated) rather than silently abandoned without disposition", "Evaluate whether quality gate rules cover the full dimensions of data quality relevant to the AI model: confirm rules address temporal freshness, referential consistency, and distribution drift relative to training baseline in addition to completeness and type accuracy" ], "blocking_effect": "blocks-runtime-action", "normative_status": "binding-law", "anti_patterns": [ "Configuring quality gates in warn mode rather than block mode so quality failures generate alerts but batches still enter the training pipeline, rendering the gate advisory-only and providing no actual protection", "Defining quality thresholds at coarse granularity (e.g., 50% completeness) that permit large volumes of missing data to pass through, providing no meaningful quality assurance for AI pipelines", "Routing failed batches to a quarantine directory with no ownership assignment so quarantined batches accumulate without investigation and are eventually deleted at storage cleanup without resolution", "Applying identical quality thresholds to both inference inputs and training data without recognizing that training data quality failures corrupt models for extended periods, requiring stricter training-phase thresholds", "Not versioning quality rules alongside model versions so a model trained on data that passed rules at version N is later evaluated against rules from version N+3 with different thresholds, breaking quality traceability" ], "update_status": "current", "layer_code": "DV" }, { "id": "DV-03", "layer": "DV", "plane": "data", "name": "Statistical Distribution Validation", "plain": "AI system inputs and training data must be continuously monitored for statistical distribution drift relative to established baselines, with automated alerts and pipeline holds triggered when drift exceeds defined thresholds.", "threat": { "tags": [ "data-drift", "distribution-shift", "covariate-shift", "model-degradation" ], "desc": "Statistical distribution shifts in input data silently degrade AI model performance without triggering schema or quality gate failures, because the data may be structurally valid and individually accurate while the aggregate population has shifted materially. Covariate shift — where the distribution of input features changes after deployment — causes models to make systematically incorrect predictions on the new data regime. Without distribution monitoring, degraded AI outputs reach production consumers before the problem is diagnosed." }, "standard": [ { "id": "eu_ai_act", "section": "Art. 9(6)-(8)", "title": "Testing against defined metrics, including real-world conditions" }, { "id": "dama_dmbok", "section": "Ch. 13", "title": "Data Quality — consistency and timeliness dimensions" }, { "id": "iso_8000", "section": "ISO 8000-8", "title": "Information and data quality — concepts and measuring" }, { "id": "dcam", "section": "Capability 5.2", "title": "Data quality is profiled and measured" } ], "sources": [ { "id": "google_dataplex_bigquery_2024", "title": "Google Cloud Dataplex & BigQuery", "authority": "Google LLC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://cloud.google.com/dataplex/docs/introduction", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "google_dataplex_bigquery_2024", "relationship": "informative_reference", "rationale": "Establishes Google Cloud Dataplex & BigQuery requirements informing the apeiris://data/controls/DV-03 Statistical Distribution Validation control.", "reviewed_on": "2026-07-01" }, { "id": "databricks_unity_catalog_2024", "title": "Databricks Unity Catalog", "authority": "Databricks, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.databricks.com/en/data-governance/unity-catalog/index.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "databricks_unity_catalog_2024", "relationship": "informative_reference", "rationale": "Establishes Databricks Unity Catalog requirements informing the apeiris://data/controls/DV-03 Statistical Distribution Validation control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "Statistical baseline profiles are computed from approved reference data windows for each AI feature set and registered in a versioned baseline registry. A distribution monitoring service evaluates each incoming data batch against the registered baseline using statistical divergence tests. Drift scores are published to the observability platform. Batches exceeding defined thresholds trigger alerts and optionally hold pipeline execution pending review.", "steps": [ "Compute statistical baseline profiles (mean, standard deviation, quantile distributions, feature histograms, null rates) from approved reference data windows for each AI feature set and store them in a versioned baseline registry linked to the model version they support.", "Deploy a distribution monitoring service that evaluates each incoming data batch against the registered baseline using Population Stability Index (PSI), KL divergence, or Wasserstein distance, and publishes drift scores with batch metadata to the observability platform.", "Define drift thresholds per feature and pipeline (e.g., PSI > 0.20 triggers warning alert, PSI > 0.25 triggers pipeline hold) and document the threshold rationale in the data quality policy for each AI use case.", "Establish a drift response procedure specifying acknowledgment SLA, root-cause analysis documentation requirements, and the process for resolving pipeline holds before resumption." ], "data_governance_officer": { "summary": "Distribution validation ensures the organization detects when AI input conditions have shifted materially from those under which the model was validated, enabling timely model revalidation or pipeline suspension before degraded outputs cause business harm.", "actions": [ "Approve drift threshold policies for each AI pipeline, aligned with the risk profile of the AI use case and the sensitivity of the downstream decision.", "Review drift alert and response records quarterly to confirm SLA compliance and root-cause documentation.", "Ensure baseline profiles are refreshed when models are retrained, when significant business changes occur, or at least annually." ], "failure_signals": [ "Any AI pipeline operating without a registered distribution baseline.", "Drift alerts not acknowledged within the defined SLA.", "Baseline profiles not refreshed within 12 months or following a model retraining event." ] }, "data_engineer": { "summary": "The statistical distribution monitoring layer must be an integral component of AI data pipelines, not a bolt-on afterthought, with baseline profiles version-controlled alongside model artifacts.", "actions": [ "Integrate distribution monitoring (e.g., Evidently AI, Whylogs, Deepchecks, Databricks Lakehouse Monitoring, or cloud-native equivalents) into all AI inference and training pipelines.", "Automate baseline profile computation and registry updates on model retraining events, ensuring baseline version is pinned to the model version it supports.", "Configure drift alerts to route to the data governance incident channel with structured metadata including drift score, affected features, and batch ID." ], "failure_signals": [ "Distribution monitoring service unavailable without fallback alerting to the governance team.", "Drift scores not published for more than one pipeline batch cycle without investigation.", "Baseline profiles computed from non-representative or stale reference data." ] }, "grc_auditor": { "summary": "Distribution validation evidence demonstrates continuous operational monitoring of AI input data conditions, satisfying EU AI Act Art. 9(5) risk management requirements for testing and monitoring in operational conditions.", "actions": [ "Request drift score histories for all active AI pipelines and verify continuous monitoring coverage during the audit period.", "Review drift alert records and confirm each alert was acknowledged and resolved within the defined SLA with documented root-cause analysis.", "Verify that baseline profiles are versioned, that changes are approved through change management, and that baseline version is linked to the corresponding model version." ], "metrics": [ "Distribution monitoring coverage: target 100% of production AI pipelines with an active baseline and monitoring configuration.", "Drift alert SLA compliance: target ≥95% of alerts acknowledged within the defined SLA.", "Baseline profile currency: target 100% refreshed within 12 months or following a model retraining event." ], "failure_signals": [ "Any production AI pipeline without an active distribution monitoring configuration.", "Drift alert acknowledgment SLA breaches exceeding 5%.", "Drift events without documented root-cause analysis records." ] }, "legal_counsel": { "summary": "Distribution drift can silently push an AI system outside the conditions under which it was assessed — including the data-representativeness assumptions in DPIAs and conformity documentation. Counsel needs drift on regulated systems surfaced, not buried.", "actions": [ "Require drift findings on high-risk AI systems to be reflected in updated risk and conformity documentation.", "Define drift severities that trigger legal review of continued operation.", "Use drift records to evidence ongoing-monitoring duties under the EU AI Act." ], "failure_signals": [ "High-risk systems operating under stale representativeness claims after material drift.", "Drift alerts on regulated systems closed with no documentation update.", "No linkage between drift monitoring and the risk-management file." ] }, "it_operations": { "summary": "Distribution validation is a continuously running statistical workload. Operations schedules profile computation, manages reference windows, and keeps drift alerting reliable across all monitored datasets.", "actions": [ "Schedule distribution profiling across all in-scope datasets with completion monitoring.", "Manage reference-baseline windows and their refresh policy as configuration under change control.", "Route drift alerts with severity-based escalation and track acknowledgment." ], "failure_signals": [ "Profiling jobs silently failing for a subset of datasets.", "Reference baselines stale beyond their refresh policy.", "Drift alerts accumulating unacknowledged." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Distribution monitoring is an emerging practice in enterprise AI governance; most organizations rely on lagging model performance metrics rather than upstream data distribution signals to detect data shift." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "cloud-native", "eu-high-risk-ai" ], "implementers": [ "Data Science", "MLOps", "Data Engineering" ], "frameworks": [ { "framework": "eu_ai_act", "requirement_id": "Art. 9(6)-(8)", "fit": "direct", "rationale": "EU AI Act Article 9(6)-(8) requires high-risk AI systems to be tested against prior defined metrics and probabilistic thresholds — including, where appropriate, testing in real-world conditions — to identify the most appropriate and targeted risk-management measures. Statistical distribution validation implements the continuous, metric-driven testing of operational data these provisions require.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 13 — Consistency", "fit": "direct", "rationale": "DAMA DMBOK2 defines consistency as a data quality dimension requiring that data values are coherent across time and systems. Statistical distribution validation operationalizes temporal consistency monitoring for AI-specific data pipelines by detecting when the population of values has shifted from the established baseline. DMBOK2 recommends continuous monitoring and trending of data quality metrics over time.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "dcam", "requirement_id": "Capability 5.2", "fit": "direct", "rationale": "DCAM v2.2 capability 5.2 requires data quality to be profiled and measured on an ongoing basis. Statistical distribution validation is continuous profiling against reference baselines — the measurement activity this capability describes.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "databricks_unity", "requirement_id": "Databricks Lakehouse Monitoring", "fit": "direct", "rationale": "Databricks Lakehouse Monitoring provides statistical profiling and drift detection for Delta tables, computing PSI and other drift metrics against baseline snapshots and surfacing results in Unity Catalog as queryable metric tables. This capability directly implements distribution validation for Databricks-hosted AI pipelines. Metric tables stored in Delta format provide durable, auditable evidence for attestation.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "google_dataplex", "requirement_id": "Dataplex Data Profile Scans", "fit": "partial", "rationale": "Google Cloud Dataplex supports data profile scans that compute statistical summaries of BigQuery and Cloud Storage datasets, which can serve as baselines for distribution comparison. Dataplex does not natively compute drift scores against prior baselines; custom Dataflow or Vertex AI pipeline components are required to complete the distribution comparison and alerting. Dataplex profile scan results provide partial evidence for distribution validation attestation.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "owasp_ai_exchange", "requirement_id": "dataqualitycontrol", "fit": "supporting", "rationale": "DV-03 computes distribution-drift scores against a registered baseline and holds the pipeline on threshold breach, guarding training-data quality against drift and poisoning.", "normative_force": "industry-framework", "source_version": "continuously-updated", "reviewed_on": "2026-07-08", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "mitre", "requirement_id": "AML.M0007", "fit": "supporting", "direction": "control-supports-requirement", "rationale": "Control \"The system must have a registered distribution baseline for every active AI pipeline,…\" enacts ATLAS mitigation AML.M0007 Sanitize Training Data; OpenCRE crosswalks this control’s OWASP AI Exchange concept (dataqualitycontrol) to this mitigation.", "source_version": "v2026.06", "reviewed_on": "2026-07-08", "source_status": "authoritative", "mapping_confidence": "medium", "legal_status": "not-applicable", "control_readiness": "approved", "implementation_maturity": "developing", "assurance_result": "not-assessed", "evidence_status": "no-evidence", "normative_force": "industry-framework", "basis": "asserted", "relation": "equivalent_to", "correction": "ap38-opencre-enhancement 2026-07-08" } ], "canonical_id": "apeiris://data/controls/DV-03", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "The system must have a registered distribution baseline for every active AI pipeline, with a monitoring service computing drift scores on each incoming data batch and triggering alerts or pipeline holds when scores exceed the documented thresholds. Drift alert acknowledgment and resolution records must exist within the defined SLA for every alert raised during the assessment period.", "evidence_required": [ "baseline_profile_registry export showing each AI pipeline ID, its reference data window, and computed feature statistics (mean, standard deviation, quantile distributions, null rates) linked to the corresponding model version", "drift_score_time_series log covering the assessment period, with entries for each batch showing pipeline ID, batch ID, drift metric used (PSI/KL/Wasserstein), per-feature drift scores, and whether the threshold was breached", "threshold_configuration record for each pipeline documenting warning and critical drift thresholds per feature and the approved rationale for each threshold value", "drift_alert_log showing each alert raised during the assessment period with timestamp, assigned owner, and acknowledgment and resolution timestamps confirming SLA compliance", "baseline_refresh_record confirming baseline profiles were updated following the most recent model retraining event and within the past 12 months" ], "machine_tests": [ "Inject a synthetic data batch with PSI score 0.30 (above critical threshold) into the monitored pipeline → assert pipeline hold is triggered and alert is emitted within 60 seconds with drift_score and affected_features fields populated", "Query the baseline profile registry for each active AI pipeline → assert 100% of pipelines have a registered baseline with a model_version link and an authored_on timestamp within the past 12 months", "Simulate a monitoring service outage for 30 minutes → assert fallback alert is generated to the governance channel indicating monitoring unavailability", "Query drift score time-series for a 90-day window → assert no gaps exceeding one pipeline batch cycle exist for any active AI pipeline" ], "human_review": [ "Review drift threshold policies for each AI use case to assess whether warning and critical thresholds are appropriately calibrated to the business risk of the downstream AI decision, not set to a generic default across all pipelines", "Inspect a sample of drift alert records to confirm root-cause analysis documentation is complete and remediation actions are genuine rather than threshold relaxation without investigation", "Assess baseline refresh governance to confirm refresh events are triggered by model retraining events and an annual review schedule, not only on an ad-hoc basis" ], "blocking_effect": "advisory", "normative_status": "binding-law", "anti_patterns": [ "Computing drift scores against a static baseline frozen at initial model training time and never refreshed, so drift is measured against an outdated population reference that no longer represents the validated operating regime", "Setting drift thresholds so loosely that PSI values above 0.30 never trigger alerts, providing a false appearance of monitoring coverage without any operational response capability", "Routing drift alerts only to engineering team chat channels with no connection to the data governance exception register, so alerts are acknowledged informally or dismissed without root-cause documentation", "Applying distribution monitoring only to training data pipelines and not to inference-time input data, missing the distribution shift that directly affects live AI decisions", "Treating all features with equal drift sensitivity regardless of predictive importance, missing material drift in high-importance features while alerting on low-importance ones" ], "update_status": "current", "layer_code": "DV" }, { "id": "DV-04", "layer": "DV", "plane": "control", "name": "AI Output Validation Rules", "plain": "AI-generated outputs must be evaluated against defined business rules, value constraints, and plausibility checks before they are consumed by downstream processes, stored as authoritative records, or presented to end users.", "threat": { "tags": [ "hallucination-propagation", "unsafe-output", "business-rule-violation", "unvalidated-ai-output" ], "desc": "AI systems can generate outputs that are syntactically valid but semantically incorrect, harmful, or in violation of business or regulatory constraints. Without post-generation validation rules, hallucinated values, out-of-range predictions, prohibited content, or regulatory violations propagate into downstream decisions, authoritative records, and user interactions before anyone detects them. Output validation is the last defensible gate before AI outputs have real-world effect." }, "standard": [ { "id": "eu_ai_act", "section": "Art. 15", "title": "Accuracy, robustness and cybersecurity" }, { "id": "dama_dmbok", "section": "Ch. 13", "title": "Data Quality — accuracy and reasonableness dimensions" }, { "id": "anthropic_privacy", "section": "Usage Policy", "title": "Conditions on use of model outputs and human oversight" } ], "sources": [ { "id": "anthropic_privacy_policy_2024", "title": "Anthropic Data Handling & Privacy Policy", "authority": "Anthropic, PBC", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-09-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.anthropic.com/privacy", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "anthropic_privacy_policy_2024", "relationship": "informative_reference", "rationale": "Establishes Anthropic Data Handling & Privacy Policy requirements informing the apeiris://data/controls/DV-04 AI Output Validation Rules control.", "reviewed_on": "2026-07-01" }, { "id": "snowflake_horizon_data_governance_2024", "title": "Snowflake Horizon (Data Governance)", "authority": "Snowflake Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.snowflake.com/en/data-cloud/workloads/data-governance/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "snowflake_horizon", "relationship": "informative_reference", "rationale": "Establishes Snowflake Horizon (Data Governance) requirements informing the apeiris://data/controls/DV-04 AI Output Validation Rules control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "A post-generation validation layer applies declarative rule sets to AI outputs before downstream delivery. Rules include value range constraints, format conformance, business rule checks (e.g., non-negative monetary amounts, valid entity references), prohibited content pattern matching, and plausibility scoring against reference distributions. Outputs failing validation are quarantined and routed to a review workflow rather than delivered to consumers.", "steps": [ "Define output validation rule sets for each AI use case, specifying value range constraints, format requirements, business rule checks, and prohibited content patterns in a versioned rule registry linked to the model version producing the outputs.", "Deploy a post-generation validation service that intercepts AI outputs before downstream delivery and evaluates them against the registered rule set, returning structured validation results including rule violated, output excerpt, and severity.", "Implement a graduated response policy: low-impact rule violations trigger logging and alerting; high-impact violations quarantine the output and require human review before release to downstream consumers.", "Integrate output validation metrics (pass rate, quarantine volume, review SLA) into the AI governance dashboard and establish SLAs for reviewing and dispositioning quarantined outputs." ], "data_governance_officer": { "summary": "Output validation rules are the final line of defense ensuring AI-generated data meets business quality and compliance requirements before it influences decisions or enters authoritative records. The rule set represents a documented policy decision about acceptable AI output for each use case.", "actions": [ "Approve output validation rule sets for each AI use case, ensuring rules address regulatory requirements and business risk tolerance for that use case.", "Review output quarantine records monthly to identify systematic AI output quality issues requiring model investigation or retraining.", "Ensure output validation coverage extends to all AI outputs that feed into authoritative records, financial calculations, or regulated communications." ], "failure_signals": [ "Any AI output pipeline delivering to authoritative systems without an active output validation rule set.", "Output quarantine queue growing without review actions within the defined SLA.", "Output validation rule sets not reviewed or updated following model version changes." ] }, "data_engineer": { "summary": "Output validation must be a mandatory pipeline stage between AI model execution and output delivery, with validation failures treated as first-class pipeline events requiring structured disposition.", "actions": [ "Implement output validation middleware for all AI output delivery pathways, integrated with the central rule registry.", "Wire quarantine routing and alert delivery for validation failures with structured failure metadata including rule violated, model version, and output sample.", "Version control output validation rule sets alongside model versions to enable reproducible evidence collection." ], "failure_signals": [ "Output validation layer bypassed or disabled for any AI output delivery pathway.", "Validation failures not producing structured log records.", "Rule set version not pinned to the model version producing the output." ] }, "legal_counsel": { "summary": "Output validation rules provide documented evidence that the organization applies technical measures to prevent AI systems from producing harmful, inaccurate, or regulation-violating outputs, supporting defensibility under EU AI Act Art. 9 and sector-specific regulations.", "actions": [ "Review output validation rule sets for AI systems producing regulated outputs (financial advice, medical information, legal documents) to confirm rules address the applicable regulatory constraints.", "Confirm that output validation quarantine records are retained for the period required by applicable regulations.", "Assess whether output validation rules address prohibited content requirements under EU AI Act prohibited practices provisions for the relevant use case." ], "failure_signals": [ "AI systems producing regulated outputs without validation rules addressing the relevant regulatory constraints.", "Quarantine records not retained for the required regulatory period.", "Output validation rule sets not reviewed by legal counsel before deployment to regulated use cases." ] }, "grc_auditor": { "summary": "Output validation evidence demonstrates that AI systems have technical controls preventing non-conformant outputs from reaching downstream processes, a key risk management requirement under EU AI Act Art. 9 and COBIT DSS06.03.", "actions": [ "Request output validation rule set exports for all active AI use cases and verify rule coverage addresses the risk scenarios identified in the AI risk assessment.", "Review output quarantine records for the audit period and confirm review and disposition were completed within the defined SLA.", "Sample AI output logs to verify the validation layer was active and enforced during the audit window." ], "metrics": [ "Output validation coverage: target 100% of AI outputs delivered to authoritative systems with an active validation rule set.", "Output quarantine review SLA compliance: target ≥95% of quarantined outputs reviewed within the defined SLA.", "Output validation rule set currency: target 100% reviewed following model version changes." ], "failure_signals": [ "Any AI output delivery pathway to an authoritative system lacking a validation configuration.", "Quarantine records not reviewed within the defined SLA.", "Rule sets not updated following model retraining events." ] }, "it_operations": { "summary": "Output validation rules execute in the serving path; operations owns their latency, coverage and the routing of failed outputs to review queues rather than to consumers.", "actions": [ "Operate output-validation execution within serving latency budgets and monitor rule coverage per output channel.", "Route validation failures to review queues with staffing-aware alerting.", "Version and change-control output rule sets with a rollback path." ], "failure_signals": [ "Output channels shipping unvalidated results after configuration changes.", "Review queues backing up until failures are auto-released.", "Rule updates deployed without a rollback path." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Output validation for AI systems is an emerging practice; most organizations rely on user-reported issues or downstream anomaly detection rather than systematic post-generation rule checking at the output boundary." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "multi-tenant" ], "implementers": [ "Data Engineering", "Application Teams", "Risk Management" ], "frameworks": [ { "framework": "eu_ai_act", "requirement_id": "Art. 15", "fit": "direct", "rationale": "EU AI Act Article 15 requires high-risk AI systems to achieve appropriate levels of accuracy, robustness and cybersecurity, and to perform consistently against declared accuracy metrics throughout their lifecycle. AI output validation rules are the runtime mechanism that detects outputs violating declared accuracy and consistency expectations before they propagate.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 13 — Accuracy", "fit": "direct", "rationale": "DAMA DMBOK2 defines data accuracy as the degree to which data correctly represents the real-world facts it models. For AI-generated outputs, accuracy validation requires comparing generated values against known constraints, reference data, and business rules. DMBOK2 recommends that accuracy rules be formally defined and applied as automated checks rather than manual review, particularly for data that will enter authoritative records.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "anthropic_privacy", "requirement_id": "Usage Policy", "fit": "adjacent", "rationale": "Anthropic's Usage Policy requires human oversight and review for high-stakes uses of model outputs; there is no 'Responsible Use — Output Evaluation' section. Output validation rules operationalize the policy's oversight expectation programmatically.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "snowflake_horizon", "requirement_id": "Data Quality — Quality Metric Functions", "fit": "adjacent", "rationale": "Snowflake Horizon's data metric functions (DMFs) support defining and monitoring data quality metrics for tables including AI output result tables. These capabilities can be applied to AI output tables to monitor output quality trends over time and detect systematic rule violations at scale. Horizon metric results are governed by Snowflake Horizon access controls and provide durable evidence for compliance reporting.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://data/controls/DV-04", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every AI output delivery pathway to an authoritative system, financial calculation, regulated communication, or end-user interface must have an active, versioned output validation rule set that is evaluated before output delivery. Outputs failing validation must be quarantined and not delivered to downstream consumers, with quarantine review completed within the defined SLA.", "evidence_required": [ "output_validation_rule_registry export listing each AI use case with its associated rule set version, rule types (range constraints, format checks, business rules, prohibited content patterns), approval date, and the model version the rules govern", "output_validation_execution_log covering the assessment period showing, for each AI output batch, the rule set version applied, pass/fail result per rule, and quarantine flag for failing outputs", "quarantine_review_log showing each quarantined output record with severity classification, assigned reviewer, review timestamp, disposition decision, and confirmation that review was completed within the defined SLA", "rule_set_change_log recording all output validation rule modifications with model version trigger, change rationale, and approval record" ], "machine_tests": [ "Submit a synthetic AI output with a known prohibited content pattern to the validation layer → assert the output is flagged, quarantined, and a structured validation failure log entry is produced before any delivery to downstream systems", "Submit an AI output with a numeric field value 10x outside the defined range constraint → assert validation rejects the output and quarantine routing is triggered with rule_violated field populated", "Attempt to deliver an AI output to a downstream system with the validation layer disabled → assert the delivery pathway is blocked and a governance alert is raised", "Query active AI output pipelines against the rule registry → assert 100% of pipelines delivering to authoritative systems have an associated active rule set with a model_version link" ], "human_review": [ "Review output validation rule sets for AI use cases producing regulated outputs (financial advice, medical information, compliance decisions) to confirm rules adequately address the regulatory constraints for each use case", "Inspect quarantine review records for the assessment period to verify disposition decisions are substantive and quarantined outputs are not systematically re-approved without root-cause investigation", "Assess whether output validation rules are updated following model version changes and whether the rule review process includes appropriate subject matter expertise for each regulated domain" ], "blocking_effect": "blocks-runtime-action", "normative_status": "binding-law", "anti_patterns": [ "Defining output validation rules that check only data types and null values but not value ranges, business rule constraints, or prohibited content patterns, providing structural validation without semantic quality assurance", "Routing quarantined outputs back to the requesting process after a configurable timeout rather than requiring explicit human disposition, allowing stale quarantine items to auto-release without review", "Maintaining a single shared output validation rule set across multiple AI model versions, so rule changes for one version inadvertently affect validation behavior for other versions in production", "Applying output validation only to the final output payload and not to intermediate output components (extracted entities, confidence scores, structured fields) that are consumed independently by downstream systems", "Treating output validation as a logging-only layer with no quarantine capability, so validation findings are recorded but non-conformant outputs are still delivered to downstream consumers" ], "update_status": "current", "layer_code": "DV" }, { "id": "DV-05", "layer": "DV", "plane": "lifecycle", "name": "Validation Exception Management", "plain": "All data validation failures must be captured in a managed exception register, triaged according to a defined severity classification, escalated through defined channels, and resolved or formally accepted through a documented decision process within defined SLAs.", "threat": { "tags": [ "silent-failure", "untracked-exception", "escalation-bypass", "unresolved-remediation" ], "desc": "Validation failures that are logged but never acted upon are equivalent to no validation at all. Without a managed exception process, teams suppress validation failures, adjust thresholds to eliminate alerts without root-cause analysis, or allow quarantined data to accumulate unresolved. Over time, validation infrastructure loses operational credibility and is bypassed in favor of pipeline throughput. Regulators reviewing AI governance find gaps between logged failures and documented remediation actions, creating material compliance exposure." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 13", "title": "Data Quality — issue management and remediation workflow" }, { "id": "cobit_2019", "section": "DSS02.02", "title": "Record, classify and prioritise requests and incidents" }, { "id": "dcam", "section": "Capability 5.3", "title": "Data quality issues are identified and remediated" }, { "id": "eu_ai_act", "section": "Art. 9(1)", "title": "Risk management system — documented risk treatment process" } ], "sources": [ { "id": "microsoft_purview_data_governance_2024", "title": "Microsoft Purview (Data Governance)", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/purview/purview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_purview_data_governance_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Purview (Data Governance) requirements informing the apeiris://data/controls/DV-05 Validation Exception Management control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "A validation exception register captures all validation failures with structured metadata. Exceptions are classified by severity (critical, high, medium, low) based on the pipeline affected, the nature of the violation, and the business impact of the affected AI use case. Each severity class has a defined SLA for acknowledgment, root-cause analysis, and resolution or formal risk acceptance. Unresolved exceptions escalate automatically through defined channels when SLAs are breached.", "steps": [ "Deploy a validation exception register that captures all validation failures with structured fields: pipeline ID, exception type, severity classification, affected record count, dataset version, timestamp, current disposition, and assigned owner.", "Define severity classification rules mapping validation failure types and business impact to severity levels (critical, high, medium, low), and document SLAs for acknowledgment (e.g., critical: 2 hours), root-cause analysis (critical: 24 hours), and resolution or formal acceptance (critical: 72 hours).", "Configure automatic escalation triggered when exceptions are not acknowledged or resolved within the SLA: SLA breaches escalate to the next level of data governance, with critical exceptions escalating to the Data Governance Officer if not resolved within the resolution SLA.", "Establish a formal risk acceptance process for exceptions that cannot be immediately remediated, requiring documented rationale, residual risk assessment, compensating controls, and an expiration date for re-evaluation." ], "data_governance_officer": { "summary": "The exception management process ensures that validation failures translate into governed remediation actions rather than silent accumulation, maintaining the integrity of the validation infrastructure and the credibility of data quality governance claims.", "actions": [ "Review the exception register weekly to monitor open exception counts, escalation backlog, and SLA compliance trends.", "Approve formal risk acceptance decisions for exceptions that exceed the standard resolution SLA.", "Conduct quarterly exception trend reviews to identify systemic data quality issues requiring structural remediation rather than case-by-case resolution." ], "failure_signals": [ "Exception register not populated despite validation failures being recorded in system logs.", "Risk acceptance records missing rationale, residual risk assessment, or expiration dates.", "Open critical or high exceptions older than the defined resolution SLA without formal acceptance on record." ] }, "data_engineer": { "summary": "All validation components must reliably emit exception events to the register, and automated escalation and SLA tracking must be operational and tested.", "actions": [ "Integrate all validation components (schema validators, quality gates, distribution monitors, output validators) with the exception register via a standardized event schema.", "Implement SLA tracking and automated escalation logic in the exception register system, with configurable escalation targets per severity class.", "Test exception register integrations after any validation component deployment or configuration change to confirm event delivery." ], "failure_signals": [ "Validation failures occurring in system logs without corresponding exception register entries.", "Escalation automation not triggering for exceptions that have exceeded their SLA.", "Exception register not accessible or queryable by the data governance team." ] }, "grc_auditor": { "summary": "Exception management records are the primary evidence that validation failures were discovered, triaged, and resolved through a governed process, directly supporting audit findings on data governance control effectiveness.", "actions": [ "Extract the exception register for the audit period and verify that every exception has acknowledgment, root-cause, and disposition records.", "Test a sample of closed exceptions to verify that remediation was genuine rather than threshold adjustments that mask the underlying quality problem.", "Review risk acceptance records to confirm they include rationale, residual risk assessment, compensating controls, and an expiration date." ], "metrics": [ "Exception register completeness: target 100% of validation failures captured in the register.", "SLA compliance by severity: target ≥95% of critical exceptions acknowledged and resolved within the defined SLA.", "Risk acceptance record completeness: target 100% include rationale, residual risk, and expiration date." ], "failure_signals": [ "Validation failures in system logs not reflected in the exception register.", "SLA breach rate for critical exceptions above 5%.", "Risk acceptance records missing required fields." ] }, "it_operations": { "summary": "The exception register infrastructure must be operated reliably with automated escalation and SLA tracking functional at all times, as its unavailability directly impairs data governance control effectiveness.", "actions": [ "Monitor exception register system availability and escalate infrastructure incidents within the defined RTO.", "Maintain escalation notification integrations (e.g., PagerDuty, Slack, email) and test them monthly to confirm delivery.", "Ensure exception register data is backed up and retained per the data retention policy, with restore procedures tested annually." ], "failure_signals": [ "Exception register system unavailable beyond the defined RTO without incident escalation.", "Escalation notifications not delivered within the defined SLA due to integration failures.", "Exception register data retention policy not enforced or backup restore untested." ] }, "legal_counsel": { "summary": "Validation exceptions are risk acceptances. When the exception concerns regulated data or consequential decisions, that acceptance needs a legally accountable owner and an expiry.", "actions": [ "Require legal review for exceptions affecting regulated data or consequential decision systems.", "Set maximum exception lifetimes with escalation on renewal.", "Review the exception register periodically for accumulating legal risk." ], "failure_signals": [ "Long-lived exceptions on regulated pipelines renewed without review.", "Exceptions granted by the team that requested them, with no independent owner.", "Incident post-mortems tracing harm to an expired-but-active exception." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Most organizations lack a structured exception management process for data validation failures, treating them as transient operational alerts rather than governed records requiring tracked remediation." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "federated-enterprise", "eu-high-risk-ai" ], "implementers": [ "Data Governance Office", "Data Engineering", "IT Operations" ], "frameworks": [ { "framework": "dama_dmbok", "requirement_id": "Ch. 13 — Issue Management", "fit": "direct", "rationale": "DAMA DMBOK2 Chapter 13 includes data quality issue management as a core process, requiring that identified quality problems be documented, prioritized, assigned, and tracked to resolution with defined SLAs. The exception register and SLA framework directly implement this requirement. DMBOK2 further distinguishes between immediate remediation and formal acceptance with documented risk treatment, both of which the exception management process must support.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "cobit_2019", "requirement_id": "DSS02.02", "fit": "direct", "rationale": "COBIT 2019 practice DSS02.02 requires incidents to be recorded, classified and prioritised. Validation exception management applies that discipline to data quality gate failures: every exception is recorded, classified by severity and routed. DSS02.03 covers service-request fulfilment, not incident classification.", "normative_force": "voluntary-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "dcam", "requirement_id": "Capability 5.3", "fit": "direct", "rationale": "DCAM v2.2 capability 5.3 requires data quality issues to be identified and remediated through a defined workflow. Validation exception management is that workflow for gate failures: recorded, classified, escalated and resolved.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "eu_ai_act", "requirement_id": "Art. 9(1)", "fit": "partial", "rationale": "EU AI Act Article 9(1) requires that risk management systems be documented, continuously updated, and systematically applied throughout the AI system lifecycle. Validation exception management is part of the systematic risk management process, ensuring that identified data quality risks are tracked and treated rather than allowed to accumulate. Exception records provide documented evidence of risk management system operation available for regulatory inspection.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "microsoft_purview", "requirement_id": "Data Health Management", "fit": "adjacent", "rationale": "Microsoft Purview's data health management capabilities provide visibility into data quality issues across the data estate, including the ability to track and manage quality findings through health scores and quality metrics. Purview health findings can be integrated with the exception register to provide centralized exception visibility for Microsoft-hosted data assets. Purview audit logs provide durable evidence of exception management activities for compliance reporting.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://data/controls/DV-05", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "All data validation failures must be captured in a managed exception register with structured metadata, classified by defined severity levels, assigned to an owner, and resolved or formally risk-accepted within the documented SLA for each severity class. No critical or high exceptions may remain open beyond the resolution SLA without a formal risk acceptance record containing rationale, residual risk assessment, compensating controls, and an expiration date.", "evidence_required": [ "validation_exception_register export covering the assessment period with entries for each exception showing pipeline ID, exception type, severity, affected record count, timestamp, assigned owner, current disposition, and acknowledgment, root-cause, and resolution timestamps", "SLA_compliance_report computing acknowledgment and resolution times against defined SLAs by severity class, showing the percentage of exceptions meeting each SLA tier", "formal_risk_acceptance_records for all exceptions resolved via risk acceptance, each containing documented rationale, residual risk assessment, compensating controls, and expiration date", "escalation_audit_log showing automatic escalation events triggered by SLA breaches, including the escalation target and the resulting response action and timestamp" ], "machine_tests": [ "Inject a synthetic critical-severity validation failure into the exception pipeline → assert an exception register entry is created within 60 seconds with severity=critical, pipeline_id, and an owner assignment notification delivered", "Advance system clock past the critical acknowledgment SLA without acknowledging the injected exception → assert automatic escalation is triggered and routed to the defined critical escalation target within the defined escalation window", "Query the exception register for all critical and high exceptions older than the resolution SLA → assert 100% have either a resolution record or a formal risk acceptance record with all required fields (rationale, residual_risk, compensating_controls, expiration_date)", "Simulate exception register system unavailability for 10 minutes → assert a governance alert is raised indicating exception capture interruption" ], "human_review": [ "Review a sample of closed exception records to verify root-cause analysis documentation is substantive and that resolution actions address the underlying data quality cause rather than threshold adjustments that mask the problem", "Assess risk acceptance records to confirm they contain genuine residual risk assessments with compensating controls and expiration dates, not boilerplate approvals that reuse identical language across unrelated exceptions", "Review the severity classification policy to confirm it appropriately maps validation failure types and business impact to severity levels aligned with the actual risk profile of each AI use case" ], "blocking_effect": "requires-review", "normative_status": "binding-law", "anti_patterns": [ "Treating validation exceptions as operational alerts that are acknowledged and closed without root-cause documentation, eliminating the audit trail that demonstrates governed remediation", "Adjusting drift or quality thresholds to eliminate existing exceptions rather than investigating and resolving the underlying data quality cause, converting a remediation action into a governance record falsification", "Classifying all exceptions at low severity to avoid escalation SLA requirements, systematically understating business impact to reduce governance process burden", "Allowing risk acceptance records to omit expiration dates, creating open-ended residual risk accumulation without scheduled re-evaluation checkpoints", "Maintaining the exception register as an untracked spreadsheet with no automated SLA tracking or escalation, making SLA compliance dependent on manual vigilance rather than system enforcement" ], "update_status": "current", "layer_code": "DV" }, { "id": "DV-06", "layer": "DV", "plane": "both", "name": "Validation Pipeline Monitoring", "plain": "Validation pass/fail rates, exception volumes, and pipeline throughput metrics must be continuously collected, trended over time, and surfaced in a data governance dashboard that enables early detection of deteriorating data quality across all AI pipelines.", "threat": { "tags": [ "quality-degradation", "silent-pipeline-failure", "metric-blindness", "sla-breach-accumulation" ], "desc": "Validation infrastructure that produces point-in-time results without surfacing trends allows quality degradation to go undetected until it causes visible AI failures. Gradual increases in validation failure rates, growing exception backlogs, and declining pass rates are early warning signals that require trending analysis to detect. Without monitoring, organizations discover data quality problems through AI system failures rather than proactive instrumentation, converting preventable quality issues into operational incidents." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 13", "title": "Data Quality — measurement, trending, and reporting" }, { "id": "cobit_2019", "section": "MEA01.04", "title": "Monitor data quality and pipeline performance metrics" }, { "id": "dcam", "section": "Capability 5.2", "title": "Data quality is profiled and measured" }, { "id": "eu_ai_act", "section": "Art. 9(6)-(8)", "title": "Testing against defined metrics, including real-world conditions" } ], "sources": [ { "id": "snowflake_horizon_data_governance_2024", "title": "Snowflake Horizon (Data Governance)", "authority": "Snowflake Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://www.snowflake.com/en/data-cloud/workloads/data-governance/", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "snowflake_horizon", "relationship": "informative_reference", "rationale": "Establishes Snowflake Horizon (Data Governance) requirements informing the apeiris://data/controls/DV-06 Validation Pipeline Monitoring control.", "reviewed_on": "2026-07-01" }, { "id": "databricks_unity_catalog_2024", "title": "Databricks Unity Catalog", "authority": "Databricks, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.databricks.com/en/data-governance/unity-catalog/index.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "databricks_unity_catalog_2024", "relationship": "informative_reference", "rationale": "Establishes Databricks Unity Catalog requirements informing the apeiris://data/controls/DV-06 Validation Pipeline Monitoring control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "A metrics collection layer aggregates validation results across all pipeline stages into a time-series data store. Dashboards display rolling pass rates, exception volumes, drift scores, and SLA compliance metrics per pipeline. Threshold-based alerts trigger when metrics cross defined warning and critical levels. Monthly trend reports are reviewed by the data governance committee as a formal governance cadence.", "steps": [ "Instrument all validation components (schema validators, quality gates, distribution monitors, output validators) to emit structured metrics events including pipeline ID, validation type, pass count, fail count, exception severity counts, and processing timestamp to a centralized metrics store.", "Build dashboards displaying rolling 7-day, 30-day, and 90-day pass rate trends, exception volume, quarantine backlog size, and SLA compliance per pipeline and validation type, with drill-down capability to individual exception events.", "Define metric alert thresholds for warning (e.g., pass rate declining 3 percentage points over 7 days) and critical (e.g., pass rate below 90% for any pipeline) conditions, and configure automated alert delivery to the data governance team.", "Establish a monthly validation health review cadence where the data governance committee reviews trend reports and formally documents responses to any identified degradation patterns, with records retained as governance evidence." ], "data_governance_officer": { "summary": "Validation pipeline monitoring transforms the validation infrastructure from a reactive check into a proactive quality intelligence capability, enabling the data governance committee to detect and respond to quality trends before they affect AI system reliability.", "actions": [ "Chair the monthly validation health review and formally document findings, decisions, and assigned remediation actions.", "Escalate any pipeline exhibiting a sustained declining pass rate trend for root-cause analysis with a defined response timeline.", "Ensure monitoring coverage extends to all active AI pipelines and that gaps in coverage are treated as governance findings." ], "failure_signals": [ "Any active AI pipeline not emitting validation metrics to the centralized store.", "Monthly trend review not conducted or documented for more than one consecutive month.", "Declining pass rate trend exceeding the warning threshold without documented investigation within the response SLA." ] }, "data_engineer": { "summary": "Build and maintain the metrics collection and aggregation infrastructure that powers validation pipeline monitoring, treating metrics collection gaps as operational incidents.", "actions": [ "Instrument all validation components with structured metrics emission, using a standardized event schema across all validation types.", "Deploy and maintain the metrics aggregation pipeline and dashboard infrastructure, with automated refresh and staleness detection.", "Test metrics collection integrity after any validation component deployment or configuration change to confirm event delivery and schema conformance." ], "failure_signals": [ "Metrics collection gaps (missing data points in time-series) not investigated within 24 hours of detection.", "Dashboard data stale beyond the defined refresh interval without alert delivery.", "Alert thresholds not configured for any active AI pipeline." ] }, "grc_auditor": { "summary": "Validation pipeline monitoring records provide evidence of continuous data quality oversight, demonstrating that the organization maintains sustained visibility rather than relying solely on point-in-time checks for data governance assurance.", "actions": [ "Request dashboard exports and metrics data for the audit period and verify continuous metrics collection for all active AI pipelines without gaps.", "Review alert history to confirm that threshold breaches triggered acknowledgment and response within the defined SLA.", "Verify that monthly trend review records exist and include documented responses to any degradation identified during the audit period." ], "metrics": [ "Monitoring coverage: target 100% of active AI pipelines emitting validation metrics to the centralized store.", "Alert response SLA compliance: target ≥95% of alerts acknowledged within the defined SLA.", "Monthly trend review cadence: target 100% of scheduled reviews conducted and documented." ], "failure_signals": [ "Metrics gaps exceeding one pipeline batch cycle for any production AI pipeline without incident record.", "Alert acknowledgment SLA breaches above 5%.", "Monthly trend review records absent for any month in the audit period." ] }, "it_operations": { "summary": "The monitoring infrastructure must be operated reliably, with alert delivery tested regularly and monitoring data retained per the data governance retention policy.", "actions": [ "Monitor metrics collection infrastructure availability and respond to collection gaps within the defined SLA.", "Maintain and test alert delivery integrations monthly, including escalation path testing for critical threshold breaches.", "Ensure monitoring data is retained per the data governance retention policy, with annual restore testing for monitoring archives." ], "failure_signals": [ "Metrics collection infrastructure unavailable beyond the defined RTO without incident escalation.", "Alert delivery failures not detected and remediated within the defined SLA.", "Monitoring data not retained for the required policy period." ] }, "legal_counsel": { "summary": "If validation itself fails silently, every downstream assurance is void. Counsel's interest is that monitoring proves validation actually ran for the periods covered by legal representations.", "actions": [ "Require validation-execution evidence for periods covered by compliance attestations.", "Confirm monitoring alerts on validation outages for legally significant pipelines are actioned.", "Use execution history to rebut claims that controls were not operating." ], "failure_signals": [ "Attestations covering periods where validation demonstrably did not run.", "Validation outages on regulated pipelines discovered by auditors rather than by monitoring.", "No retained execution history for disputed periods." ] } }, "maturity": { "current": "initial", "target": "managed", "notes": "Most organizations have point-in-time validation results but lack aggregated trending and governance dashboards for data quality across AI pipelines; monitoring is often scoped to infrastructure rather than data quality metrics." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "high-risk-sector", "multi-tenant" ], "implementers": [ "Data Engineering", "IT Operations", "Data Governance Office" ], "frameworks": [ { "framework": "dama_dmbok", "requirement_id": "Ch. 13 — Measurement and Reporting", "fit": "direct", "rationale": "DAMA DMBOK2 Chapter 13 identifies data quality measurement and reporting as a core management process, requiring that quality metrics be collected, trended, and reported to governance stakeholders on a regular cadence. Validation pipeline monitoring is the technical implementation of this process for AI-specific data pipelines. DMBOK2 further recommends that quality metrics drive continuous improvement rather than serve solely as compliance evidence.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "cobit_2019", "requirement_id": "MEA01.04", "fit": "direct", "rationale": "COBIT 2019 MEA01.04 requires that performance monitoring systems collect and analyze metrics over time to support management decision-making and enable early detection of degradation. Validation pipeline monitoring directly implements this requirement by providing time-series quality metrics for AI data pipelines. COBIT further requires that monitoring results be reviewed against defined targets and that degradation triggers corrective action through a documented process.", "normative_force": "voluntary-standard", "source_version": "2019", "reviewed_on": "2026-06-29", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "dcam", "requirement_id": "Capability 5.2", "fit": "direct", "rationale": "DCAM v2.2 capability 5.2 (data quality is profiled and measured) implies the measurements themselves are produced reliably over time. Validation pipeline monitoring watches the measurement machinery — proving profiling ran, trends were computed and alerts fired.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "eu_ai_act", "requirement_id": "Art. 9(6)-(8)", "fit": "partial", "rationale": "EU AI Act Article 9(6)-(8) requires testing of high-risk AI systems against defined metrics and probabilistic thresholds, including in real-world conditions where appropriate. Validation pipeline monitoring is the operational evidence that this metric-driven testing keeps running and keeps being acted on after deployment.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "satisfies" }, { "framework": "databricks_unity", "requirement_id": "Databricks Lakehouse Monitoring — Metric Tables", "fit": "direct", "rationale": "Databricks Lakehouse Monitoring produces metric tables in Delta format that provide durable, queryable records of validation and data quality performance over time. These metric tables directly power governance dashboards and alert queries for Databricks-hosted AI pipelines. Unity Catalog governance applies to metric tables, providing access control and audit lineage for all monitoring data.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "snowflake_horizon", "requirement_id": "Snowflake Data Metric Functions — History", "fit": "partial", "rationale": "Snowflake Horizon's data metric functions (DMFs) compute quality metrics on a scheduled basis and store result history in queryable system tables, enabling trend analysis for Snowflake-hosted AI pipeline data. These capabilities support validation pipeline monitoring for Snowflake environments but require custom aggregation logic to combine metrics across multi-stage pipelines into a unified governance view.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://data/controls/DV-06", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "The organization must maintain a continuously updated time-series record of validation pass rates, exception volumes, and SLA compliance metrics for all active AI pipelines, surfaced in a governance dashboard with at least 90 days of rolling trend data. Monthly trend review meetings must be conducted and documented, with responses to all identified degradation patterns recorded and assigned to owners.", "evidence_required": [ "validation_metrics_time_series export covering the assessment period showing pass rate, fail rate, exception volume by severity, and SLA compliance per pipeline and validation type, with no gaps exceeding one batch cycle for any active pipeline", "dashboard_configuration_record documenting which AI pipelines are covered, the metric refresh interval, warning and critical alert thresholds configured per pipeline, and the alert delivery routing destinations", "alert_history_log for the assessment period showing each threshold breach alert with trigger timestamp, acknowledgment record, and response action taken", "monthly_trend_review_minutes for each month in the assessment period, documenting the reviewing body, findings identified, and assigned remediation actions with owners and target dates" ], "machine_tests": [ "Query the metrics time-series store for all active AI pipelines over a 90-day window → assert no pipeline has a gap in metric records exceeding one pipeline batch cycle", "Set an AI pipeline pass rate to 85% (below the 90% critical threshold) → assert a critical alert is generated and delivered to the data governance team within the defined alert SLA", "Disable metrics emission from a validation component for 2 batch cycles → assert a staleness alert is raised indicating the monitoring gap within the defined detection window", "Query dashboard configuration records → assert every active AI pipeline has a configured alert threshold and routing destination with a last-reviewed date within 12 months" ], "human_review": [ "Review the monthly trend review records to confirm findings are substantiated by metric data and remediation actions are assigned with owners and target dates, not merely noted without accountability", "Assess whether alert thresholds are appropriately calibrated per pipeline based on the risk profile of the downstream AI use case, rather than set to a single generic threshold across all pipelines regardless of business impact", "Inspect the monitoring coverage inventory to confirm that all AI pipelines producing outputs consumed by authoritative systems are included and that coverage gaps are treated as governance findings requiring remediation" ], "blocking_effect": "advisory", "normative_status": "binding-law", "anti_patterns": [ "Publishing validation metrics to dashboards with no configured alert thresholds, converting the monitoring capability into a passive reporting tool that records degradation without triggering any operational response", "Resetting trend history when pipelines are redeployed or reconfigured, destroying the longitudinal data needed to detect gradual quality degradation patterns that develop across deployment cycles", "Reviewing dashboard data only when AI quality issues are reported by users rather than conducting monthly structured governance reviews with documented findings and assigned remediation actions", "Aggregating validation metrics across all pipelines into a single enterprise-wide pass rate that hides per-pipeline degradation behind high-performing pipelines, masking localized quality failures", "Treating metrics collection gaps as routine operational events rather than governance incidents requiring investigation and records, normalizing data blind spots in the monitoring coverage" ], "update_status": "current", "layer_code": "DV" }, { "id": "DV-07", "layer": "DV", "plane": "lifecycle", "name": "Data Validation Tooling Governance", "plain": "The tools, frameworks, and platforms used to enforce data validation in AI pipelines must be formally selected, approved, operated, and periodically reviewed through a governed tooling lifecycle that prevents unauthorized validation bypass and ensures tooling capabilities remain aligned with organizational data quality requirements.", "threat": { "tags": [ "untrusted-tooling", "validation-bypass-via-tooling-gap", "tooling-capability-drift", "unapproved-tool-substitution" ], "desc": "Ad-hoc or unapproved validation tooling introduces inconsistent enforcement across AI pipelines, creating gaps where data quality requirements are nominally met but not actually enforced. Tooling that is not maintained may develop security vulnerabilities or capability regressions over time. Without a governed tooling lifecycle, teams quietly replace or disable validation tools when they cause pipeline slowdowns, eliminating quality controls without governance visibility and creating undocumented risk exposure." }, "standard": [ { "id": "dama_dmbok", "section": "Ch. 13", "title": "Data Quality — tools, techniques, and tooling governance" }, { "id": "cobit_2019", "section": "BAI03.08", "title": "Execute solution testing" }, { "id": "dcam", "section": "Capability 5.1", "title": "The data quality management program is established" }, { "id": "iso_8000", "section": "ISO 8000-61", "title": "Data quality management — process reference model" } ], "sources": [ { "id": "aws_lake_formation_macie_2024", "title": "AWS data governance services (Lake Formation, Macie, Glue, S3)", "authority": "Amazon Web Services, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.aws.amazon.com/lake-formation/latest/dg/what-is-lake-formation.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "aws_lake_formation_macie_2024", "relationship": "informative_reference", "rationale": "Establishes AWS data governance services (Lake Formation, Macie, Glue, S3) requirements informing the apeiris://data/controls/DV-07 Data Validation Tooling Governance control.", "reviewed_on": "2026-07-02" }, { "id": "databricks_unity_catalog_2024", "title": "Databricks Unity Catalog", "authority": "Databricks, Inc.", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://docs.databricks.com/en/data-governance/unity-catalog/index.html", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "databricks_unity_catalog_2024", "relationship": "informative_reference", "rationale": "Establishes Databricks Unity Catalog requirements informing the apeiris://data/controls/DV-07 Data Validation Tooling Governance control.", "reviewed_on": "2026-07-02" }, { "id": "microsoft_purview_data_governance_2024", "title": "Microsoft Purview (Data Governance)", "authority": "Microsoft Corporation", "source_type": "vendor-guidance", "normative_force": "best-practice", "version": "2024", "published_on": "2024-01-01", "retrieved_on": "2026-06-29", "canonical_url": "https://learn.microsoft.com/en-us/purview/purview", "license": "proprietary-free", "status": "current", "flagship": false, "source_id": "microsoft_purview_data_governance_2024", "relationship": "informative_reference", "rationale": "Establishes Microsoft Purview (Data Governance) requirements informing the apeiris://data/controls/DV-07 Data Validation Tooling Governance control.", "reviewed_on": "2026-07-01" } ], "implementation": { "pattern": "A validation tooling registry maintains the approved set of data quality tools with their approval status, supported use cases, version, operational owner, vendor, and next review date. New tooling requires formal evaluation and approval before use in AI pipelines. Existing tooling undergoes periodic review for capability alignment, security posture, and vendor health. All tooling changes require change management review and impact assessment.", "steps": [ "Establish a validation tooling registry recording each approved tool with its name, version, supported validation types (schema, quality gate, distribution, output), approval date, operational owner, vendor, license type, and scheduled review date.", "Define a tooling evaluation and approval process requiring assessment of capability coverage, integration complexity, security posture, vendor financial health, licensing terms, and support model before any tool is approved for production AI pipelines.", "Implement change management controls for validation tooling: tool version upgrades, configuration changes, and replacements must go through a change request process with impact assessment on existing validation coverage before deployment.", "Conduct annual tooling reviews covering capability alignment with evolving data quality requirements, vendor roadmap and health, security vulnerability status, licensing compliance, and whether consolidation opportunities exist to reduce tooling sprawl." ], "data_governance_officer": { "summary": "Validation tooling governance ensures the organization maintains a coherent, approved, and effective set of data quality enforcement tools across its AI pipeline portfolio, with no unauthorized tooling creating untracked quality control gaps.", "actions": [ "Maintain and publish the validation tooling registry as part of the data governance policy framework.", "Chair annual tooling reviews and document decisions regarding tool retention, upgrade, or replacement.", "Ensure that any tooling coverage gap identified in a validation audit triggers a formal tooling evaluation within a defined response window." ], "failure_signals": [ "AI pipelines using validation tooling not listed in the approved registry.", "Tooling registry not reviewed within the past 12 months.", "Critical security vulnerabilities in validation tooling not remediated within the defined patching SLA." ] }, "data_engineer": { "summary": "Deploy only registry-approved validation tooling in AI pipelines and surface tooling capability gaps or coverage limitations through the formal evaluation process rather than working around them informally.", "actions": [ "Deploy only validation tooling listed in the approved registry for all AI pipeline validation requirements.", "Submit change requests for any tooling version upgrade or configuration change with impact assessment on validation coverage.", "Identify and report tooling capability gaps to the data governance team through the formal evaluation process." ], "failure_signals": [ "Unapproved validation tooling deployed in production AI pipelines.", "Tooling version upgrades deployed without change management records.", "Known tooling capability gaps addressed by informal workarounds rather than the formal evaluation process." ] }, "grc_auditor": { "summary": "Tooling governance evidence demonstrates that validation controls are enforced through a coherent, approved tooling set governed by a documented lifecycle process, supporting audit findings on the reliability and consistency of data quality controls.", "actions": [ "Cross-reference production AI pipeline validation configurations against the approved tooling registry to identify any unapproved tooling in use.", "Review annual tooling review records to confirm they were conducted, documented, and addressed all tools in the registry.", "Check tooling security vulnerability records to verify patching SLA compliance for all approved tools." ], "metrics": [ "Tooling registry coverage: target 100% of production validation tooling registered and approved.", "Annual tooling review completion: target 100% of registered tools reviewed within the defined annual cycle.", "Tooling security patching SLA compliance: target ≥95% of vulnerabilities remediated within the defined SLA." ], "failure_signals": [ "Production validation tooling not listed in the approved registry.", "Annual tooling review overdue by more than 90 days for any registered tool.", "Critical security vulnerabilities in validation tooling outstanding beyond the defined patching SLA." ] }, "it_operations": { "summary": "Operate approved validation tooling infrastructure reliably, apply security patches within the defined SLA, and maintain tooling availability and operational runbooks.", "actions": [ "Monitor validation tooling infrastructure health and availability, escalating incidents within the defined RTO.", "Apply security patches to all registered validation tooling within the defined patching SLA and record patch application in the tooling registry.", "Maintain operational runbooks for each approved validation tool and update them following tooling version changes." ], "failure_signals": [ "Validation tooling unavailable beyond the defined RTO without incident escalation.", "Security patches not applied within the defined SLA with no documented exception.", "Operational runbooks not updated within 30 days of a tooling version change." ] }, "legal_counsel": { "summary": "Validation tools are part of the control environment regulators assess. Counsel cares that tooling changes cannot silently weaken controls that legal representations depend on.", "actions": [ "Require change control on validation rules and tooling that implement legally mandated checks.", "Review tooling replacement plans for continuity of required validations.", "Confirm tool-generated evidence remains producible after upgrades and migrations." ], "failure_signals": [ "Legally required validation rules dropped during a tool migration.", "Rule changes deployed without any review trail.", "Historical validation evidence unreadable after a tooling upgrade." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Data quality tooling is typically selected ad-hoc by engineering teams without formal governance; tooling registries and governed evaluation processes are uncommon outside of highly regulated sectors." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "cloud-native", "federated-enterprise", "multi-tenant" ], "implementers": [ "Data Governance Office", "IT Operations", "Platform Engineering" ], "frameworks": [ { "framework": "dama_dmbok", "requirement_id": "Ch. 13 — Tools and Techniques", "fit": "direct", "rationale": "DAMA DMBOK2 Chapter 13 identifies tooling selection and governance as a component of data quality management, noting that tools must be selected to match organizational requirements and operated in a governed manner. Tooling governance directly implements this recommendation by creating a formal lifecycle for validation tool selection, approval, and review. DMBOK2 further notes that tooling capability gaps must be proactively identified and addressed.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "cobit_2019", "requirement_id": "BAI03.08", "fit": "direct", "rationale": "COBIT 2019 practice BAI03.08 (execute solution testing) requires defined test plans and acceptance criteria before solutions are promoted. Applied to validation tooling, it requires the rules and tooling that gate AI data to be themselves tested and approved before deployment.", "normative_force": "voluntary-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "dcam", "requirement_id": "Capability 5.1", "fit": "direct", "rationale": "DCAM v2.2 capability 5.1 (the data quality management program is established) encompasses the tooling through which the program operates. Validation tooling governance keeps that tooling inventoried, tested and under change control.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "aws_lake_formation", "requirement_id": "AWS Glue Data Quality (DQDL rulesets)", "fit": "partial", "rationale": "AWS Glue Data Quality — a Glue capability, not Lake Formation — defines rules in the Data Quality Definition Language (DQDL), evaluates them over Glue tables and pipelines, and stores versioned rulesets: validation tooling whose governance DV-07 requires.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" }, { "framework": "databricks_unity", "requirement_id": "Lakehouse Monitoring", "fit": "adjacent", "rationale": "Databricks Lakehouse Monitoring attaches metric tables and quality expectations to governed assets; monitors and their configurations are themselves governed objects, giving validation tooling governance a concrete implementation on Databricks.", "normative_force": "best-practice", "source_version": "2024", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "informs" } ], "canonical_id": "apeiris://data/controls/DV-07", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "Every validation tool deployed in a production AI pipeline must be listed in the approved tooling registry with its version, operational owner, supported validation types, and next scheduled review date. No validation tool may be deployed, upgraded, or replaced without a change management record including impact assessment, and all registered tools must be patched for critical security vulnerabilities within the defined patching SLA.", "evidence_required": [ "validation_tooling_registry export listing each approved tool with name, version, vendor, supported validation types, approval date, operational owner, license type, and scheduled review date", "change_management_records for all validation tooling changes during the assessment period, including tool additions, version upgrades, and configuration changes, each with impact assessment on validation coverage and an approval record", "annual_tooling_review_records for each tool in the registry, documenting reviewer, review date, capability alignment findings, vendor health assessment, and retention, upgrade, or replace decision", "vulnerability_remediation_log showing identified security vulnerabilities in registered validation tools with discovery date, patch application date, and SLA compliance status" ], "machine_tests": [ "Enumerate all validation tool instances deployed in production AI pipeline environments → assert 100% are listed in the approved tooling registry at the exact deployed version", "Query change management system for validation tooling changes in the past 12 months → assert 100% have an associated impact assessment on validation coverage and an approval record", "Query the tooling registry for annual review dates → assert no registered tool has a review date older than 12 months without a documented exception approved by the Data Governance Officer", "Query the vulnerability remediation log for critical CVEs affecting registered tools → assert all were remediated within the defined patching SLA or have an active exception record with compensating controls" ], "human_review": [ "Review annual tooling review records to assess whether capability alignment findings are substantive and vendor health assessments address financial stability, roadmap viability, and security posture rather than restating vendor marketing", "Assess whether the tooling evaluation process for new tools includes independent security testing or relies solely on vendor-provided security documentation and attestations", "Inspect the tooling registry for completeness by cross-referencing pipeline deployment configurations against the registry to surface any unapproved tools deployed outside the governed lifecycle" ], "blocking_effect": "requires-review", "normative_status": "best-practice", "anti_patterns": [ "Allowing data engineering teams to add, replace, or configure validation tooling without change management records, creating undocumented variations in validation coverage across pipelines that are invisible to governance oversight", "Maintaining the tooling registry as a static document rather than a living system linked to deployed tool instances, so the registry becomes stale and loses authority as a governance artifact", "Deferring security patching for validation tooling when patches cause pipeline disruption, creating extended windows of known vulnerability without compensating controls or documented exceptions", "Evaluating new validation tools based solely on engineering team familiarity and implementation convenience rather than against formal criteria including security posture, vendor financial health, and capability coverage alignment", "Treating validation tooling governance as a one-time selection activity with no periodic review, allowing tool capabilities to drift from organizational data quality requirements as AI use cases evolve and regulatory expectations change" ], "update_status": "current", "layer_code": "DV" }, { "id": "DV-08", "layer": "DV", "plane": "lifecycle", "name": "DataGovernanceAttestation Production", "plain": "The DataGovernanceAttestation artifact must be produced by consolidating evidence from all DX, DI, DM, DL, DA, and DV layer controls into a signed, versioned attestation record that certifies the state of data governance implementation across the organization's AI data pipelines at a specific point in time.", "threat": { "tags": [ "ungoverned-ai-data", "attestation-gap", "evidence-absence", "compliance-assertion-without-proof" ], "desc": "Organizations that assert data governance compliance without producing a structured attestation artifact cannot defend those assertions under regulatory scrutiny or adversarial audit. Without a consolidated attestation, evidence is fragmented across systems and teams, making it impossible to demonstrate the state of data governance at a specific point in time. Attestation production forces the evidence collection discipline that makes governance claims verifiable and creates accountability for gaps in control implementation." }, "standard": [ { "id": "eu_ai_act", "section": "Art. 11 & 12", "title": "Technical documentation and record-keeping for high-risk AI systems" }, { "id": "dama_dmbok", "section": "Ch. 3", "title": "Data Governance — documentation, accountability, and oversight" }, { "id": "cobit_2019", "section": "MEA04", "title": "Managed assurance" }, { "id": "dcam", "section": "Capability 6.1", "title": "The data governance function is established" } ], "sources": [ { "id": "eu_ai_act", "title": "EU AI Act", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2024/1689", "published_on": "2024-07-12", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2024/1689/oj/eng", "license": "open-access", "status": "current", "flagship": false, "source_id": "eu_ai_act", "relationship": "normative_requirement", "rationale": "Establishes EU AI Act Art. 11 & 12 requirements informing the apeiris://data/controls/DV-08 DataGovernanceAttestation Production control.", "reviewed_on": "2026-07-02" }, { "id": "dama_dmbok", "title": "DAMA-DMBOK 2nd Ed", "authority": "DAMA International", "source_type": "standard", "normative_force": "best-practice", "version": "2 (2nd Edition, Revised 2024)", "published_on": "2017-07-05", "retrieved_on": "2026-07-02", "canonical_url": "https://www.dama.org/cpages/body-of-knowledge", "license": "proprietary", "status": "current", "flagship": false, "source_id": "dama_dmbok", "relationship": "informative_reference", "rationale": "Establishes DAMA-DMBOK 2nd Ed Ch. 3 — Data Governance Documentation requirements informing the apeiris://data/controls/DV-08 DataGovernanceAttestation Production control.", "reviewed_on": "2026-07-02" }, { "id": "cobit_2019", "title": "COBIT 2019", "authority": "ISACA", "source_type": "standard", "normative_force": "industry-framework", "version": "2019", "published_on": "2018-11-12", "retrieved_on": "2026-07-02", "canonical_url": "https://www.isaca.org/resources/cobit", "license": "proprietary", "status": "current", "flagship": false, "source_id": "cobit_2019", "relationship": "informative_reference", "rationale": "Establishes COBIT 2019 MEA04 requirements informing the apeiris://data/controls/DV-08 DataGovernanceAttestation Production control.", "reviewed_on": "2026-07-02" }, { "id": "dcam", "title": "DCAM v2.2", "authority": "EDM Council", "source_type": "standard", "normative_force": "industry-framework", "version": "2.2", "published_on": "2022-01-01", "retrieved_on": "2026-07-02", "canonical_url": "https://edmcouncil.org/frameworks/dcam/", "license": "proprietary", "status": "current", "flagship": false, "source_id": "dcam", "relationship": "informative_reference", "rationale": "Establishes DCAM v2.2 Capability 6.1 requirements informing the apeiris://data/controls/DV-08 DataGovernanceAttestation Production control.", "reviewed_on": "2026-07-02" }, { "id": "gdpr", "title": "GDPR", "authority": "European Union", "source_type": "regulation", "normative_force": "binding-law", "version": "2016/679", "published_on": "2016-04-27", "retrieved_on": "2026-07-02", "canonical_url": "https://eur-lex.europa.eu/eli/reg/2016/679/oj", "license": "open-access", "status": "current", "flagship": false, "source_id": "gdpr", "relationship": "normative_requirement", "rationale": "Establishes GDPR Art. 5(2) — Accountability requirements informing the apeiris://data/controls/DV-08 DataGovernanceAttestation Production control.", "reviewed_on": "2026-07-02" }, { "id": "iso_8000", "title": "ISO 8000 Data Quality", "authority": "ISO", "source_type": "standard", "normative_force": "voluntary-standard", "version": "multi-part (8:2015, 61:2016, 110:2021, 120:2016)", "published_on": "2022-01-01", "retrieved_on": "2026-07-02", "canonical_url": "https://www.iso.org/standard/60805.html", "license": "proprietary", "status": "current", "flagship": false, "source_id": "iso_8000", "relationship": "implementation_pattern", "rationale": "Establishes ISO 8000 Data Quality ISO 8000-61 requirements informing the apeiris://data/controls/DV-08 DataGovernanceAttestation Production control.", "reviewed_on": "2026-07-02" } ], "implementation": { "pattern": "An attestation production workflow queries evidence from all active data domain controls (DX-01 through DV-07), evaluates each control's evidence status against its defined criteria, and produces a signed DataGovernanceAttestation JSON document following the Apeiris evidence ontology. The attestation is versioned, timestamped, cryptographically signed, and registered in the evidence store with a defined validity period.", "steps": [ "Define the DataGovernanceAttestation schema extending the Apeiris evidence ontology with all required fields: evidence_id, actor, intent, action, resource, policy, obligation, verdict (pass/fail/conditional/inconclusive/not-applicable), blocking_effect, confidence, confidence_basis, collected_at, valid_from, valid_until, reviewed_on, source_freshness_status, residual_risk, producer_verifier, consumer_verifiers, evidence_completeness_status, runtime_gate_required, and integrity fields (sha256 hash, Ed25519 signature).", "Build an attestation production pipeline that queries evidence artifacts from all active data domain controls (DX-01 through DV-07), evaluates each control's evidence status against its defined criteria, and aggregates results into an overall governance posture verdict with per-layer summaries.", "Implement cryptographic signing of the produced attestation using the organization's Ed25519 attestation signing key, and register the signed attestation in the Apeiris evidence store with canonical_id apeiris://data/controls/DV-08 and a unique evidence_id.", "Establish an attestation production schedule (e.g., monthly for standard governance cycles, on-demand for regulatory audit requests) with a Data Governance Officer review and approval step before finalization.", "Define attestation validity periods (e.g., valid_until 90 days for monthly attestations) and configure expiration alerting to ensure relying parties are notified before the attestation lapses and a replacement must be produced." ], "data_governance_officer": { "summary": "The DataGovernanceAttestation is the authoritative governance record certifying the state of data controls at a specific point in time; the Data Governance Officer is the approver and accountable party for every attestation produced.", "actions": [ "Review and formally approve each DataGovernanceAttestation before it is finalized and registered in the evidence store.", "Ensure attestation production is triggered for all regulatory audit requests within the required response window.", "Review attestation verdict trends across consecutive production cycles to identify improving or declining control posture and direct remediation accordingly." ], "failure_signals": [ "DataGovernanceAttestation not produced for more than one scheduled production cycle.", "Attestation produced with a fail verdict for critical controls without a documented remediation plan.", "Attestation validity period expired without a replacement attestation registered in the evidence store." ] }, "data_engineer": { "summary": "Build and operate the attestation production pipeline that collects evidence from all data domain control layers, evaluates control status, and produces the signed DataGovernanceAttestation artifact reliably on the production schedule.", "actions": [ "Implement the attestation production pipeline with evidence collection integrations for all DX through DV layer controls, using the canonical evidence schema for each control layer.", "Maintain the Ed25519 signing key integration and ensure signing failures block attestation finalization with alert delivery to the data governance team.", "Test the attestation production pipeline after any control evidence schema change to confirm evidence collection and verdict computation remain accurate." ], "failure_signals": [ "Attestation production pipeline failing without alert delivery to the data governance team.", "Evidence collection gaps for one or more control layers causing evidence_completeness_status to be non-complete.", "Signing key rotation not triggering re-signing review for attestations with remaining validity." ] }, "legal_counsel": { "summary": "The DataGovernanceAttestation is the primary documentary evidence the organization would produce in a regulatory investigation, supervisory inquiry, or audit of its AI data governance practices under the EU AI Act or GDPR.", "actions": [ "Review the DataGovernanceAttestation schema and production process to confirm it captures evidence sufficient to satisfy EU AI Act Art. 11 and 12 technical documentation requirements for data governance.", "Confirm that attestation records are retained for the period required by applicable regulations, including the EU AI Act's 10-year retention requirement for high-risk system documentation.", "Assess whether the on-demand attestation production capability can meet regulatory response deadlines for supervisory authority requests." ], "failure_signals": [ "Attestation schema not covering EU AI Act Art. 11 technical documentation requirements for data governance.", "Attestation records not retained for the legally required period.", "On-demand attestation production time exceeding the typical regulatory response deadline." ] }, "grc_auditor": { "summary": "The DataGovernanceAttestation is the consolidation point for all data governance control evidence; auditing the attestation production process validates the integrity and completeness of the entire data governance control framework across all six domain layers.", "actions": [ "Request the DataGovernanceAttestation for the audit period and verify it covers evidence from all six data domain control layers (DX, DI, DM, DL, DA, DV).", "Verify the cryptographic signature on the attestation artifact using the organization's published Ed25519 verification key.", "Cross-reference attestation verdict fields for sampled controls against the underlying control evidence to confirm verdict accuracy and evidence freshness." ], "metrics": [ "Attestation production schedule compliance: target 100% of scheduled attestations produced on time.", "Attestation evidence completeness: target 100% of DX through DV layer controls covered with non-stale evidence.", "Attestation validity continuity: target no gap period during which no valid attestation is registered in the evidence store." ], "failure_signals": [ "Attestation not produced for any scheduled cycle in the audit period.", "Attestation signature verification failure using the organization's published verification key.", "Attestation verdict fields inconsistent with the underlying control evidence for sampled controls." ] }, "it_operations": { "summary": "The DataGovernanceAttestation is only as good as the telemetry beneath it. Operations guarantees the collection, integrity and reproducibility of every machine-generated evidence stream the attestation cites.", "actions": [ "Automate collection and hashing of all evidence streams feeding the attestation.", "Monitor evidence-feed freshness so the attestation never cites stale telemetry.", "Support attestation regeneration on demand, with identical inputs producing identical artifacts." ], "failure_signals": [ "Attestations built on evidence feeds that stopped updating mid-period.", "Evidence hashes failing verification at review.", "Regeneration producing different results from the same period." ] } }, "maturity": { "current": "initial", "target": "defined", "notes": "Structured attestation production for data governance is an emerging practice; most organizations produce point-in-time compliance reports rather than signed, versioned attestation artifacts with defined validity periods and cryptographic integrity." }, "capability_risk": { "capability_level": "none" }, "tiers": [ "universal-enterprise", "high-risk-sector", "eu-high-risk-ai", "federated-enterprise" ], "implementers": [ "Data Governance Office", "GRC / Internal Audit", "Data Engineering" ], "frameworks": [ { "framework": "eu_ai_act", "requirement_id": "Art. 11 & 12", "fit": "direct", "rationale": "EU AI Act Articles 11 and 12 require providers of high-risk AI systems to maintain technical documentation and automatic logging records demonstrating compliance throughout the system lifecycle. The DataGovernanceAttestation directly satisfies the data governance component of this documentation requirement. The signed, versioned attestation with defined validity periods supports the ongoing record-keeping obligations under Art. 12 and constitutes the type of technical documentation Art. 11 requires.", "normative_force": "binding-law", "source_version": "2024/1689", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "dama_dmbok", "requirement_id": "Ch. 3 — Data Governance Documentation", "fit": "direct", "rationale": "DAMA DMBOK2 Chapter 3 identifies documentation and accountability as core data governance management practices, requiring that governance decisions, control states, and accountability records be formally documented and maintained. The DataGovernanceAttestation is the consolidated governance documentation artifact that DMBOK2's governance framework implies. It makes the state of data governance verifiable at a specific point in time and creates clear accountability for the governance posture.", "normative_force": "best-practice", "source_version": "2", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "informs" }, { "framework": "cobit_2019", "requirement_id": "MEA04", "fit": "direct", "rationale": "COBIT 2019 objective MEA04 (Managed Assurance) covers planning and executing assurance initiatives and providing assurance over governance and controls. The DataGovernanceAttestation is precisely such an assurance artifact: independently reviewable evidence that the data control environment operates as claimed. MEA02 management practices end at MEA02.04; the previously cited practice number does not exist in COBIT 2019.", "normative_force": "voluntary-standard", "source_version": "2019", "reviewed_on": "2026-07-02", "basis": "asserted", "relation": "equivalent_to" }, { "framework": "dcam", "requirement_id": "Capability 6.1", "fit": "direct", "rationale": "DCAM v2.2 capability 6.1 requires the data governance function to be established with accountability and oversight. The DataGovernanceAttestation is the governance function's account of itself: signed evidence that the data control environment it oversees operates as claimed.", "normative_force": "industry-framework", "source_version": "2.0", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" }, { "framework": "gdpr", "requirement_id": "Art. 5(2) — Accountability", "fit": "direct", "rationale": "GDPR Article 5(2) establishes the accountability principle, requiring that data controllers be able to demonstrate compliance with Article 5(1) data quality and processing principles. For AI systems processing personal data, the DataGovernanceAttestation provides structured documentary evidence of data quality and governance compliance. The signed attestation with cryptographic integrity is precisely the type of accountability documentation GDPR's accountability principle requires for data controllers to produce on regulatory request.", "normative_force": "binding-law", "source_version": "2018", "reviewed_on": "2026-06-29", "basis": "anchored", "relation": "satisfies" }, { "framework": "iso_8000", "requirement_id": "ISO 8000-61", "fit": "partial", "rationale": "ISO 8000-61's process reference model includes data quality planning, control, assurance and improvement processes; the DataGovernanceAttestation is the management-review-grade record that those processes ran and what they found.", "normative_force": "certification-standard", "source_version": "2015", "reviewed_on": "2026-07-02", "basis": "anchored", "relation": "equivalent_to" } ], "canonical_id": "apeiris://data/controls/DV-08", "meta": { "authored_on": "2026-06-29", "schema_version": "1.1.0" }, "validation_objective": "A signed DataGovernanceAttestation artifact must be produced on the defined schedule, covering evidence from all six data domain control layers (DX, DI, DM, DL, DA, DV), with a valid Ed25519 cryptographic signature, a defined validity period, and a Data Governance Officer approval record. The attestation must be registered in the evidence store before the previous attestation's validity period expires, ensuring no gap in continuous governance coverage.", "evidence_required": [ "DataGovernanceAttestation JSON artifact containing all Apeiris evidence ontology fields: evidence_id, actor, verdict, blocking_effect, confidence, confidence_basis, collected_at, valid_from, valid_until, reviewed_on, evidence_completeness_status, runtime_gate_required, and integrity.hash and integrity.signature fields", "per_layer_evidence_coverage_summary showing which controls from each of the six layers (DX, DI, DM, DL, DA, DV) were assessed, the evidence status for each, and the per-layer verdict contribution to the overall attestation verdict", "Ed25519_signature_verification_record confirming the attestation signature was verified against the organization's published verification key at time of registration in the evidence store", "DGO_approval_record documenting the Data Governance Officer review and approval of the attestation before finalization, with reviewer identity, approval timestamp, and any conditions or gaps noted", "evidence_store_registration_record showing the attestation was registered with canonical_id apeiris://data/controls/DV-08 and that the prior attestation had not yet expired at the time of registration" ], "machine_tests": [ "Verify the Ed25519 signature on the DataGovernanceAttestation using the organization's published verification key → assert signature is valid and the hash matches the canonical JSON serialization of the attestation body", "Query the evidence store for the DV-08 attestation history → assert no gap period exists between consecutive attestation validity windows where no valid attestation was registered", "Parse the DataGovernanceAttestation and check evidence_completeness_status → assert value is 'complete' with all six layer control sections (DX, DI, DM, DL, DA, DV) represented and no layer returning null", "Query the attestation against the required Apeiris evidence ontology field checklist → assert all mandatory fields are present and non-null with valid typed values" ], "human_review": [ "Cross-reference the attestation verdict fields for a sample of controls from each of the six domain layers (DX, DI, DM, DL, DA, DV) against the underlying control evidence artifacts to confirm verdict accuracy reflects the actual evidence state at the time of attestation", "Review the Data Governance Officer approval record to confirm it reflects genuine review rather than automatic approval, and assess whether the DGO documented any conditions, caveats, or gaps identified during the review", "Assess whether the on-demand attestation production capability can be executed within the organization's typical regulatory response window for supervisory authority requests under the EU AI Act or GDPR" ], "blocking_effect": "blocks-deployment", "normative_status": "binding-law", "anti_patterns": [ "Producing the DataGovernanceAttestation by manually assembling evidence without a systematic pipeline, making the attestation non-reproducible and creating undocumented gaps when evidence sources are unavailable at production time", "Signing the attestation with a shared or unmanaged signing key not controlled through a key management service, making key rotation impractical and creating accountability gaps about who authorized each attestation", "Including only passing control verdicts in the attestation while omitting failing or inconclusive controls, producing a misleading governance posture that misrepresents the actual state of data governance implementation", "Setting attestation validity periods longer than the production schedule (such as 365-day validity for a quarterly production cycle), creating extended periods where stale evidence is relied upon as current by downstream consumers", "Treating the DataGovernanceAttestation as a one-time regulatory filing artifact rather than a continuously produced governance record, resulting in evidence gaps between regulatory events that leave the organization unable to demonstrate ongoing governance posture" ], "update_status": "current", "cross_domain": { "feeds": [ "apeiris://knowledge/controls/KG-08", "apeiris://compliance/controls/AU-08" ] }, "layer_code": "DV" } ] } }