Initial commit

tests/unit/coverage/test_policy_load_and_merge.py

@@ -0,0 +1,346 @@
+"""Unit tests for policy loading and merging functionality."""
+
+# FILE: tests/unit/coverage/test_policy_load_and_merge.py
+
+import tempfile
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+import yaml
+
+from libs.policy import PolicyLoader
+from libs.schemas import CoveragePolicy, PolicyError
+
+# pylint: disable=wrong-import-position,import-error,too-few-public-methods,global-statement
+# pylint: disable=raise-missing-from,unused-argument,too-many-arguments,too-many-positional-arguments
+# pylint: disable=too-many-locals,import-outside-toplevel
+# mypy: disable-error-code=union-attr
+# mypy: disable-error-code=no-untyped-def
+
+
+class TestPolicyLoader:
+    """Test policy loading and merging functionality"""
+
+    @pytest.fixture
+    def temp_config_dir(self):
+        """Create temporary config directory with test files"""
+        with tempfile.TemporaryDirectory() as temp_dir:
+            config_dir = Path(temp_dir)
+
+            # Create baseline policy
+            baseline_policy = {
+                "version": "1.0",
+                "jurisdiction": "UK",
+                "tax_year": "2024-25",
+                "tax_year_boundary": {"start": "2024-04-06", "end": "2025-04-05"},
+                "defaults": {"confidence_thresholds": {"ocr": 0.82, "extract": 0.85}},
+                "document_kinds": ["P60", "P11D"],
+                "triggers": {
+                    "SA102": {"any_of": ["exists(IncomeItem[type='Employment'])"]}
+                },
+                "schedules": {
+                    "SA102": {
+                        "evidence": [
+                            {"id": "P60", "role": "REQUIRED", "boxes": ["SA102_b1"]}
+                        ]
+                    }
+                },
+                "status_classifier": {
+                    "present_verified": {"min_ocr": 0.82},
+                    "present_unverified": {"min_ocr": 0.60},
+                    "conflicting": {"conflict_rules": []},
+                    "missing": {"default": True},
+                },
+                "conflict_resolution": {"precedence": ["P60"]},
+                "question_templates": {
+                    "default": {
+                        "text": "Need {evidence}",
+                        "why": "Required for {schedule}",
+                    }
+                },
+                "privacy": {"vector_pii_free": True, "redact_patterns": []},
+            }
+
+            with open(config_dir / "coverage.yaml", "w") as f:
+                yaml.dump(baseline_policy, f)
+
+            # Create jurisdiction overlay
+            jurisdiction_overlay = {
+                "defaults": {
+                    "confidence_thresholds": {"ocr": 0.85}  # Override threshold
+                },
+                "document_kinds": ["P60", "P11D", "P45"],  # Add P45
+            }
+
+            with open(config_dir / "coverage.UK.2024-25.yaml", "w") as f:
+                yaml.dump(jurisdiction_overlay, f)
+
+            # Create tenant overlay
+            (config_dir / "overrides").mkdir()
+            tenant_overlay = {
+                "defaults": {"date_tolerance_days": 60}  # Override tolerance
+            }
+
+            with open(config_dir / "overrides" / "tenant123.yaml", "w") as f:
+                yaml.dump(tenant_overlay, f)
+
+            yield config_dir
+
+    @pytest.fixture
+    def policy_loader(self, temp_config_dir):
+        """Create policy loader with temp config"""
+        return PolicyLoader(str(temp_config_dir))
+
+    def test_load_baseline_policy(self, policy_loader, temp_config_dir):
+        """Test loading baseline policy without overlays"""
+        policy = policy_loader.load_policy(
+            baseline_path=str(temp_config_dir / "coverage.yaml"),
+            jurisdiction="US",  # No overlay exists
+            tax_year="2023-24",  # No overlay exists
+            tenant_id=None,
+        )
+
+        assert isinstance(policy, CoveragePolicy)
+        assert policy.version == "1.0"
+        assert policy.jurisdiction == "UK"
+        assert policy.defaults.confidence_thresholds["ocr"] == 0.82
+        assert len(policy.document_kinds) == 2
+
+    def test_load_policy_with_jurisdiction_overlay(self, policy_loader):
+        """Test loading policy with jurisdiction overlay applied"""
+        policy = policy_loader.load_policy(jurisdiction="UK", tax_year="2024-25")
+
+        # Should have jurisdiction overlay applied
+        assert policy.defaults.confidence_thresholds["ocr"] == 0.85  # Overridden
+        assert len(policy.document_kinds) == 3  # P45 added
+        assert "P45" in policy.document_kinds
+
+    def test_load_policy_with_tenant_overlay(self, policy_loader):
+        """Test loading policy with tenant overlay applied"""
+        policy = policy_loader.load_policy(
+            jurisdiction="UK", tax_year="2024-25", tenant_id="tenant123"
+        )
+
+        # Should have both jurisdiction and tenant overlays
+        assert policy.defaults.confidence_thresholds["ocr"] == 0.85  # From jurisdiction
+        assert policy.defaults.date_tolerance_days == 60  # From tenant
+        assert len(policy.document_kinds) == 3  # From jurisdiction
+
+    def test_merge_overlays(self, policy_loader):
+        """Test overlay merging logic"""
+        base = {"a": 1, "b": {"x": 10, "y": 20}, "c": [1, 2, 3]}
+
+        overlay1 = {
+            "b": {"x": 15, "z": 30},  # Merge into b, override x, add z
+            "d": 4,  # Add new key
+        }
+
+        overlay2 = {
+            "b": {"y": 25},  # Override y in b
+            "c": [4, 5, 6],  # Replace entire list
+        }
+
+        result = policy_loader.merge_overlays(base, overlay1, overlay2)
+
+        assert result["a"] == 1
+        assert result["b"]["x"] == 15  # From overlay1
+        assert result["b"]["y"] == 25  # From overlay2
+        assert result["b"]["z"] == 30  # From overlay1
+        assert result["c"] == [4, 5, 6]  # From overlay2
+        assert result["d"] == 4  # From overlay1
+
+    def test_compile_predicates(self, policy_loader):
+        """Test predicate compilation"""
+        policy = policy_loader.load_policy()
+        compiled = policy_loader.compile_predicates(policy)
+
+        assert compiled.policy == policy
+        assert len(compiled.compiled_predicates) > 0
+        assert "exists(IncomeItem[type='Employment'])" in compiled.compiled_predicates
+        assert compiled.hash is not None
+        assert len(compiled.source_files) > 0
+
+    def test_predicate_execution(self, policy_loader):
+        """Test that compiled predicates are callable"""
+        policy = policy_loader.load_policy()
+        compiled = policy_loader.compile_predicates(policy)
+
+        predicate = compiled.compiled_predicates[
+            "exists(IncomeItem[type='Employment'])"
+        ]
+
+        # Should be callable and return boolean
+        result = predicate("T-001", "2024-25")
+        assert isinstance(result, bool)
+
+    def test_invalid_yaml_file(self, temp_config_dir):
+        """Test handling of invalid YAML file"""
+        # Create invalid YAML
+        with open(temp_config_dir / "invalid.yaml", "w") as f:
+            f.write("invalid: yaml: content: [")
+
+        loader = PolicyLoader(str(temp_config_dir))
+
+        with pytest.raises(PolicyError, match="Invalid YAML"):
+            loader._load_yaml_file(str(temp_config_dir / "invalid.yaml"))
+
+    def test_missing_file(self, temp_config_dir):
+        """Test handling of missing file"""
+        loader = PolicyLoader(str(temp_config_dir))
+
+        with pytest.raises(PolicyError, match="Policy file not found"):
+            loader._load_yaml_file(str(temp_config_dir / "missing.yaml"))
+
+    def test_schema_validation_success(self, policy_loader, temp_config_dir):
+        """Test successful schema validation"""
+        policy_dict = policy_loader._load_yaml_file(
+            str(temp_config_dir / "coverage.yaml")
+        )
+
+        # Should not raise exception
+        policy_loader._validate_policy(policy_dict)
+
+    def test_schema_validation_failure(self, policy_loader):
+        """Test schema validation failure"""
+        invalid_policy = {
+            "version": "1.0",
+            # Missing required fields
+        }
+
+        with pytest.raises(Exception):  # ValidationError from jsonschema
+            policy_loader._validate_policy(invalid_policy)
+
+    def test_business_rules_validation(self, policy_loader, temp_config_dir):
+        """Test business rules validation"""
+        policy_dict = policy_loader._load_yaml_file(
+            str(temp_config_dir / "coverage.yaml")
+        )
+
+        result = policy_loader.validate_policy(policy_dict)
+        assert result.ok is True
+        assert len(result.errors) == 0
+
+    def test_business_rules_validation_failure(self, policy_loader):
+        """Test business rules validation with errors"""
+        invalid_policy = {
+            "version": "1.0",
+            "jurisdiction": "UK",
+            "tax_year": "2024-25",
+            "tax_year_boundary": {"start": "2024-04-06", "end": "2025-04-05"},
+            "defaults": {"confidence_thresholds": {"ocr": 0.82}},
+            "document_kinds": ["P60"],
+            "triggers": {"SA102": {"any_of": ["test"]}},
+            "schedules": {
+                "SA102": {
+                    "evidence": [
+                        {
+                            "id": "P11D",  # Not in document_kinds
+                            "role": "REQUIRED",
+                            "boxes": ["SA102_b1"],
+                        }
+                    ]
+                }
+            },
+            "status_classifier": {
+                "present_verified": {"min_ocr": 0.82},
+                "present_unverified": {"min_ocr": 0.60},
+                "conflicting": {"conflict_rules": []},
+                "missing": {"default": True},
+            },
+            "conflict_resolution": {"precedence": ["P60"]},
+            "question_templates": {"default": {"text": "test", "why": "test"}},
+        }
+
+        result = policy_loader.validate_policy(invalid_policy)
+        assert result.ok is False
+        assert len(result.errors) > 0
+        assert any("P11D" in error for error in result.errors)
+
+    def test_apply_feature_flags_placeholder(self, policy_loader):
+        """Test feature flags application (placeholder)"""
+        policy_dict = {"test": "value"}
+        result = policy_loader.apply_feature_flags(policy_dict)
+
+        # Currently just returns unchanged
+        assert result == policy_dict
+
+    @patch("libs.policy.utils.get_policy_loader")
+    def test_convenience_functions(self, mock_get_loader, policy_loader):
+        """Test convenience functions"""
+        # Create a valid mock policy for testing
+        from unittest.mock import MagicMock
+
+        from libs.schemas import (
+            ConflictRules,
+            CoveragePolicy,
+            Defaults,
+            Privacy,
+            QuestionTemplates,
+            StatusClassifier,
+            StatusClassifierConfig,
+            TaxYearBoundary,
+        )
+
+        mock_policy = CoveragePolicy(
+            version="1.0",
+            jurisdiction="UK",
+            tax_year="2024-25",
+            tax_year_boundary=TaxYearBoundary(start="2024-04-06", end="2025-04-05"),
+            defaults=Defaults(
+                confidence_thresholds={"ocr": 0.82, "extract": 0.85},
+                date_tolerance_days=30,
+            ),
+            document_kinds=["P60"],
+            status_classifier=StatusClassifierConfig(
+                present_verified=StatusClassifier(min_ocr=0.82, min_extract=0.85),
+                present_unverified=StatusClassifier(min_ocr=0.60, min_extract=0.70),
+                conflicting=StatusClassifier(),
+                missing=StatusClassifier(),
+            ),
+            triggers={},
+            conflict_resolution=ConflictRules(precedence=["P60"]),
+            question_templates=QuestionTemplates(
+                default={"text": "test", "why": "test"}
+            ),
+            privacy=Privacy(vector_pii_free=True, redact_patterns=[]),
+        )
+
+        # Mock the policy loader to return our test policy
+        from datetime import datetime
+
+        from libs.schemas import CompiledCoveragePolicy
+
+        mock_compiled_policy = CompiledCoveragePolicy(
+            policy=mock_policy,
+            compiled_predicates={},
+            compiled_at=datetime.now(),
+            hash="test-hash",
+            source_files=["test.yaml"],
+        )
+
+        mock_loader = MagicMock()
+        mock_loader.load_policy.return_value = mock_policy
+        mock_loader.merge_overlays.side_effect = lambda base, *overlays: {
+            **base,
+            **{k: v for overlay in overlays for k, v in overlay.items()},
+        }
+        mock_loader.compile_predicates.return_value = mock_compiled_policy
+        mock_get_loader.return_value = mock_loader
+
+        from libs.policy import compile_predicates, load_policy, merge_overlays
+
+        # Test load_policy - use the mock policy directly since we're testing the convenience function
+        policy = load_policy()
+        assert isinstance(policy, CoveragePolicy)
+        assert policy.version == "1.0"
+
+        # Test merge_overlays
+        base = {"a": 1}
+        overlay = {"b": 2}
+        merged = merge_overlays(base, overlay)
+        assert merged == {"a": 1, "b": 2}
+
+        # Test compile_predicates
+        compiled = compile_predicates(policy)
+        assert compiled.policy == policy

tests/unit/coverage/test_predicate_compilation.py

@@ -0,0 +1,270 @@
+"""Unit tests for predicate compilation and DSL parsing."""
+
+# FILE: tests/unit/coverage/test_predicate_compilation.py
+
+import pytest
+
+from libs.policy import PolicyLoader
+
+# pylint: disable=wrong-import-position,import-error,too-few-public-methods,global-statement
+# pylint: disable=raise-missing-from,unused-argument,too-many-arguments,too-many-positional-arguments
+# pylint: disable=too-many-locals,import-outside-toplevel
+# mypy: disable-error-code=union-attr
+# mypy: disable-error-code=no-untyped-def
+
+
+class TestPredicateCompilation:
+    """Test predicate compilation and DSL parsing"""
+
+    @pytest.fixture
+    def policy_loader(self):
+        """Create policy loader for testing"""
+        return PolicyLoader()
+
+    def test_compile_exists_condition(self, policy_loader):
+        """Test compilation of exists() conditions"""
+        condition = "exists(IncomeItem[type='Employment'])"
+        predicate = policy_loader._compile_condition(condition)
+
+        assert callable(predicate)
+        result = predicate("T-001", "2024-25")
+        assert isinstance(result, bool)
+
+    def test_compile_exists_condition_with_filters(self, policy_loader):
+        """Test exists() with complex filters"""
+        condition = "exists(IncomeItem[type='SelfEmployment' AND turnover_lt_vat_threshold=true])"
+        predicate = policy_loader._compile_condition(condition)
+
+        assert callable(predicate)
+        result = predicate("T-001", "2024-25")
+        assert isinstance(result, bool)
+
+    def test_compile_property_conditions(self, policy_loader):
+        """Test compilation of property conditions"""
+        conditions = [
+            "property_joint_ownership",
+            "candidate_FHL",
+            "claims_FTCR",
+            "claims_remittance_basis",
+            "received_estate_income",
+        ]
+
+        for condition in conditions:
+            predicate = policy_loader._compile_condition(condition)
+            assert callable(predicate)
+            result = predicate("T-001", "2024-25")
+            assert isinstance(result, bool)
+
+    def test_compile_computed_conditions(self, policy_loader):
+        """Test compilation of computed conditions"""
+        conditions = [
+            "turnover_lt_vat_threshold",
+            "turnover_ge_vat_threshold",
+        ]
+
+        for condition in conditions:
+            predicate = policy_loader._compile_condition(condition)
+            assert callable(predicate)
+            result = predicate("T-001", "2024-25")
+            assert isinstance(result, bool)
+
+    def test_compile_taxpayer_flag_conditions(self, policy_loader):
+        """Test compilation of taxpayer flag conditions"""
+        condition = "taxpayer_flag:has_employment"
+        predicate = policy_loader._compile_condition(condition)
+
+        assert callable(predicate)
+        result = predicate("T-001", "2024-25")
+        assert isinstance(result, bool)
+
+    def test_compile_filing_mode_conditions(self, policy_loader):
+        """Test compilation of filing mode conditions"""
+        condition = "filing_mode:paper"
+        predicate = policy_loader._compile_condition(condition)
+
+        assert callable(predicate)
+        result = predicate("T-001", "2024-25")
+        assert isinstance(result, bool)
+
+    def test_compile_unknown_condition(self, policy_loader):
+        """Test compilation of unknown condition defaults to False"""
+        condition = "unknown_condition_type"
+        predicate = policy_loader._compile_condition(condition)
+
+        assert callable(predicate)
+        result = predicate("T-001", "2024-25")
+        assert result is False  # Unknown conditions default to False
+
+    def test_exists_predicate_creation(self, policy_loader):
+        """Test exists predicate creation with different entity types"""
+        entity_types = [
+            "IncomeItem",
+            "ExpenseItem",
+            "PropertyAsset",
+            "TrustDistribution",
+        ]
+
+        for entity_type in entity_types:
+            predicate = policy_loader._create_exists_predicate(
+                entity_type, "type='test'"
+            )
+            assert callable(predicate)
+            result = predicate("T-001", "2024-25")
+            assert isinstance(result, bool)
+
+    def test_property_predicate_creation(self, policy_loader):
+        """Test property predicate creation"""
+        properties = [
+            "property_joint_ownership",
+            "candidate_FHL",
+            "claims_FTCR",
+        ]
+
+        for prop in properties:
+            predicate = policy_loader._create_property_predicate(prop)
+            assert callable(predicate)
+            result = predicate("T-001", "2024-25")
+            assert isinstance(result, bool)
+
+    def test_computed_predicate_creation(self, policy_loader):
+        """Test computed predicate creation"""
+        computations = [
+            "turnover_lt_vat_threshold",
+            "turnover_ge_vat_threshold",
+        ]
+
+        for comp in computations:
+            predicate = policy_loader._create_computed_predicate(comp)
+            assert callable(predicate)
+            result = predicate("T-001", "2024-25")
+            assert isinstance(result, bool)
+
+    def test_flag_predicate_creation(self, policy_loader):
+        """Test flag predicate creation"""
+        flags = [
+            "has_employment",
+            "is_self_employed_short",
+            "has_property_income",
+            "has_foreign_income",
+        ]
+
+        for flag in flags:
+            predicate = policy_loader._create_flag_predicate(flag)
+            assert callable(predicate)
+            result = predicate("T-001", "2024-25")
+            assert isinstance(result, bool)
+
+    def test_filing_mode_predicate_creation(self, policy_loader):
+        """Test filing mode predicate creation"""
+        modes = ["paper", "online", "agent"]
+
+        for mode in modes:
+            predicate = policy_loader._create_filing_mode_predicate(mode)
+            assert callable(predicate)
+            result = predicate("T-001", "2024-25")
+            assert isinstance(result, bool)
+
+    def test_exists_condition_regex_parsing(self, policy_loader):
+        """Test regex parsing of exists conditions"""
+        test_cases = [
+            (
+                "exists(IncomeItem[type='Employment'])",
+                "IncomeItem",
+                "type='Employment'",
+            ),
+            (
+                "exists(ExpenseItem[category='FinanceCosts'])",
+                "ExpenseItem",
+                "category='FinanceCosts'",
+            ),
+            (
+                "exists(PropertyAsset[joint_ownership=true])",
+                "PropertyAsset",
+                "joint_ownership=true",
+            ),
+        ]
+
+        for condition, expected_entity, expected_filters in test_cases:
+            # Test that the regex matches correctly
+            import re
+
+            exists_match = re.match(r"exists\((\w+)\[([^\]]+)\]\)", condition)
+            assert exists_match is not None
+            assert exists_match.group(1) == expected_entity
+            assert exists_match.group(2) == expected_filters
+
+    def test_condition_whitespace_handling(self, policy_loader):
+        """Test that conditions handle whitespace correctly"""
+        conditions_with_whitespace = [
+            "  exists(IncomeItem[type='Employment'])  ",
+            "\tproperty_joint_ownership\t",
+            "\n taxpayer_flag:has_employment \n",
+        ]
+
+        for condition in conditions_with_whitespace:
+            predicate = policy_loader._compile_condition(condition)
+            assert callable(predicate)
+            result = predicate("T-001", "2024-25")
+            assert isinstance(result, bool)
+
+    def test_complex_exists_filters(self, policy_loader):
+        """Test exists conditions with complex filter expressions"""
+        complex_conditions = [
+            "exists(IncomeItem[type='SelfEmployment' AND turnover_lt_vat_threshold=true])",
+            "exists(ExpenseItem[category='CapitalAllowances'])",
+            "exists(IncomeItem[type IN ['ForeignInterest','ForeignDividends']])",
+        ]
+
+        for condition in complex_conditions:
+            predicate = policy_loader._compile_condition(condition)
+            assert callable(predicate)
+            result = predicate("T-001", "2024-25")
+            assert isinstance(result, bool)
+
+    def test_predicate_consistency(self, policy_loader):
+        """Test that predicates return consistent results for same inputs"""
+        condition = "exists(IncomeItem[type='Employment'])"
+        predicate = policy_loader._compile_condition(condition)
+
+        # Call multiple times with same inputs
+        result1 = predicate("T-001", "2024-25")
+        result2 = predicate("T-001", "2024-25")
+        result3 = predicate("T-001", "2024-25")
+
+        # Should be consistent
+        assert result1 == result2 == result3
+
+    def test_predicate_different_inputs(self, policy_loader):
+        """Test predicates with different input combinations"""
+        condition = "exists(IncomeItem[type='Employment'])"
+        predicate = policy_loader._compile_condition(condition)
+
+        # Test with different taxpayer IDs and tax years
+        test_inputs = [
+            ("T-001", "2024-25"),
+            ("T-002", "2024-25"),
+            ("T-001", "2023-24"),
+            ("T-999", "2025-26"),
+        ]
+
+        for taxpayer_id, tax_year in test_inputs:
+            result = predicate(taxpayer_id, tax_year)
+            assert isinstance(result, bool)
+
+    def test_edge_case_conditions(self, policy_loader):
+        """Test edge cases in condition parsing"""
+        edge_cases = [
+            "",  # Empty string
+            "   ",  # Whitespace only
+            "exists()",  # Empty exists
+            "exists(Entity[])",  # Empty filter
+            "taxpayer_flag:",  # Empty flag
+            "filing_mode:",  # Empty mode
+        ]
+
+        for condition in edge_cases:
+            predicate = policy_loader._compile_condition(condition)
+            assert callable(predicate)
+            # Should default to False for malformed conditions
+            result = predicate("T-001", "2024-25")
+            assert result is False

tests/unit/coverage/test_question_templates.py

@@ -0,0 +1,272 @@
+"""Unit tests for question template generation."""
+
+# FILE: tests/unit/coverage/test_question_templates.py
+
+import pytest
+
+from libs.schemas import Citation, ClarifyContext, CoverageGap, Role, UploadOption
+
+# pylint: disable=wrong-import-position,import-error,too-few-public-methods,global-statement
+# pylint: disable=raise-missing-from,unused-argument,too-many-arguments,too-many-positional-arguments
+# pylint: disable=too-many-locals,import-outside-toplevel
+# mypy: disable-error-code=union-attr
+# mypy: disable-error-code=no-untyped-def
+
+
+class TestQuestionTemplates:
+    """Test question template generation and formatting"""
+
+    @pytest.fixture
+    def sample_gap(self):
+        """Create sample coverage gap for testing"""
+        return CoverageGap(
+            schedule_id="SA102",
+            evidence_id="P60",
+            role=Role.REQUIRED,
+            reason="P60 provides year-end pay and PAYE tax figures",
+            boxes=["SA102_b1", "SA102_b2"],
+            citations=[
+                Citation(
+                    rule_id="UK.SA102.P60.Required",
+                    doc_id="SA102-Notes-2025",
+                    locator="p.3 §1.1",
+                )
+            ],
+            acceptable_alternatives=["P45", "FinalPayslipYTD"],
+        )
+
+    @pytest.fixture
+    def sample_context(self):
+        """Create sample clarify context for testing"""
+        return ClarifyContext(
+            tax_year="2024-25",
+            taxpayer_id="T-001",
+            jurisdiction="UK",
+        )
+
+    def test_question_text_formatting(self, sample_gap, sample_context):
+        """Test basic question text formatting"""
+        # Mock the _generate_clarifying_question function behavior
+        evidence_name = sample_gap.evidence_id
+        schedule_name = sample_gap.schedule_id
+        boxes_text = ", ".join(sample_gap.boxes)
+        alternatives_text = ", ".join(sample_gap.acceptable_alternatives)
+
+        # Template format
+        template_text = "To complete the {schedule} for {tax_year}, we need {evidence}. These documents support boxes {boxes}. If you don't have this, you can provide {alternatives}."
+
+        question_text = template_text.format(
+            schedule=schedule_name,
+            tax_year=sample_context.tax_year,
+            evidence=evidence_name,
+            boxes=boxes_text,
+            alternatives=alternatives_text,
+        )
+
+        expected = "To complete the SA102 for 2024-25, we need P60. These documents support boxes SA102_b1, SA102_b2. If you don't have this, you can provide P45, FinalPayslipYTD."
+        assert question_text == expected
+
+    def test_why_text_formatting(self, sample_gap):
+        """Test why explanation formatting"""
+        template_why = "{why}. See guidance: {guidance_doc}."
+
+        why_text = template_why.format(
+            why=sample_gap.reason,
+            guidance_doc="policy guidance",
+        )
+
+        expected = "P60 provides year-end pay and PAYE tax figures. See guidance: policy guidance."
+        assert why_text == expected
+
+    def test_upload_options_generation(self, sample_gap):
+        """Test upload options generation"""
+        options = []
+
+        # Generate options for alternatives
+        for alt in sample_gap.acceptable_alternatives:
+            options.append(
+                UploadOption(
+                    label=f"Upload {alt} (PDF/CSV)",
+                    accepted_formats=["pdf", "csv"],
+                    upload_endpoint=f"/v1/ingest/upload?tag={alt}",
+                )
+            )
+
+        assert len(options) == 2
+        assert options[0].label == "Upload P45 (PDF/CSV)"
+        assert options[0].accepted_formats == ["pdf", "csv"]
+        assert options[0].upload_endpoint == "/v1/ingest/upload?tag=P45"
+        assert options[1].label == "Upload FinalPayslipYTD (PDF/CSV)"
+        assert options[1].upload_endpoint == "/v1/ingest/upload?tag=FinalPayslipYTD"
+
+    def test_upload_options_no_alternatives(self):
+        """Test upload options when no alternatives available"""
+        gap_no_alternatives = CoverageGap(
+            schedule_id="SA102",
+            evidence_id="P60",
+            role=Role.REQUIRED,
+            reason="Required document",
+            boxes=["SA102_b1"],
+            acceptable_alternatives=[],
+        )
+
+        options = []
+
+        # When no alternatives, create option for main evidence
+        if not gap_no_alternatives.acceptable_alternatives:
+            options.append(
+                UploadOption(
+                    label=f"Upload {gap_no_alternatives.evidence_id} (PDF/CSV)",
+                    accepted_formats=["pdf", "csv"],
+                    upload_endpoint=f"/v1/ingest/upload?tag={gap_no_alternatives.evidence_id}",
+                )
+            )
+
+        assert len(options) == 1
+        assert options[0].label == "Upload P60 (PDF/CSV)"
+        assert options[0].upload_endpoint == "/v1/ingest/upload?tag=P60"
+
+    def test_blocking_determination(self, sample_gap):
+        """Test blocking status determination"""
+        # Required evidence should be blocking
+        assert sample_gap.role == Role.REQUIRED
+        blocking = sample_gap.role.value == "REQUIRED"
+        assert blocking is True
+
+        # Optional evidence should not be blocking
+        optional_gap = CoverageGap(
+            schedule_id="SA102",
+            evidence_id="PayslipMonthly",
+            role=Role.OPTIONAL,
+            reason="Optional supporting document",
+            boxes=["SA102_b3"],
+        )
+
+        blocking_optional = optional_gap.role.value == "REQUIRED"
+        assert blocking_optional is False
+
+    def test_boxes_affected_formatting(self, sample_gap):
+        """Test boxes affected list formatting"""
+        boxes_affected = sample_gap.boxes
+        assert boxes_affected == ["SA102_b1", "SA102_b2"]
+
+        # Test empty boxes
+        gap_no_boxes = CoverageGap(
+            schedule_id="SA102",
+            evidence_id="EmploymentContract",
+            role=Role.OPTIONAL,
+            reason="Used for disambiguation",
+            boxes=[],
+        )
+
+        assert gap_no_boxes.boxes == []
+
+    def test_citations_preservation(self, sample_gap):
+        """Test that citations are preserved in response"""
+        citations = sample_gap.citations
+        assert len(citations) == 1
+        assert citations[0].rule_id == "UK.SA102.P60.Required"
+        assert citations[0].doc_id == "SA102-Notes-2025"
+        assert citations[0].locator == "p.3 §1.1"
+
+    def test_multiple_alternatives_formatting(self):
+        """Test formatting with multiple alternatives"""
+        gap_many_alternatives = CoverageGap(
+            schedule_id="SA105",
+            evidence_id="LettingAgentStatements",
+            role=Role.REQUIRED,
+            reason="Evidence of rental income",
+            boxes=["SA105_b5", "SA105_b20"],
+            acceptable_alternatives=[
+                "TenancyLedger",
+                "BankStatements",
+                "RentalAgreements",
+            ],
+        )
+
+        alternatives_text = ", ".join(gap_many_alternatives.acceptable_alternatives)
+        expected = "TenancyLedger, BankStatements, RentalAgreements"
+        assert alternatives_text == expected
+
+    def test_empty_boxes_formatting(self):
+        """Test formatting when no boxes specified"""
+        gap_no_boxes = CoverageGap(
+            schedule_id="SA102",
+            evidence_id="EmploymentContract",
+            role=Role.OPTIONAL,
+            reason="Used for disambiguation",
+            boxes=[],
+        )
+
+        boxes_text = (
+            ", ".join(gap_no_boxes.boxes) if gap_no_boxes.boxes else "relevant boxes"
+        )
+        assert boxes_text == "relevant boxes"
+
+    def test_special_characters_in_evidence_names(self):
+        """Test handling of special characters in evidence names"""
+        gap_special_chars = CoverageGap(
+            schedule_id="SA106",
+            evidence_id="EEA_FHL",
+            role=Role.CONDITIONALLY_REQUIRED,
+            reason="European Economic Area Furnished Holiday Lettings",
+            boxes=["SA106_b14"],
+        )
+
+        # Should handle underscores and other characters
+        assert gap_special_chars.evidence_id == "EEA_FHL"
+
+        # Upload endpoint should handle special characters
+        upload_endpoint = f"/v1/ingest/upload?tag={gap_special_chars.evidence_id}"
+        assert upload_endpoint == "/v1/ingest/upload?tag=EEA_FHL"
+
+    def test_long_reason_text(self):
+        """Test handling of long reason text"""
+        long_reason = "This is a very long reason that explains in great detail why this particular piece of evidence is absolutely essential for completing the tax return accurately and in compliance with HMRC requirements and regulations."
+
+        gap_long_reason = CoverageGap(
+            schedule_id="SA108",
+            evidence_id="CGT_BrokerAnnualReport",
+            role=Role.REQUIRED,
+            reason=long_reason,
+            boxes=["SA108_b4", "SA108_b5"],
+        )
+
+        # Should preserve full reason text
+        assert gap_long_reason.reason == long_reason
+        assert len(gap_long_reason.reason) > 100
+
+    def test_multiple_upload_formats(self):
+        """Test generation of upload options with different formats"""
+        evidence_id = "AccountsPAndL"
+
+        # Different evidence types might accept different formats
+        formats_map = {
+            "AccountsPAndL": ["pdf", "xlsx", "csv"],
+            "BankStatements": ["pdf", "csv", "ofx"],
+            "P60": ["pdf", "jpg", "png"],
+        }
+
+        for evidence, formats in formats_map.items():
+            option = UploadOption(
+                label=f"Upload {evidence}",
+                accepted_formats=formats,
+                upload_endpoint=f"/v1/ingest/upload?tag={evidence}",
+            )
+
+            assert option.accepted_formats == formats
+            assert evidence in option.upload_endpoint
+
+    def test_context_variations(self):
+        """Test question generation with different contexts"""
+        contexts = [
+            ClarifyContext(tax_year="2024-25", taxpayer_id="T-001", jurisdiction="UK"),
+            ClarifyContext(tax_year="2023-24", taxpayer_id="T-002", jurisdiction="UK"),
+            ClarifyContext(tax_year="2024-25", taxpayer_id="T-003", jurisdiction="US"),
+        ]
+
+        for context in contexts:
+            # Each context should be valid
+            assert context.tax_year.startswith("20")
+            assert context.taxpayer_id.startswith("T-")
+            assert context.jurisdiction in ["UK", "US", "CA", "AU"]

tests/unit/coverage/test_status_classifier.py

@@ -0,0 +1,338 @@
+"""Unit tests for evidence status classification."""
+
+# FILE: tests/unit/coverage/test_status_classifier.py
+
+from datetime import datetime
+
+import pytest
+
+from libs.coverage.evaluator import CoverageEvaluator
+from libs.schemas import (
+    CompiledCoveragePolicy,
+    CoveragePolicy,
+    Defaults,
+    FoundEvidence,
+    Status,
+    StatusClassifier,
+    StatusClassifierConfig,
+    TaxYearBoundary,
+)
+from libs.schemas.coverage.core import ConflictRules, Privacy, QuestionTemplates
+
+# pylint: disable=wrong-import-position,import-error,too-few-public-methods,global-statement
+# pylint: disable=raise-missing-from,unused-argument,too-many-arguments,too-many-positional-arguments
+# pylint: disable=too-many-locals,import-outside-toplevel
+# mypy: disable-error-code=union-attr
+# mypy: disable-error-code=no-untyped-def
+
+
+class TestStatusClassifier:
+    """Test evidence status classification logic"""
+
+    @pytest.fixture
+    def mock_policy(self):
+        """Create mock compiled policy for testing"""
+        policy = CoveragePolicy(
+            version="1.0",
+            jurisdiction="UK",
+            tax_year="2024-25",
+            tax_year_boundary=TaxYearBoundary(start="2024-04-06", end="2025-04-05"),
+            defaults=Defaults(
+                confidence_thresholds={"ocr": 0.82, "extract": 0.85},
+                date_tolerance_days=30,
+            ),
+            document_kinds=["P60"],
+            status_classifier=StatusClassifierConfig(
+                present_verified=StatusClassifier(
+                    min_ocr=0.82,
+                    min_extract=0.85,
+                    date_in_year=True,
+                ),
+                present_unverified=StatusClassifier(
+                    min_ocr=0.60,
+                    min_extract=0.70,
+                    date_in_year_or_tolerance=True,
+                ),
+                conflicting=StatusClassifier(
+                    conflict_rules=["Same doc kind, different totals"]
+                ),
+                missing=StatusClassifier(),
+            ),
+            conflict_resolution=ConflictRules(precedence=["P60"]),
+            question_templates=QuestionTemplates(
+                default={"text": "test", "why": "test"}
+            ),
+            privacy=Privacy(vector_pii_free=True, redact_patterns=[]),
+        )
+
+        return CompiledCoveragePolicy(
+            policy=policy,
+            compiled_predicates={},
+            compiled_at=datetime.utcnow(),
+            hash="test-hash",
+            source_files=["test.yaml"],
+        )
+
+    @pytest.fixture
+    def evaluator(self):
+        """Create coverage evaluator for testing"""
+        return CoverageEvaluator()
+
+    def test_classify_missing_evidence(self, evaluator, mock_policy):
+        """Test classification when no evidence found"""
+        found = []
+        status = evaluator.classify_status(found, mock_policy, "2024-25")
+        assert status == Status.MISSING
+
+    def test_classify_verified_evidence(self, evaluator, mock_policy):
+        """Test classification of verified evidence"""
+        found = [
+            FoundEvidence(
+                doc_id="DOC-001",
+                kind="P60",
+                ocr_confidence=0.85,
+                extract_confidence=0.90,
+                date="2024-05-15T10:00:00Z",
+            )
+        ]
+
+        status = evaluator.classify_status(found, mock_policy, "2024-25")
+        assert status == Status.PRESENT_VERIFIED
+
+    def test_classify_unverified_evidence(self, evaluator, mock_policy):
+        """Test classification of unverified evidence"""
+        found = [
+            FoundEvidence(
+                doc_id="DOC-001",
+                kind="P60",
+                ocr_confidence=0.70,  # Below verified threshold
+                extract_confidence=0.75,  # Below verified threshold
+                date="2024-05-15T10:00:00Z",
+            )
+        ]
+
+        status = evaluator.classify_status(found, mock_policy, "2024-25")
+        assert status == Status.PRESENT_UNVERIFIED
+
+    def test_classify_low_confidence_evidence(self, evaluator, mock_policy):
+        """Test classification of very low confidence evidence"""
+        found = [
+            FoundEvidence(
+                doc_id="DOC-001",
+                kind="P60",
+                ocr_confidence=0.50,  # Below unverified threshold
+                extract_confidence=0.55,  # Below unverified threshold
+                date="2024-05-15T10:00:00Z",
+            )
+        ]
+
+        status = evaluator.classify_status(found, mock_policy, "2024-25")
+        assert status == Status.MISSING
+
+    def test_classify_conflicting_evidence(self, evaluator, mock_policy):
+        """Test classification when multiple conflicting documents found"""
+        found = [
+            FoundEvidence(
+                doc_id="DOC-001",
+                kind="P60",
+                ocr_confidence=0.85,
+                extract_confidence=0.90,
+                date="2024-05-15T10:00:00Z",
+            ),
+            FoundEvidence(
+                doc_id="DOC-002",
+                kind="P60",
+                ocr_confidence=0.85,
+                extract_confidence=0.90,
+                date="2024-05-20T10:00:00Z",
+            ),
+        ]
+
+        status = evaluator.classify_status(found, mock_policy, "2024-25")
+        assert status == Status.CONFLICTING
+
+    def test_classify_evidence_outside_tax_year(self, evaluator, mock_policy):
+        """Test classification of evidence outside tax year"""
+        found = [
+            FoundEvidence(
+                doc_id="DOC-001",
+                kind="P60",
+                ocr_confidence=0.85,
+                extract_confidence=0.90,
+                date="2023-03-15T10:00:00Z",  # Outside tax year
+            )
+        ]
+
+        status = evaluator.classify_status(found, mock_policy, "2024-25")
+        # Evidence outside tax year should be unverified even with high confidence
+        # This is correct business logic - date validation is part of verification
+        assert status == Status.PRESENT_UNVERIFIED
+
+    def test_classify_evidence_no_date(self, evaluator, mock_policy):
+        """Test classification of evidence without date"""
+        found = [
+            FoundEvidence(
+                doc_id="DOC-001",
+                kind="P60",
+                ocr_confidence=0.85,
+                extract_confidence=0.90,
+                date=None,
+            )
+        ]
+
+        status = evaluator.classify_status(found, mock_policy, "2024-25")
+        # Evidence without date cannot be fully verified, even with high confidence
+        # This is correct business logic - date validation is required for verification
+        assert status == Status.PRESENT_UNVERIFIED
+
+    def test_parse_tax_year_bounds(self, evaluator):
+        """Test parsing of tax year boundary strings"""
+        start_str = "2024-04-06"
+        end_str = "2025-04-05"
+
+        start, end = evaluator._parse_tax_year_bounds(start_str, end_str)
+
+        assert isinstance(start, datetime)
+        assert isinstance(end, datetime)
+        assert start.year == 2024
+        assert start.month == 4
+        assert start.day == 6
+        assert end.year == 2025
+        assert end.month == 4
+        assert end.day == 5
+
+    def test_evidence_within_tax_year(self, evaluator, mock_policy):
+        """Test evidence date validation within tax year"""
+        # Evidence within tax year
+        found = [
+            FoundEvidence(
+                doc_id="DOC-001",
+                kind="P60",
+                ocr_confidence=0.85,
+                extract_confidence=0.90,
+                date="2024-06-15T10:00:00Z",  # Within 2024-25 tax year
+            )
+        ]
+
+        status = evaluator.classify_status(found, mock_policy, "2024-25")
+        assert status == Status.PRESENT_VERIFIED
+
+    def test_evidence_boundary_dates(self, evaluator, mock_policy):
+        """Test evidence on tax year boundary dates"""
+        # Test start boundary
+        found_start = [
+            FoundEvidence(
+                doc_id="DOC-001",
+                kind="P60",
+                ocr_confidence=0.85,
+                extract_confidence=0.90,
+                date="2024-04-06T00:00:00Z",  # Exact start date
+            )
+        ]
+
+        status = evaluator.classify_status(found_start, mock_policy, "2024-25")
+        assert status == Status.PRESENT_VERIFIED
+
+        # Test end boundary
+        found_end = [
+            FoundEvidence(
+                doc_id="DOC-002",
+                kind="P60",
+                ocr_confidence=0.85,
+                extract_confidence=0.90,
+                date="2025-04-05T23:59:59Z",  # Exact end date
+            )
+        ]
+
+        status = evaluator.classify_status(found_end, mock_policy, "2024-25")
+        assert status == Status.PRESENT_VERIFIED
+
+    def test_threshold_edge_cases(self, evaluator, mock_policy):
+        """Test classification at threshold boundaries"""
+        # Exactly at verified threshold
+        found_exact = [
+            FoundEvidence(
+                doc_id="DOC-001",
+                kind="P60",
+                ocr_confidence=0.82,  # Exactly at threshold
+                extract_confidence=0.85,  # Exactly at threshold
+                date="2024-06-15T10:00:00Z",
+            )
+        ]
+
+        status = evaluator.classify_status(found_exact, mock_policy, "2024-25")
+        assert status == Status.PRESENT_VERIFIED
+
+        # Just below verified threshold
+        found_below = [
+            FoundEvidence(
+                doc_id="DOC-002",
+                kind="P60",
+                ocr_confidence=0.81,  # Just below threshold
+                extract_confidence=0.84,  # Just below threshold
+                date="2024-06-15T10:00:00Z",
+            )
+        ]
+
+        status = evaluator.classify_status(found_below, mock_policy, "2024-25")
+        assert status == Status.PRESENT_UNVERIFIED
+
+    def test_mixed_confidence_levels(self, evaluator, mock_policy):
+        """Test classification with mixed OCR and extract confidence"""
+        # High OCR, low extract
+        found_mixed1 = [
+            FoundEvidence(
+                doc_id="DOC-001",
+                kind="P60",
+                ocr_confidence=0.90,  # High
+                extract_confidence=0.70,  # Low
+                date="2024-06-15T10:00:00Z",
+            )
+        ]
+
+        status = evaluator.classify_status(found_mixed1, mock_policy, "2024-25")
+        assert status == Status.PRESENT_UNVERIFIED  # Both must meet threshold
+
+        # Low OCR, high extract
+        found_mixed2 = [
+            FoundEvidence(
+                doc_id="DOC-002",
+                kind="P60",
+                ocr_confidence=0.70,  # Low
+                extract_confidence=0.90,  # High
+                date="2024-06-15T10:00:00Z",
+            )
+        ]
+
+        status = evaluator.classify_status(found_mixed2, mock_policy, "2024-25")
+        assert status == Status.PRESENT_UNVERIFIED  # Both must meet threshold
+
+    def test_zero_confidence_evidence(self, evaluator, mock_policy):
+        """Test classification of zero confidence evidence"""
+        found = [
+            FoundEvidence(
+                doc_id="DOC-001",
+                kind="P60",
+                ocr_confidence=0.0,
+                extract_confidence=0.0,
+                date="2024-06-15T10:00:00Z",
+            )
+        ]
+
+        status = evaluator.classify_status(found, mock_policy, "2024-25")
+        assert status == Status.MISSING
+
+    def test_perfect_confidence_evidence(self, evaluator, mock_policy):
+        """Test classification of perfect confidence evidence"""
+        found = [
+            FoundEvidence(
+                doc_id="DOC-001",
+                kind="P60",
+                ocr_confidence=1.0,
+                extract_confidence=1.0,
+                date="2024-06-15T10:00:00Z",
+            )
+        ]
+
+        status = evaluator.classify_status(found, mock_policy, "2024-25")
+        assert status == Status.PRESENT_VERIFIED