Quotation denormalization

machine/corpora/__init__.py

@@ -7,6 +7,7 @@
 from .dbl_bundle_text_corpus import DblBundleTextCorpus
 from .dictionary_alignment_corpus import DictionaryAlignmentCorpus
 from .dictionary_text_corpus import DictionaryTextCorpus
+from .fallback_quotation_mark_resolver import FallbackQuotationMarkResolver
 from .file_paratext_project_settings_parser import FileParatextProjectSettingsParser
 from .file_paratext_project_text_updater import FileParatextProjectTextUpdater
 from .flatten import flatten
@@ -24,6 +25,13 @@
 from .paratext_project_text_updater_base import ParatextProjectTextUpdaterBase
 from .paratext_text_corpus import ParatextTextCorpus
 from .place_markers_usfm_update_block_handler import PlaceMarkersAlignmentInfo, PlaceMarkersUsfmUpdateBlockHandler
+from .quotation_mark_denormalization_first_pass import QuotationMarkDenormalizationFirstPass
+from .quotation_mark_denormalization_usfm_update_block_handler import QuotationMarkDenormalizationUsfmUpdateBlockHandler
+from .quotation_mark_update_first_pass import QuotationMarkUpdateFirstPass
+from .quotation_mark_update_resolution_settings import QuotationMarkUpdateResolutionSettings
+from .quotation_mark_update_settings import QuotationMarkUpdateSettings
+from .quotation_mark_update_strategy import QuotationMarkUpdateStrategy
+from .quote_convention_changing_usfm_update_block_handler import QuoteConventionChangingUsfmUpdateBlockHandler
 from .scripture_element import ScriptureElement
 from .scripture_ref import EMPTY_SCRIPTURE_REF, ScriptureRef
 from .scripture_ref_usfm_parser_handler import ScriptureRefUsfmParserHandler, ScriptureTextType
@@ -86,6 +94,7 @@
     "AlignmentCollection",
     "AlignmentCorpus",
     "AlignmentRow",
+    "FallbackQuotationMarkResolver",
     "batch",
     "Corpus",
     "create_versification_ref_corpus",
@@ -121,6 +130,13 @@
     "PlaceMarkersAlignmentInfo",
     "PlaceMarkersUsfmUpdateBlockHandler",
     "parse_usfm",
+    "QuoteConventionChangingUsfmUpdateBlockHandler",
+    "QuotationMarkUpdateResolutionSettings",
+    "QuotationMarkUpdateStrategy",
+    "QuotationMarkUpdateFirstPass",
+    "QuotationMarkDenormalizationFirstPass",
+    "QuotationMarkDenormalizationUsfmUpdateBlockHandler",
+    "QuotationMarkUpdateSettings",
     "RtlReferenceOrder",
     "ScriptureElement",
     "ScriptureRef",

machine/corpora/fallback_quotation_mark_resolver.py

@@ -0,0 +1,133 @@
+from typing import Generator, Optional, Set
+
+from .punctuation_analysis.quotation_mark_direction import QuotationMarkDirection
+from .punctuation_analysis.quotation_mark_metadata import QuotationMarkMetadata
+from .punctuation_analysis.quotation_mark_resolution_issue import QuotationMarkResolutionIssue
+from .punctuation_analysis.quotation_mark_resolution_settings import QuotationMarkResolutionSettings
+from .punctuation_analysis.quotation_mark_resolver import QuotationMarkResolver
+from .punctuation_analysis.quotation_mark_string_match import QuotationMarkStringMatch
+
+
+class FallbackQuotationMarkResolver(QuotationMarkResolver):
+
+    def __init__(self, settings: QuotationMarkResolutionSettings):
+        self._settings: QuotationMarkResolutionSettings = settings
+        self._last_quotation_mark: Optional[QuotationMarkMetadata] = None
+        self._issues: Set[QuotationMarkResolutionIssue] = set()
+
+    def reset(self) -> None:
+        self._last_quotation_mark = None
+        self._issues = set()
+
+    def resolve_quotation_marks(
+        self, quotation_mark_matches: list[QuotationMarkStringMatch]
+    ) -> Generator[QuotationMarkMetadata, None, None]:
+        for quotation_mark_match in quotation_mark_matches:
+            yield from self._resolve_quotation_mark(quotation_mark_match)
+
+    def _resolve_quotation_mark(
+        self,
+        quotation_mark_match: QuotationMarkStringMatch,
+    ) -> Generator[QuotationMarkMetadata, None, None]:
+        if self._is_opening_quotation_mark(quotation_mark_match):
+            quotation_mark: Optional[QuotationMarkMetadata] = self._resolve_opening_mark(quotation_mark_match)
+            if quotation_mark is not None:
+                yield quotation_mark
+            else:
+                self._issues.add(QuotationMarkResolutionIssue.UNEXPECTED_QUOTATION_MARK)
+        elif self._is_closing_quotation_mark(quotation_mark_match):
+            quotation_mark: Optional[QuotationMarkMetadata] = self._resolve_closing_mark(quotation_mark_match)
+            if quotation_mark is not None:
+                yield quotation_mark
+            else:
+                self._issues.add(QuotationMarkResolutionIssue.UNEXPECTED_QUOTATION_MARK)
+        else:
+            # Make a reasonable guess about the direction of the quotation mark
+            if (
+                self._last_quotation_mark is None
+                or self._last_quotation_mark.direction is QuotationMarkDirection.CLOSING
+            ):
+                quotation_mark: Optional[QuotationMarkMetadata] = self._resolve_opening_mark(quotation_mark_match)
+                if quotation_mark is not None:
+                    yield quotation_mark
+            else:
+                quotation_mark: Optional[QuotationMarkMetadata] = self._resolve_closing_mark(quotation_mark_match)
+                if quotation_mark is not None:
+                    yield quotation_mark
+
+            self._issues.add(QuotationMarkResolutionIssue.AMBIGUOUS_QUOTATION_MARK)
+
+    def _is_opening_quotation_mark(
+        self,
+        match: QuotationMarkStringMatch,
+    ) -> bool:
+
+        if self._settings.is_valid_opening_quotation_mark(match) and self._settings.is_valid_closing_quotation_mark(
+            match
+        ):
+            return (
+                match.is_at_start_of_segment()
+                or match.has_leading_whitespace()
+                or self._does_most_recent_opening_mark_immediately_precede(match)
+                or match.has_quote_introducer_in_leading_substring()
+            ) and not (match.has_trailing_whitespace() or match.has_trailing_punctuation())
+        elif self._settings.is_valid_opening_quotation_mark(match):
+            return True
+
+        return False
+
+    def _does_most_recent_opening_mark_immediately_precede(
+        self,
+        match: QuotationMarkStringMatch,
+    ) -> bool:
+        if (
+            self._last_quotation_mark is None
+            or self._last_quotation_mark.direction is not QuotationMarkDirection.OPENING
+        ):
+            return False
+
+        return (
+            self._last_quotation_mark.text_segment == match.text_segment
+            and self._last_quotation_mark.end_index == match.start_index
+        )
+
+    def _is_closing_quotation_mark(
+        self,
+        match: QuotationMarkStringMatch,
+    ) -> bool:
+
+        if self._settings.is_valid_opening_quotation_mark(match) and self._settings.is_valid_closing_quotation_mark(
+            match
+        ):
+            return (
+                match.has_trailing_whitespace() or match.has_trailing_punctuation() or match.is_at_end_of_segment()
+            ) and not match.has_leading_whitespace()
+        elif self._settings.is_valid_closing_quotation_mark(match):
+            return True
+
+        return False
+
+    def _resolve_opening_mark(self, quotation_mark_match: QuotationMarkStringMatch) -> Optional[QuotationMarkMetadata]:
+        possible_depths: Set[int] = self._settings.get_possible_depths(
+            quotation_mark_match.quotation_mark, QuotationMarkDirection.OPENING
+        )
+        if len(possible_depths) == 0:
+            return None
+
+        quotation_mark = quotation_mark_match.resolve(min(possible_depths), QuotationMarkDirection.OPENING)
+        self._last_quotation_mark = quotation_mark
+        return quotation_mark
+
+    def _resolve_closing_mark(self, quotation_mark_match: QuotationMarkStringMatch) -> Optional[QuotationMarkMetadata]:
+        possible_depths: Set[int] = self._settings.get_possible_depths(
+            quotation_mark_match.quotation_mark, QuotationMarkDirection.CLOSING
+        )
+        if len(possible_depths) == 0:
+            return None
+
+        quotation_mark = quotation_mark_match.resolve(min(possible_depths), QuotationMarkDirection.CLOSING)
+        self._last_quotation_mark = quotation_mark
+        return quotation_mark
+
+    def get_issues(self) -> Set[QuotationMarkResolutionIssue]:
+        return self._issues

machine/corpora/punctuation_analysis/__init__.py

@@ -0,0 +1,68 @@
+from .chapter import Chapter
+from .depth_based_quotation_mark_resolver import (
+    DepthBasedQuotationMarkResolver,
+    QuotationMarkCategorizer,
+    QuotationMarkResolverState,
+    QuoteContinuerState,
+    QuoteContinuerStyle,
+)
+from .preliminary_quotation_mark_analyzer import (
+    ApostropheProportionStatistics,
+    PreliminaryApostropheAnalyzer,
+    PreliminaryQuotationMarkAnalyzer,
+    QuotationMarkGrouper,
+    QuotationMarkSequences,
+    QuotationMarkWordPositions,
+)
+from .quotation_mark_direction import QuotationMarkDirection
+from .quotation_mark_finder import QuotationMarkFinder
+from .quotation_mark_metadata import QuotationMarkMetadata
+from .quotation_mark_resolution_issue import QuotationMarkResolutionIssue
+from .quotation_mark_resolution_settings import QuotationMarkResolutionSettings
+from .quotation_mark_resolver import QuotationMarkResolver
+from .quotation_mark_string_match import QuotationMarkStringMatch
+from .quotation_mark_tabulator import QuotationMarkCounts, QuotationMarkTabulator
+from .quote_convention import QuoteConvention, SingleLevelQuoteConvention
+from .quote_convention_detection_resolution_settings import QuoteConventionDetectionResolutionSettings
+from .quote_convention_detector import QuoteConventionAnalysis, QuoteConventionDetector
+from .quote_convention_set import QuoteConventionSet
+from .standard_quote_conventions import STANDARD_QUOTE_CONVENTIONS
+from .text_segment import TextSegment
+from .usfm_marker_type import UsfmMarkerType
+from .usfm_structure_extractor import UsfmStructureExtractor
+from .verse import Verse
+
+__all__ = [
+    "ApostropheProportionStatistics",
+    "Chapter",
+    "DepthBasedQuotationMarkResolver",
+    "PreliminaryApostropheAnalyzer",
+    "PreliminaryQuotationMarkAnalyzer",
+    "SingleLevelQuoteConvention",
+    "QuoteContinuerState",
+    "QuoteContinuerStyle",
+    "QuotationMarkCategorizer",
+    "QuotationMarkCounts",
+    "QuotationMarkDirection",
+    "QuotationMarkGrouper",
+    "QuotationMarkMetadata",
+    "QuotationMarkResolverState",
+    "QuotationMarkSequences",
+    "QuotationMarkStringMatch",
+    "QuotationMarkWordPositions",
+    "QuoteConvention",
+    "QuoteConventionAnalysis",
+    "QuoteConventionDetectionResolutionSettings",
+    "QuotationMarkFinder",
+    "QuotationMarkResolutionIssue",
+    "QuotationMarkResolutionSettings",
+    "QuotationMarkResolver",
+    "QuotationMarkTabulator",
+    "QuoteConventionDetector",
+    "QuoteConventionSet",
+    "STANDARD_QUOTE_CONVENTIONS",
+    "TextSegment",
+    "UsfmMarkerType",
+    "UsfmStructureExtractor",
+    "Verse",
+]

machine/corpora/punctuation_analysis/chapter.py

@@ -0,0 +1,8 @@
+from dataclasses import dataclass
+
+from .verse import Verse
+
+
+@dataclass(frozen=True)
+class Chapter:
+    verses: list[Verse]