diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs index a982edd1..9335d920 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs @@ -42,6 +42,14 @@ public IEnumerable Apply(Word input) _morpher.TraceManager.BeginUnapplyTemplate(_template, input); Word inWord = input.Clone(); + if ( + (!_morpher.IsPartial || _morpher.AlwaysEnforceFinalTemplates) + && inWord.FinalTemplateState == FinalTemplateState.NonTemplate + && _template.IsFinal + ) + { + inWord.FinalTemplateState = FinalTemplateState.FinalTemplateAfterNonTemplate; + } inWord.Freeze(); var output = new HashSet(FreezableEqualityComparer.Default); diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisStateKey.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStateKey.cs index 8575ff40..634dfee0 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisStateKey.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStateKey.cs @@ -12,9 +12,11 @@ namespace SIL.Machine.Morphology.HermitCrab /// re-run whenever an Analysis*.cs rule changes: /// /// : Shape (FST pattern match), - /// (unifiability gate), per-rule unapplication count. + /// (unifiability gate), per-rule unapplication count + /// and FinalTemplateState. /// : adds - /// (MaxStemCount gate) -- never the non-heads' own content, only the count. + /// (MaxStemCount gate) -- never the non-heads' own content, only the count + /// and FinalTemplateState. /// : adds /// . /// @@ -31,6 +33,7 @@ namespace SIL.Machine.Morphology.HermitCrab private readonly FeatureStruct _realizationalFS; private readonly int _nonHeadCount; private readonly IReadOnlyDictionary _ruleCounts; + private readonly FinalTemplateState _finalTemplateState; private readonly int _hashCode; /// @@ -62,6 +65,7 @@ private AnalysisStateKey(Word word) _realizationalFS = word.RealizationalFeatureStruct; _nonHeadCount = word.NonHeadCount; _ruleCounts = word.UnappliedRuleCounts; + _finalTemplateState = word.FinalTemplateState; // See PinAndKey for why the key pins these rather than just reading them. _shape.Freeze(); @@ -74,6 +78,7 @@ private AnalysisStateKey(Word word) hash = hash * 31 + _syntacticFS.GetFrozenHashCode(); hash = hash * 31 + _realizationalFS.GetFrozenHashCode(); hash = hash * 31 + _nonHeadCount; + hash = hash * 31 + _finalTemplateState.GetHashCode(); if (_ruleCounts != null) { // XOR rather than the usual *31 rolling combine: the multiset is unordered, so entries @@ -100,6 +105,8 @@ public bool Equals(AnalysisStateKey other) return false; if (!_syntacticFS.ValueEquals(other._syntacticFS) || !_realizationalFS.ValueEquals(other._realizationalFS)) return false; + if (_finalTemplateState != other._finalTemplateState) + return false; return RuleCountsEqual(_ruleCounts, other._ruleCounts); } diff --git a/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs index 48bc2474..dcac50b9 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs @@ -161,9 +161,17 @@ internal IEnumerable Apply(Word input, ref int alternativeCount) } shapeWord[shape] = mruleOutWord; } - output.Add(mruleOutWord); + Word newMruleOutWord = mruleOutWord; + if (mruleOutWord.FinalTemplateState != FinalTemplateState.None) + { + // Clear FinalTemplateState to allow clitics. + newMruleOutWord = mruleOutWord.Clone(); + newMruleOutWord.FinalTemplateState = FinalTemplateState.None; + newMruleOutWord.Freeze(); + } + output.Add(newMruleOutWord); if (_morpher.TraceManager.IsTracing) - _morpher.TraceManager.EndUnapplyStratum(_stratum, mruleOutWord); + _morpher.TraceManager.EndUnapplyStratum(_stratum, newMruleOutWord); } return output; } diff --git a/src/SIL.Machine.Morphology.HermitCrab/FinalTemplateState.cs b/src/SIL.Machine.Morphology.HermitCrab/FinalTemplateState.cs new file mode 100644 index 00000000..0af491af --- /dev/null +++ b/src/SIL.Machine.Morphology.HermitCrab/FinalTemplateState.cs @@ -0,0 +1,9 @@ +namespace SIL.Machine.Morphology.HermitCrab +{ + enum FinalTemplateState : byte + { + None, + NonTemplate, + FinalTemplateAfterNonTemplate, + } +} diff --git a/src/SIL.Machine.Morphology.HermitCrab/ITraceManager.cs b/src/SIL.Machine.Morphology.HermitCrab/ITraceManager.cs index 31d8fa49..eda5381b 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/ITraceManager.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/ITraceManager.cs @@ -47,6 +47,13 @@ public interface ITraceManager void MorphologicalRuleUnapplied(IMorphologicalRule rule, int subruleIndex, Word input, Word output); void MorphologicalRuleNotUnapplied(IMorphologicalRule rule, int subruleIndex, Word input); + void MorphologicalRuleNotUnapplied( + IMorphologicalRule rule, + int subruleIndex, + Word input, + FailureReason reason, + object failureObj + ); void CompoundingRuleNotUnapplied( IMorphologicalRule rule, diff --git a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs index 54ac7b5e..57ada613 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Morpher.cs @@ -65,6 +65,18 @@ public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParalle RuleSelector = rule => true; _morphemes = new ReadOnlyObservableCollection(morphemes); + IsPartial = GetPartialMorphemes().Count() > 0; + } + + public IEnumerable GetPartialMorphemes() + { + var morphemes = new HashSet(); + foreach (Morpheme morpheme in _morphemes) + { + if (morpheme.IsPartial) + morphemes.Add(morpheme); + } + return morphemes; } public ITraceManager TraceManager @@ -88,6 +100,16 @@ public ITraceManager TraceManager /// public bool MergeEquivalentAnalyses { get; set; } + /// + /// A Morpher is partial if any of the elements are partial. + /// + public bool IsPartial { get; set; } + + /// + /// Enforce final templates even if some of the morphemes were partial. + /// + public bool AlwaysEnforceFinalTemplates { get; set; } + /// /// Caps the concurrency used within a single parse or generation -- analysis cascade, /// affix-template unapplication and synthesis alike. A value of 1 runs the work fully diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs index 4e89fef9..1eb58fe6 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs @@ -49,6 +49,26 @@ public IEnumerable Apply(Word input) return Enumerable.Empty(); } + // Do not allow a final template to unapply if the grammar is not partial + // and a non-template was last unapplied. + if ( + (!_morpher.IsPartial || _morpher.AlwaysEnforceFinalTemplates) + && input.FinalTemplateState == FinalTemplateState.FinalTemplateAfterNonTemplate + ) + { + if (_morpher.TraceManager.IsTracing) + { + _morpher.TraceManager.MorphologicalRuleNotUnapplied( + _rule, + -1, + input, + FailureReason.NonPartialRuleProhibitedAfterFinalTemplate, + null + ); + } + return Enumerable.Empty(); + } + var output = new List(); for (int i = 0; i < _rules.Count; i++) { @@ -59,6 +79,9 @@ public IEnumerable Apply(Word input) outWord.SyntacticFeatureStruct.Add(_rule.RequiredSyntacticFeatureStruct); else if (_rule.OutSyntacticFeatureStruct.IsEmpty) outWord.SyntacticFeatureStruct.Clear(); + outWord.FinalTemplateState = !_rule.IsTemplateRule + ? FinalTemplateState.NonTemplate + : FinalTemplateState.None; outWord.MorphologicalRuleUnapplied(_rule); outWord.Freeze(); if (_morpher.TraceManager.IsTracing) diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs index 6dc2a0c2..72da794b 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs @@ -130,6 +130,7 @@ RootAllomorph allo in _morpher.SearchRootAllomorphs(_rule.Stratum, outWord.Curre outWord.SyntacticFeatureStruct.Add(_rule.HeadRequiredSyntacticFeatureStruct); else if (_rule.OutSyntacticFeatureStruct.IsEmpty) outWord.SyntacticFeatureStruct.Clear(); + outWord.FinalTemplateState = FinalTemplateState.NonTemplate; outWord.MorphologicalRuleUnapplied(_rule); outWord.Freeze(); diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs index 749aa7b4..0c4dc93f 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs @@ -46,6 +46,26 @@ public IEnumerable Apply(Word input) if (!_rule.RealizationalFeatureStruct.Unify(input.RealizationalFeatureStruct, out realFS)) return Enumerable.Empty(); + // Do not allow a final template to unapply if the grammar is not partial + // and a non-template was last unapplied. + if ( + (!_morpher.IsPartial || _morpher.AlwaysEnforceFinalTemplates) + && input.FinalTemplateState == FinalTemplateState.FinalTemplateAfterNonTemplate + ) + { + if (_morpher.TraceManager.IsTracing) + { + _morpher.TraceManager.MorphologicalRuleNotUnapplied( + _rule, + -1, + input, + FailureReason.NonPartialRuleProhibitedAfterFinalTemplate, + null + ); + } + return Enumerable.Empty(); + } + var output = new List(); for (int i = 0; i < _rules.Count; i++) { diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs index f7dc9c0d..0b40d5a6 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs @@ -63,8 +63,7 @@ public IEnumerable Apply(Word input) if ( !_rule.IsTemplateRule && (input.IsLastAppliedRuleFinal ?? false) - && !input.IsPartial - && !_rule.IsPartial + && ((!input.IsPartial && !_rule.IsPartial) || _morpher.AlwaysEnforceFinalTemplates) ) { if (_morpher.TraceManager.IsTracing) diff --git a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisCompoundingRule.cs b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisCompoundingRule.cs index a8f16e65..d2d6b9f5 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisCompoundingRule.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisCompoundingRule.cs @@ -62,7 +62,7 @@ public IEnumerable Apply(Word input) return Enumerable.Empty(); } - if ((input.IsLastAppliedRuleFinal ?? false) && !input.IsPartial) + if ((input.IsLastAppliedRuleFinal ?? false) && (!input.IsPartial || _morpher.AlwaysEnforceFinalTemplates)) { if (_morpher.TraceManager.IsTracing) { diff --git a/src/SIL.Machine.Morphology.HermitCrab/TraceManager.cs b/src/SIL.Machine.Morphology.HermitCrab/TraceManager.cs index 56a4119a..0335bcbe 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/TraceManager.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/TraceManager.cs @@ -84,6 +84,24 @@ public void MorphologicalRuleNotUnapplied(IMorphologicalRule rule, int subruleIn ); } + public void MorphologicalRuleNotUnapplied( + IMorphologicalRule rule, + int subruleIndex, + Word input, + FailureReason reason, + object failureObj + ) + { + ((Trace)input.CurrentTrace).Children.Add( + new Trace(TraceType.MorphologicalRuleAnalysis, rule) + { + SubruleIndex = subruleIndex, + Input = input, + FailureReason = reason, + } + ); + } + public void CompoundingRuleNotUnapplied( IMorphologicalRule rule, int subruleIndex, diff --git a/src/SIL.Machine.Morphology.HermitCrab/Word.cs b/src/SIL.Machine.Morphology.HermitCrab/Word.cs index 51c3c53c..931b35b3 100644 --- a/src/SIL.Machine.Morphology.HermitCrab/Word.cs +++ b/src/SIL.Machine.Morphology.HermitCrab/Word.cs @@ -28,6 +28,7 @@ public class Word : Freezable, IAnnotatedData, ICloneable private FeatureStruct _realizationalFS; private Stratum _stratum; private bool? _isLastAppliedRuleFinal; + private FinalTemplateState _finalTemplateState; private bool _isPartial; private readonly Dictionary> _disjunctiveAllomorphIndices; private int _mruleAppCount = 0; @@ -47,6 +48,7 @@ public Word(RootAllomorph rootAllomorph, FeatureStruct realizationalFS) _nonHeadApps = new List(); _obligatorySyntacticFeatures = new IDBearerSet(); _isLastAppliedRuleFinal = null; + _finalTemplateState = FinalTemplateState.None; _disjunctiveAllomorphIndices = new Dictionary>(); } @@ -65,6 +67,7 @@ public Word(Stratum stratum, Shape shape) _nonHeadApps = new List(); _obligatorySyntacticFeatures = new IDBearerSet(); _isLastAppliedRuleFinal = null; + _finalTemplateState = FinalTemplateState.None; _isPartial = false; _disjunctiveAllomorphIndices = new Dictionary>(); } @@ -93,6 +96,7 @@ private Word(Word word, bool cloneNonHeadApps) _nonHeadAppIndex = word._nonHeadAppIndex; _obligatorySyntacticFeatures = new IDBearerSet(word._obligatorySyntacticFeatures); _isLastAppliedRuleFinal = word._isLastAppliedRuleFinal; + _finalTemplateState = word._finalTemplateState; _isPartial = word._isPartial; CurrentTrace = word.CurrentTrace; AnalysisScope = word.AnalysisScope; @@ -392,6 +396,16 @@ internal bool? IsLastAppliedRuleFinal } } + internal FinalTemplateState FinalTemplateState + { + get { return _finalTemplateState; } + set + { + CheckFrozen(); + _finalTemplateState = value; + } + } + /// /// Gets the number of times the specified morphological rule has been applied. /// @@ -621,6 +635,7 @@ protected override int FreezeImpl() code = code * 31 + _mruleApps.GetSequenceHashCode(); code = code * 31 + _mruleAppIndex.GetHashCode(); code = code * 31 + _isLastAppliedRuleFinal.GetHashCode(); + code = code * 31 + _finalTemplateState.GetHashCode(); return code; } @@ -640,7 +655,8 @@ public override bool ValueEquals(Word other) && _rootAllomorph == other._rootAllomorph && _mruleApps.SequenceEqual(other._mruleApps) && _mruleAppIndex == other._mruleAppIndex - && _isLastAppliedRuleFinal == other._isLastAppliedRuleFinal; + && _isLastAppliedRuleFinal == other._isLastAppliedRuleFinal + && _finalTemplateState == other._finalTemplateState; } public Word Clone() diff --git a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AffixTemplateTests.cs b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AffixTemplateTests.cs index 54786c82..696d031f 100644 --- a/tests/SIL.Machine.Morphology.HermitCrab.Tests/AffixTemplateTests.cs +++ b/tests/SIL.Machine.Morphology.HermitCrab.Tests/AffixTemplateTests.cs @@ -348,6 +348,163 @@ public void NonFinalTemplate() AssertMorphsEqual(morpher.ParseWord("sagdmis"), "32 PAST 53 PL"); } + [Test] + public void EarlyPruningOfFinalTemplate() + { + var any = FeatureStruct.New().Symbol(HCFeatureSystem.Segment).Value; + var alvStop = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("cons+") + .Symbol("strident-") + .Symbol("del_rel-") + .Symbol("alveolar") + .Value; + var voicelessCons = FeatureStruct + .New(Language.PhonologicalFeatureSystem) + .Symbol(HCFeatureSystem.Segment) + .Symbol("cons+") + .Symbol("vd-") + .Value; + + var edSuffix = new AffixProcessRule { Name = "ed_suffix", Gloss = "PAST" }; + edSuffix.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = + { + Pattern.New("1").Annotation(any).OneOrMore.Value, + Pattern.New("2").Annotation(alvStop).Value, + }, + Rhs = { new CopyFromInput("1"), new CopyFromInput("2"), new InsertSegments(Table3, "ɯd") }, + } + ); + edSuffix.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Annotation(voicelessCons).Value }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "t") }, + } + ); + edSuffix.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "d") }, + } + ); + + var verbTemplate = new AffixTemplate + { + Name = "verb", + RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + }; + verbTemplate.Slots.Add(new AffixTemplateSlot(edSuffix)); + Morphophonemic.AffixTemplates.Add(verbTemplate); + + var nominalizer = new AffixProcessRule + { + Name = "nominalizer", + Gloss = "NOM", + RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + OutSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("N").Value, + }; + nominalizer.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "v") }, + } + ); + Morphophonemic.MorphologicalRules.Add(nominalizer); + + var crule = new CompoundingRule + { + Name = "rule1", + HeadRequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("V").Value, + NonHeadRequiredSyntacticFeatureStruct = FeatureStruct + .New(Language.SyntacticFeatureSystem) + .Symbol("N") + .Value, + OutSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("N").Value, + }; + crule.Subrules.Add( + new CompoundingSubrule + { + HeadLhs = { Pattern.New("head").Annotation(any).OneOrMore.Value }, + NonHeadLhs = { Pattern.New("nonHead").Annotation(any).OneOrMore.Value }, + Rhs = { new CopyFromInput("head"), new InsertSegments(Table3, "+"), new CopyFromInput("nonHead") }, + } + ); + Morphophonemic.MorphologicalRules.Add(crule); + + var sSuffix = new AffixProcessRule + { + Name = "s_suffix", + Gloss = "PL", + RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("N").Value, + }; + sSuffix.Allomorphs.Add( + new AffixProcessAllomorph + { + Lhs = { Pattern.New("1").Annotation(any).OneOrMore.Value }, + Rhs = { new CopyFromInput("1"), new InsertSegments(Table3, "s") }, + } + ); + + var nounTemplate = new AffixTemplate + { + Name = "noun", + RequiredSyntacticFeatureStruct = FeatureStruct.New(Language.SyntacticFeatureSystem).Symbol("N").Value, + }; + nounTemplate.Slots.Add(new AffixTemplateSlot(sSuffix) { Optional = true }); + Morphophonemic.AffixTemplates.Add(nounTemplate); + + // Verify early pruning of final template. + var morpher = new Morpher(TraceManager, Language); + morpher.IsPartial = false; // Override for testing purposes. + TraceManager.IsTracing = true; + AssertMorphsEqual(morpher.ParseWord("sagdv", out object trace)); + Assert.That( + GetFailureDepth((Trace)trace, FailureReason.NonPartialRuleProhibitedAfterFinalTemplate), + Is.EqualTo(2) + ); + AssertMorphsEqual(morpher.ParseWord("sagdvs")); + TraceManager.IsTracing = false; + + // Verify correctness when non-partial and final. + AssertMorphsEqual(morpher.ParseWord("sagd"), "32 PAST"); + AssertMorphsEqual(morpher.ParseWord("sagdv")); + AssertMorphsEqual(morpher.ParseWord("sagdvs")); + AssertMorphsEqual(morpher.ParseWord("sagdmi")); + AssertMorphsEqual(morpher.ParseWord("sagdmis")); + + // Verify correctness when non-partial and non-final. + verbTemplate.IsFinal = false; + morpher = new Morpher(TraceManager, Language); + morpher.IsPartial = false; + AssertMorphsEqual(morpher.ParseWord("sagd")); + AssertMorphsEqual(morpher.ParseWord("sagdv"), "32 PAST NOM"); + AssertMorphsEqual(morpher.ParseWord("sagdvs"), "32 PAST NOM PL"); + AssertMorphsEqual(morpher.ParseWord("sagdmi"), "32 PAST 53"); + AssertMorphsEqual(morpher.ParseWord("sagdmis"), "32 PAST 53 PL"); + } + + private static int GetFailureDepth(Trace trace, FailureReason reason) + { + if (trace == null) + return 0; + if (trace.FailureReason == reason) + return trace.Depth; + foreach (var child in trace.Children) + { + int depth = GetFailureDepth(child, reason); + if (depth > 0) + return depth; + } + return 0; + } + [Test] public void AffixTemplateAppliedAfterMorphologicalRule() {