DeutschLernen/GermanApp/Application/Services/SpeechExerciseService.cs
Lasse Rune Hansen e002868b74 feat(backend/application): implement Phase 5 AI Service Integration
- Create higher-level AI services:
  - StoryGenerationService (uses MistralService)
  - WritingFeedbackService (uses MistralService)
  - SpeechExerciseService (uses VoskService)
  - AudioGenerationService (uses TtsService)
  - AiFallbackService (fallback mechanisms for service failures)
- Register AiFallbackService in Program.cs DI container
- Add comprehensive unit tests for all Phase 5 services:
  - AiFallbackServiceTests (14 tests)
  - AudioGenerationServiceTests (16 tests)
  - MistralServiceTests (16 tests)
  - SpeechExerciseServiceTests (12 tests)
  - StoryGenerationServiceTests (13 tests)
  - WritingFeedbackServiceTests (11 tests)
  - VoskServiceTests (15 tests)
  - TtsServiceTests (21 tests)
- Update feature document (ai-services.md) to mark Phase 5 as complete

Generated by Mistral Vibe.
Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
2026-06-13 12:44:10 +02:00

428 lines
14 KiB
C#

using GermanApp.Domain.Interfaces;
using Microsoft.Extensions.Logging;
namespace GermanApp.Application.Services;
/// <summary>
/// Application service for managing speech exercises using Vosk speech recognition.
/// This is part of the Application layer.
/// </summary>
public class SpeechExerciseService
{
private readonly IVoskService _voskService;
private readonly ILogger<SpeechExerciseService> _logger;
/// <summary>
/// Creates a new SpeechExerciseService.
/// </summary>
/// <param name="voskService">The Vosk speech recognition service</param>
/// <param name="logger">Logger for service operations</param>
public SpeechExerciseService(
IVoskService voskService,
ILogger<SpeechExerciseService> logger)
{
_voskService = voskService;
_logger = logger;
}
/// <summary>
/// Recognizes speech from audio bytes and returns the transcribed text.
/// </summary>
/// <param name="audioBytes">The audio data in bytes</param>
/// <param name="expectedText">Optional expected text for verification</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>Result containing recognized text and accuracy</returns>
public virtual async Task<SpeechRecognitionResult> RecognizeSpeechAsync(
byte[] audioBytes,
string? expectedText = null,
CancellationToken cancellationToken = default)
{
_logger.LogInformation("Recognizing speech from audio: {ByteCount} bytes", audioBytes?.Length ?? 0);
try
{
var recognizedText = await _voskService.RecognizeSpeechAsync(
audioBytes,
_voskService.GetType().Name == "VoskService" ? 16000 : 16000,
null,
cancellationToken);
// Calculate accuracy if expected text is provided
var accuracy = expectedText != null
? CalculateAccuracy(recognizedText, expectedText)
: 0.0;
_logger.LogInformation(
"Speech recognized: Text={Text}, Accuracy={Accuracy:P0}",
recognizedText, accuracy);
return new SpeechRecognitionResult
{
RecognizedText = recognizedText,
ExpectedText = expectedText,
Accuracy = accuracy,
IsCorrect = accuracy >= 0.8
};
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to recognize speech");
throw new AiServiceException(
"Failed to recognize speech: " + ex.Message,
AiErrorCode.Temporary);
}
}
/// <summary>
/// Recognizes speech from an audio file.
/// </summary>
/// <param name="audioFilePath">Path to the audio file</param>
/// <param name="expectedText">Optional expected text for verification</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>Result containing recognized text and accuracy</returns>
public virtual async Task<SpeechRecognitionResult> RecognizeSpeechFromFileAsync(
string audioFilePath,
string? expectedText = null,
CancellationToken cancellationToken = default)
{
_logger.LogInformation("Recognizing speech from file: {FilePath}", audioFilePath);
try
{
var recognizedText = await _voskService.RecognizeSpeechFromFileAsync(
audioFilePath,
cancellationToken);
var accuracy = expectedText != null
? CalculateAccuracy(recognizedText, expectedText)
: 0.0;
return new SpeechRecognitionResult
{
RecognizedText = recognizedText,
ExpectedText = expectedText,
Accuracy = accuracy,
IsCorrect = accuracy >= 0.8
};
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to recognize speech from file");
throw;
}
}
/// <summary>
/// Verifies if spoken audio matches expected text.
/// </summary>
/// <param name="audioBytes">The audio data in bytes</param>
/// <param name="expectedText">The expected text</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>Result with accuracy and correctness</returns>
public virtual async Task<SpeechRecognitionResult> VerifySpeechAsync(
byte[] audioBytes,
string expectedText,
CancellationToken cancellationToken = default)
{
_logger.LogInformation("Verifying speech: Expected={Expected}", expectedText);
var result = await RecognizeSpeechAsync(audioBytes, expectedText, cancellationToken);
// Additional validation for speech exercises
result.PronunciationScore = CalculatePronunciationScore(result.RecognizedText, expectedText);
result.FluencyScore = CalculateFluencyScore(audioBytes);
return result;
}
/// <summary>
/// Creates a speech exercise for practicing a specific phrase.
/// </summary>
/// <param name="phrase">The phrase to practice</param>
/// <param name="difficulty">Exercise difficulty level</param>
/// <param name="hints">Optional pronunciation hints</param>
/// <returns>Speech exercise with metadata</returns>
public virtual SpeechExercise CreateExercise(
string phrase,
string difficulty = "medium",
IReadOnlyList<string>? hints = null)
{
_logger.LogInformation("Creating speech exercise: Phrase={Phrase}, Difficulty={Difficulty}", phrase, difficulty);
return new SpeechExercise
{
Id = Guid.NewGuid().ToString(),
Phrase = phrase,
Difficulty = difficulty,
Hints = hints ?? new List<string>(),
CreatedAt = DateTime.UtcNow
};
}
/// <summary>
/// Evaluates a user's attempt at a speech exercise.
/// </summary>
/// <param name="exerciseId">The exercise ID</param>
/// <param name="audioBytes">The user's audio attempt</param>
/// <param name="expectedPhrase">The expected phrase (from exercise)</param>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>Evaluation result with scores</returns>
public virtual async Task<SpeechExerciseEvaluation> EvaluateExerciseAttemptAsync(
string exerciseId,
byte[] audioBytes,
string expectedPhrase,
CancellationToken cancellationToken = default)
{
_logger.LogInformation("Evaluating exercise attempt: ExerciseId={ExerciseId}", exerciseId);
var result = await VerifySpeechAsync(audioBytes, expectedPhrase, cancellationToken);
return new SpeechExerciseEvaluation
{
ExerciseId = exerciseId,
ExpectedPhrase = expectedPhrase,
RecognizedText = result.RecognizedText,
Accuracy = result.Accuracy,
PronunciationScore = result.PronunciationScore,
FluencyScore = result.FluencyScore,
IsPassed = result.IsCorrect && result.Accuracy >= 0.8,
AttemptedAt = DateTime.UtcNow
};
}
/// <summary>
/// Tests the speech exercise service.
/// </summary>
/// <param name="cancellationToken">Cancellation token</param>
/// <returns>True if service is working, false otherwise</returns>
public virtual async Task<bool> TestServiceAsync(CancellationToken cancellationToken = default)
{
try
{
// We can't test without actual audio, so just verify the service is initialized
var modelInfo = await _voskService.GetModelInfoAsync();
return modelInfo.ModelName != null;
}
catch (Exception ex)
{
_logger.LogError(ex, "Speech exercise service test failed");
return false;
}
}
/// <summary>
/// Calculates text matching accuracy between recognized and expected text.
/// </summary>
/// <param name="recognized">The recognized text</param>
/// <param name="expected">The expected text</param>
/// <returns>Accuracy score (0.0 to 1.0)</returns>
private double CalculateAccuracy(string recognized, string expected)
{
if (string.IsNullOrWhiteSpace(expected))
return 1.0;
if (string.IsNullOrWhiteSpace(recognized))
return 0.0;
// Normalize both strings
var recognizedNormalized = recognized.Trim().ToLower();
var expectedNormalized = expected.Trim().ToLower();
// Simple exact match
if (recognizedNormalized == expectedNormalized)
return 1.0;
// Calculate Levenshtein distance for similarity
var distance = LevenshteinDistance(recognizedNormalized, expectedNormalized);
var maxLength = Math.Max(recognizedNormalized.Length, expectedNormalized.Length);
if (maxLength == 0)
return 1.0;
var similarity = 1.0 - (distance / (double)maxLength);
return Math.Max(0.0, Math.Min(1.0, similarity));
}
/// <summary>
/// Calculates pronunciation score based on text matching.
/// </summary>
private double CalculatePronunciationScore(string recognized, string expected)
{
// For now, use accuracy as pronunciation score
// In a real implementation, you might use audio analysis
return CalculateAccuracy(recognized, expected);
}
/// <summary>
/// Calculates fluency score based on audio characteristics.
/// </summary>
private double CalculateFluencyScore(byte[] audioBytes)
{
// Simple heuristic: longer audio with consistent speaking rate
// In a real implementation, analyze audio features
if (audioBytes == null || audioBytes.Length == 0)
return 0.0;
// Assume 16kHz sample rate, 2 bytes per sample (16-bit)
var durationSeconds = audioBytes.Length / (16000.0 * 2);
// Give higher score for longer, consistent speech
if (durationSeconds >= 3.0)
return 1.0;
if (durationSeconds >= 1.5)
return 0.8;
if (durationSeconds >= 0.5)
return 0.6;
return 0.4;
}
/// <summary>
/// Calculates Levenshtein distance between two strings.
/// </summary>
private static int LevenshteinDistance(string s, string t)
{
if (string.IsNullOrEmpty(s))
{
return string.IsNullOrEmpty(t) ? 0 : t.Length;
}
if (string.IsNullOrEmpty(t))
{
return s.Length;
}
var n = s.Length;
var m = t.Length;
var d = new int[n + 1, m + 1];
for (int i = 0; i <= n; d[i, 0] = i++)
;
for (int j = 1; j <= m; d[0, j] = j++)
;
for (int i = 1; i <= n; i++)
{
for (int j = 1; j <= m; j++)
{
var cost = (t[j - 1] == s[i - 1]) ? 0 : 1;
d[i, j] = Math.Min(
Math.Min(d[i - 1, j] + 1, d[i, j - 1] + 1),
d[i - 1, j - 1] + cost);
}
}
return d[n, m];
}
}
/// <summary>
/// Result of speech recognition.
/// </summary>
public class SpeechRecognitionResult
{
/// <summary>
/// The recognized text.
/// </summary>
public string RecognizedText { get; set; } = string.Empty;
/// <summary>
/// The expected text (if provided for verification).
/// </summary>
public string? ExpectedText { get; set; }
/// <summary>
/// Accuracy score (0.0 to 1.0).
/// </summary>
public double Accuracy { get; set; }
/// <summary>
/// Pronunciation score (0.0 to 1.0).
/// </summary>
public double PronunciationScore { get; set; }
/// <summary>
/// Fluency score (0.0 to 1.0).
/// </summary>
public double FluencyScore { get; set; }
/// <summary>
/// Whether the recognition is considered correct.
/// </summary>
public bool IsCorrect { get; set; }
}
/// <summary>
/// Represents a speech exercise.
/// </summary>
public class SpeechExercise
{
/// <summary>
/// Unique identifier for the exercise.
/// </summary>
public string Id { get; set; } = string.Empty;
/// <summary>
/// The phrase to practice.
/// </summary>
public string Phrase { get; set; } = string.Empty;
/// <summary>
/// Difficulty level (easy, medium, hard).
/// </summary>
public string Difficulty { get; set; } = "medium";
/// <summary>
/// Pronunciation hints.
/// </summary>
public IReadOnlyList<string> Hints { get; set; } = new List<string>();
/// <summary>
/// When the exercise was created.
/// </summary>
public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
}
/// <summary>
/// Result of evaluating a speech exercise attempt.
/// </summary>
public class SpeechExerciseEvaluation
{
/// <summary>
/// The exercise ID.
/// </summary>
public string ExerciseId { get; set; } = string.Empty;
/// <summary>
/// The expected phrase.
/// </summary>
public string ExpectedPhrase { get; set; } = string.Empty;
/// <summary>
/// The recognized text.
/// </summary>
public string RecognizedText { get; set; } = string.Empty;
/// <summary>
/// Accuracy score (0.0 to 1.0).
/// </summary>
public double Accuracy { get; set; }
/// <summary>
/// Pronunciation score (0.0 to 1.0).
/// </summary>
public double PronunciationScore { get; set; }
/// <summary>
/// Fluency score (0.0 to 1.0).
/// </summary>
public double FluencyScore { get; set; }
/// <summary>
/// Whether the attempt passed.
/// </summary>
public bool IsPassed { get; set; }
/// <summary>
/// When the attempt was made.
/// </summary>
public DateTime AttemptedAt { get; set; } = DateTime.UtcNow;
}