- Create higher-level AI services: - StoryGenerationService (uses MistralService) - WritingFeedbackService (uses MistralService) - SpeechExerciseService (uses VoskService) - AudioGenerationService (uses TtsService) - AiFallbackService (fallback mechanisms for service failures) - Register AiFallbackService in Program.cs DI container - Add comprehensive unit tests for all Phase 5 services: - AiFallbackServiceTests (14 tests) - AudioGenerationServiceTests (16 tests) - MistralServiceTests (16 tests) - SpeechExerciseServiceTests (12 tests) - StoryGenerationServiceTests (13 tests) - WritingFeedbackServiceTests (11 tests) - VoskServiceTests (15 tests) - TtsServiceTests (21 tests) - Update feature document (ai-services.md) to mark Phase 5 as complete Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe <vibe@mistral.ai>
428 lines
14 KiB
C#
428 lines
14 KiB
C#
using GermanApp.Domain.Interfaces;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace GermanApp.Application.Services;
|
|
|
|
/// <summary>
|
|
/// Application service for managing speech exercises using Vosk speech recognition.
|
|
/// This is part of the Application layer.
|
|
/// </summary>
|
|
public class SpeechExerciseService
|
|
{
|
|
private readonly IVoskService _voskService;
|
|
private readonly ILogger<SpeechExerciseService> _logger;
|
|
|
|
/// <summary>
|
|
/// Creates a new SpeechExerciseService.
|
|
/// </summary>
|
|
/// <param name="voskService">The Vosk speech recognition service</param>
|
|
/// <param name="logger">Logger for service operations</param>
|
|
public SpeechExerciseService(
|
|
IVoskService voskService,
|
|
ILogger<SpeechExerciseService> logger)
|
|
{
|
|
_voskService = voskService;
|
|
_logger = logger;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Recognizes speech from audio bytes and returns the transcribed text.
|
|
/// </summary>
|
|
/// <param name="audioBytes">The audio data in bytes</param>
|
|
/// <param name="expectedText">Optional expected text for verification</param>
|
|
/// <param name="cancellationToken">Cancellation token</param>
|
|
/// <returns>Result containing recognized text and accuracy</returns>
|
|
public virtual async Task<SpeechRecognitionResult> RecognizeSpeechAsync(
|
|
byte[] audioBytes,
|
|
string? expectedText = null,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
_logger.LogInformation("Recognizing speech from audio: {ByteCount} bytes", audioBytes?.Length ?? 0);
|
|
|
|
try
|
|
{
|
|
var recognizedText = await _voskService.RecognizeSpeechAsync(
|
|
audioBytes,
|
|
_voskService.GetType().Name == "VoskService" ? 16000 : 16000,
|
|
null,
|
|
cancellationToken);
|
|
|
|
// Calculate accuracy if expected text is provided
|
|
var accuracy = expectedText != null
|
|
? CalculateAccuracy(recognizedText, expectedText)
|
|
: 0.0;
|
|
|
|
_logger.LogInformation(
|
|
"Speech recognized: Text={Text}, Accuracy={Accuracy:P0}",
|
|
recognizedText, accuracy);
|
|
|
|
return new SpeechRecognitionResult
|
|
{
|
|
RecognizedText = recognizedText,
|
|
ExpectedText = expectedText,
|
|
Accuracy = accuracy,
|
|
IsCorrect = accuracy >= 0.8
|
|
};
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Failed to recognize speech");
|
|
throw new AiServiceException(
|
|
"Failed to recognize speech: " + ex.Message,
|
|
AiErrorCode.Temporary);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Recognizes speech from an audio file.
|
|
/// </summary>
|
|
/// <param name="audioFilePath">Path to the audio file</param>
|
|
/// <param name="expectedText">Optional expected text for verification</param>
|
|
/// <param name="cancellationToken">Cancellation token</param>
|
|
/// <returns>Result containing recognized text and accuracy</returns>
|
|
public virtual async Task<SpeechRecognitionResult> RecognizeSpeechFromFileAsync(
|
|
string audioFilePath,
|
|
string? expectedText = null,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
_logger.LogInformation("Recognizing speech from file: {FilePath}", audioFilePath);
|
|
|
|
try
|
|
{
|
|
var recognizedText = await _voskService.RecognizeSpeechFromFileAsync(
|
|
audioFilePath,
|
|
cancellationToken);
|
|
|
|
var accuracy = expectedText != null
|
|
? CalculateAccuracy(recognizedText, expectedText)
|
|
: 0.0;
|
|
|
|
return new SpeechRecognitionResult
|
|
{
|
|
RecognizedText = recognizedText,
|
|
ExpectedText = expectedText,
|
|
Accuracy = accuracy,
|
|
IsCorrect = accuracy >= 0.8
|
|
};
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Failed to recognize speech from file");
|
|
throw;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies if spoken audio matches expected text.
|
|
/// </summary>
|
|
/// <param name="audioBytes">The audio data in bytes</param>
|
|
/// <param name="expectedText">The expected text</param>
|
|
/// <param name="cancellationToken">Cancellation token</param>
|
|
/// <returns>Result with accuracy and correctness</returns>
|
|
public virtual async Task<SpeechRecognitionResult> VerifySpeechAsync(
|
|
byte[] audioBytes,
|
|
string expectedText,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
_logger.LogInformation("Verifying speech: Expected={Expected}", expectedText);
|
|
|
|
var result = await RecognizeSpeechAsync(audioBytes, expectedText, cancellationToken);
|
|
|
|
// Additional validation for speech exercises
|
|
result.PronunciationScore = CalculatePronunciationScore(result.RecognizedText, expectedText);
|
|
result.FluencyScore = CalculateFluencyScore(audioBytes);
|
|
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates a speech exercise for practicing a specific phrase.
|
|
/// </summary>
|
|
/// <param name="phrase">The phrase to practice</param>
|
|
/// <param name="difficulty">Exercise difficulty level</param>
|
|
/// <param name="hints">Optional pronunciation hints</param>
|
|
/// <returns>Speech exercise with metadata</returns>
|
|
public virtual SpeechExercise CreateExercise(
|
|
string phrase,
|
|
string difficulty = "medium",
|
|
IReadOnlyList<string>? hints = null)
|
|
{
|
|
_logger.LogInformation("Creating speech exercise: Phrase={Phrase}, Difficulty={Difficulty}", phrase, difficulty);
|
|
|
|
return new SpeechExercise
|
|
{
|
|
Id = Guid.NewGuid().ToString(),
|
|
Phrase = phrase,
|
|
Difficulty = difficulty,
|
|
Hints = hints ?? new List<string>(),
|
|
CreatedAt = DateTime.UtcNow
|
|
};
|
|
}
|
|
|
|
/// <summary>
|
|
/// Evaluates a user's attempt at a speech exercise.
|
|
/// </summary>
|
|
/// <param name="exerciseId">The exercise ID</param>
|
|
/// <param name="audioBytes">The user's audio attempt</param>
|
|
/// <param name="expectedPhrase">The expected phrase (from exercise)</param>
|
|
/// <param name="cancellationToken">Cancellation token</param>
|
|
/// <returns>Evaluation result with scores</returns>
|
|
public virtual async Task<SpeechExerciseEvaluation> EvaluateExerciseAttemptAsync(
|
|
string exerciseId,
|
|
byte[] audioBytes,
|
|
string expectedPhrase,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
_logger.LogInformation("Evaluating exercise attempt: ExerciseId={ExerciseId}", exerciseId);
|
|
|
|
var result = await VerifySpeechAsync(audioBytes, expectedPhrase, cancellationToken);
|
|
|
|
return new SpeechExerciseEvaluation
|
|
{
|
|
ExerciseId = exerciseId,
|
|
ExpectedPhrase = expectedPhrase,
|
|
RecognizedText = result.RecognizedText,
|
|
Accuracy = result.Accuracy,
|
|
PronunciationScore = result.PronunciationScore,
|
|
FluencyScore = result.FluencyScore,
|
|
IsPassed = result.IsCorrect && result.Accuracy >= 0.8,
|
|
AttemptedAt = DateTime.UtcNow
|
|
};
|
|
}
|
|
|
|
/// <summary>
|
|
/// Tests the speech exercise service.
|
|
/// </summary>
|
|
/// <param name="cancellationToken">Cancellation token</param>
|
|
/// <returns>True if service is working, false otherwise</returns>
|
|
public virtual async Task<bool> TestServiceAsync(CancellationToken cancellationToken = default)
|
|
{
|
|
try
|
|
{
|
|
// We can't test without actual audio, so just verify the service is initialized
|
|
var modelInfo = await _voskService.GetModelInfoAsync();
|
|
return modelInfo.ModelName != null;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Speech exercise service test failed");
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Calculates text matching accuracy between recognized and expected text.
|
|
/// </summary>
|
|
/// <param name="recognized">The recognized text</param>
|
|
/// <param name="expected">The expected text</param>
|
|
/// <returns>Accuracy score (0.0 to 1.0)</returns>
|
|
private double CalculateAccuracy(string recognized, string expected)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(expected))
|
|
return 1.0;
|
|
|
|
if (string.IsNullOrWhiteSpace(recognized))
|
|
return 0.0;
|
|
|
|
// Normalize both strings
|
|
var recognizedNormalized = recognized.Trim().ToLower();
|
|
var expectedNormalized = expected.Trim().ToLower();
|
|
|
|
// Simple exact match
|
|
if (recognizedNormalized == expectedNormalized)
|
|
return 1.0;
|
|
|
|
// Calculate Levenshtein distance for similarity
|
|
var distance = LevenshteinDistance(recognizedNormalized, expectedNormalized);
|
|
var maxLength = Math.Max(recognizedNormalized.Length, expectedNormalized.Length);
|
|
|
|
if (maxLength == 0)
|
|
return 1.0;
|
|
|
|
var similarity = 1.0 - (distance / (double)maxLength);
|
|
return Math.Max(0.0, Math.Min(1.0, similarity));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Calculates pronunciation score based on text matching.
|
|
/// </summary>
|
|
private double CalculatePronunciationScore(string recognized, string expected)
|
|
{
|
|
// For now, use accuracy as pronunciation score
|
|
// In a real implementation, you might use audio analysis
|
|
return CalculateAccuracy(recognized, expected);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Calculates fluency score based on audio characteristics.
|
|
/// </summary>
|
|
private double CalculateFluencyScore(byte[] audioBytes)
|
|
{
|
|
// Simple heuristic: longer audio with consistent speaking rate
|
|
// In a real implementation, analyze audio features
|
|
if (audioBytes == null || audioBytes.Length == 0)
|
|
return 0.0;
|
|
|
|
// Assume 16kHz sample rate, 2 bytes per sample (16-bit)
|
|
var durationSeconds = audioBytes.Length / (16000.0 * 2);
|
|
|
|
// Give higher score for longer, consistent speech
|
|
if (durationSeconds >= 3.0)
|
|
return 1.0;
|
|
if (durationSeconds >= 1.5)
|
|
return 0.8;
|
|
if (durationSeconds >= 0.5)
|
|
return 0.6;
|
|
|
|
return 0.4;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Calculates Levenshtein distance between two strings.
|
|
/// </summary>
|
|
private static int LevenshteinDistance(string s, string t)
|
|
{
|
|
if (string.IsNullOrEmpty(s))
|
|
{
|
|
return string.IsNullOrEmpty(t) ? 0 : t.Length;
|
|
}
|
|
if (string.IsNullOrEmpty(t))
|
|
{
|
|
return s.Length;
|
|
}
|
|
|
|
var n = s.Length;
|
|
var m = t.Length;
|
|
var d = new int[n + 1, m + 1];
|
|
|
|
for (int i = 0; i <= n; d[i, 0] = i++)
|
|
;
|
|
for (int j = 1; j <= m; d[0, j] = j++)
|
|
;
|
|
|
|
for (int i = 1; i <= n; i++)
|
|
{
|
|
for (int j = 1; j <= m; j++)
|
|
{
|
|
var cost = (t[j - 1] == s[i - 1]) ? 0 : 1;
|
|
d[i, j] = Math.Min(
|
|
Math.Min(d[i - 1, j] + 1, d[i, j - 1] + 1),
|
|
d[i - 1, j - 1] + cost);
|
|
}
|
|
}
|
|
return d[n, m];
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Result of speech recognition.
|
|
/// </summary>
|
|
public class SpeechRecognitionResult
|
|
{
|
|
/// <summary>
|
|
/// The recognized text.
|
|
/// </summary>
|
|
public string RecognizedText { get; set; } = string.Empty;
|
|
|
|
/// <summary>
|
|
/// The expected text (if provided for verification).
|
|
/// </summary>
|
|
public string? ExpectedText { get; set; }
|
|
|
|
/// <summary>
|
|
/// Accuracy score (0.0 to 1.0).
|
|
/// </summary>
|
|
public double Accuracy { get; set; }
|
|
|
|
/// <summary>
|
|
/// Pronunciation score (0.0 to 1.0).
|
|
/// </summary>
|
|
public double PronunciationScore { get; set; }
|
|
|
|
/// <summary>
|
|
/// Fluency score (0.0 to 1.0).
|
|
/// </summary>
|
|
public double FluencyScore { get; set; }
|
|
|
|
/// <summary>
|
|
/// Whether the recognition is considered correct.
|
|
/// </summary>
|
|
public bool IsCorrect { get; set; }
|
|
}
|
|
|
|
/// <summary>
|
|
/// Represents a speech exercise.
|
|
/// </summary>
|
|
public class SpeechExercise
|
|
{
|
|
/// <summary>
|
|
/// Unique identifier for the exercise.
|
|
/// </summary>
|
|
public string Id { get; set; } = string.Empty;
|
|
|
|
/// <summary>
|
|
/// The phrase to practice.
|
|
/// </summary>
|
|
public string Phrase { get; set; } = string.Empty;
|
|
|
|
/// <summary>
|
|
/// Difficulty level (easy, medium, hard).
|
|
/// </summary>
|
|
public string Difficulty { get; set; } = "medium";
|
|
|
|
/// <summary>
|
|
/// Pronunciation hints.
|
|
/// </summary>
|
|
public IReadOnlyList<string> Hints { get; set; } = new List<string>();
|
|
|
|
/// <summary>
|
|
/// When the exercise was created.
|
|
/// </summary>
|
|
public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Result of evaluating a speech exercise attempt.
|
|
/// </summary>
|
|
public class SpeechExerciseEvaluation
|
|
{
|
|
/// <summary>
|
|
/// The exercise ID.
|
|
/// </summary>
|
|
public string ExerciseId { get; set; } = string.Empty;
|
|
|
|
/// <summary>
|
|
/// The expected phrase.
|
|
/// </summary>
|
|
public string ExpectedPhrase { get; set; } = string.Empty;
|
|
|
|
/// <summary>
|
|
/// The recognized text.
|
|
/// </summary>
|
|
public string RecognizedText { get; set; } = string.Empty;
|
|
|
|
/// <summary>
|
|
/// Accuracy score (0.0 to 1.0).
|
|
/// </summary>
|
|
public double Accuracy { get; set; }
|
|
|
|
/// <summary>
|
|
/// Pronunciation score (0.0 to 1.0).
|
|
/// </summary>
|
|
public double PronunciationScore { get; set; }
|
|
|
|
/// <summary>
|
|
/// Fluency score (0.0 to 1.0).
|
|
/// </summary>
|
|
public double FluencyScore { get; set; }
|
|
|
|
/// <summary>
|
|
/// Whether the attempt passed.
|
|
/// </summary>
|
|
public bool IsPassed { get; set; }
|
|
|
|
/// <summary>
|
|
/// When the attempt was made.
|
|
/// </summary>
|
|
public DateTime AttemptedAt { get; set; } = DateTime.UtcNow;
|
|
}
|