From 2c873a153f00b0d12240f40b03803303cc74ea57 Mon Sep 17 00:00:00 2001 From: RedInJector Date: Wed, 17 Sep 2025 05:10:18 +0200 Subject: [PATCH 1/6] feat: Face autocalibration - implementation of the same algorithm used in the legacy app --- src/Baballonia/Assets/Resources.Designer.cs | 18 + src/Baballonia/Assets/Resources.af-ZA.resx | 6 + src/Baballonia/Assets/Resources.ar-SA.resx | 6 + src/Baballonia/Assets/Resources.ca-ES.resx | 6 + src/Baballonia/Assets/Resources.cs-CZ.resx | 6 + src/Baballonia/Assets/Resources.da-DK.resx | 6 + src/Baballonia/Assets/Resources.de-DE.resx | 6 + src/Baballonia/Assets/Resources.el-GR.resx | 6 + src/Baballonia/Assets/Resources.en-US.resx | 6 + src/Baballonia/Assets/Resources.es-ES.resx | 6 + src/Baballonia/Assets/Resources.fi-FI.resx | 6 + src/Baballonia/Assets/Resources.fr-FR.resx | 6 + src/Baballonia/Assets/Resources.he-IL.resx | 6 + src/Baballonia/Assets/Resources.hu-HU.resx | 6 + src/Baballonia/Assets/Resources.it-IT.resx | 6 + src/Baballonia/Assets/Resources.ja-JP.resx | 6 + src/Baballonia/Assets/Resources.ko-KR.resx | 6 + src/Baballonia/Assets/Resources.nl-NL.resx | 6 + src/Baballonia/Assets/Resources.pl-PL.resx | 6 + src/Baballonia/Assets/Resources.pt-BR.resx | 6 + src/Baballonia/Assets/Resources.pt-PT.resx | 6 + src/Baballonia/Assets/Resources.resx | 6 + src/Baballonia/Assets/Resources.ro-RO.resx | 6 + src/Baballonia/Assets/Resources.ru-RU.resx | 6 + src/Baballonia/Assets/Resources.sv-SE.resx | 6 + src/Baballonia/Assets/Resources.tr-TR.resx | 6 + src/Baballonia/Assets/Resources.uk-UA.resx | 6 + src/Baballonia/Assets/Resources.vi-VN.resx | 6 + src/Baballonia/Assets/Resources.zh-CN.resx | 6 + src/Baballonia/Assets/Resources.zh-TW.resx | 6 + .../Contracts/ICalibrationService.cs | 5 + src/Baballonia/Services/CalibrationService.cs | 47 +- .../Inference/FaceProcessingPipeline.cs | 16 +- .../Inference/Filters/AutocalibOptimized.cs | 216 +++++++ .../Services/ParameterSenderService.cs | 18 +- .../Services/ProcessingLoopService.cs | 4 + .../Themes/Fluent/Controls/RangeSlider.axaml | 6 + .../SplitViewPane/CalibrationViewModel.cs | 34 +- src/Baballonia/Views/CalibrationView.axaml | 558 +++++++++--------- 39 files changed, 788 insertions(+), 308 deletions(-) create mode 100644 src/Baballonia/Services/Inference/Filters/AutocalibOptimized.cs diff --git a/src/Baballonia/Assets/Resources.Designer.cs b/src/Baballonia/Assets/Resources.Designer.cs index 73681c19..e0e6fb0c 100644 --- a/src/Baballonia/Assets/Resources.Designer.cs +++ b/src/Baballonia/Assets/Resources.Designer.cs @@ -321,6 +321,24 @@ public static string Calibration_Tongue_Expressions_Header { } } + /// + /// Looks up a localized string similar to Enables autocalibrations of values over time. + /// + public static string Calibration_UseAutocalibration_Description { + get { + return ResourceManager.GetString("Calibration_UseAutocalibration_Description", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Enable Face Autocalibration. + /// + public static string Calibration_UseAutocalibration_Title { + get { + return ResourceManager.GetString("Calibration_UseAutocalibration_Title", resourceCulture); + } + } + /// /// Looks up a localized string similar to Start typing.... /// diff --git a/src/Baballonia/Assets/Resources.af-ZA.resx b/src/Baballonia/Assets/Resources.af-ZA.resx index 73b898d1..e923ce6c 100644 --- a/src/Baballonia/Assets/Resources.af-ZA.resx +++ b/src/Baballonia/Assets/Resources.af-ZA.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.ar-SA.resx b/src/Baballonia/Assets/Resources.ar-SA.resx index 7cd1708b..36d4b70e 100644 --- a/src/Baballonia/Assets/Resources.ar-SA.resx +++ b/src/Baballonia/Assets/Resources.ar-SA.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.ca-ES.resx b/src/Baballonia/Assets/Resources.ca-ES.resx index 0a67058f..53d2bbbd 100644 --- a/src/Baballonia/Assets/Resources.ca-ES.resx +++ b/src/Baballonia/Assets/Resources.ca-ES.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.cs-CZ.resx b/src/Baballonia/Assets/Resources.cs-CZ.resx index ccd21fb1..11f40a0a 100644 --- a/src/Baballonia/Assets/Resources.cs-CZ.resx +++ b/src/Baballonia/Assets/Resources.cs-CZ.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.da-DK.resx b/src/Baballonia/Assets/Resources.da-DK.resx index 6de2b8a2..ae7f781f 100644 --- a/src/Baballonia/Assets/Resources.da-DK.resx +++ b/src/Baballonia/Assets/Resources.da-DK.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.de-DE.resx b/src/Baballonia/Assets/Resources.de-DE.resx index 291eaf6e..c772969c 100644 --- a/src/Baballonia/Assets/Resources.de-DE.resx +++ b/src/Baballonia/Assets/Resources.de-DE.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.el-GR.resx b/src/Baballonia/Assets/Resources.el-GR.resx index 4825fb2c..c9ddbc34 100644 --- a/src/Baballonia/Assets/Resources.el-GR.resx +++ b/src/Baballonia/Assets/Resources.el-GR.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.en-US.resx b/src/Baballonia/Assets/Resources.en-US.resx index 73b898d1..e923ce6c 100644 --- a/src/Baballonia/Assets/Resources.en-US.resx +++ b/src/Baballonia/Assets/Resources.en-US.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.es-ES.resx b/src/Baballonia/Assets/Resources.es-ES.resx index 62d8e0e7..40e4b967 100644 --- a/src/Baballonia/Assets/Resources.es-ES.resx +++ b/src/Baballonia/Assets/Resources.es-ES.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.fi-FI.resx b/src/Baballonia/Assets/Resources.fi-FI.resx index 9492e0ba..f48b17d5 100644 --- a/src/Baballonia/Assets/Resources.fi-FI.resx +++ b/src/Baballonia/Assets/Resources.fi-FI.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.fr-FR.resx b/src/Baballonia/Assets/Resources.fr-FR.resx index c027dfd0..f8875b90 100644 --- a/src/Baballonia/Assets/Resources.fr-FR.resx +++ b/src/Baballonia/Assets/Resources.fr-FR.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.he-IL.resx b/src/Baballonia/Assets/Resources.he-IL.resx index 73b898d1..e923ce6c 100644 --- a/src/Baballonia/Assets/Resources.he-IL.resx +++ b/src/Baballonia/Assets/Resources.he-IL.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.hu-HU.resx b/src/Baballonia/Assets/Resources.hu-HU.resx index 73b898d1..e923ce6c 100644 --- a/src/Baballonia/Assets/Resources.hu-HU.resx +++ b/src/Baballonia/Assets/Resources.hu-HU.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.it-IT.resx b/src/Baballonia/Assets/Resources.it-IT.resx index 4f895a11..07438cb7 100644 --- a/src/Baballonia/Assets/Resources.it-IT.resx +++ b/src/Baballonia/Assets/Resources.it-IT.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.ja-JP.resx b/src/Baballonia/Assets/Resources.ja-JP.resx index ee3b1d51..7325e034 100644 --- a/src/Baballonia/Assets/Resources.ja-JP.resx +++ b/src/Baballonia/Assets/Resources.ja-JP.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.ko-KR.resx b/src/Baballonia/Assets/Resources.ko-KR.resx index bb97129f..8e438658 100644 --- a/src/Baballonia/Assets/Resources.ko-KR.resx +++ b/src/Baballonia/Assets/Resources.ko-KR.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.nl-NL.resx b/src/Baballonia/Assets/Resources.nl-NL.resx index d12f70f0..b5398d47 100644 --- a/src/Baballonia/Assets/Resources.nl-NL.resx +++ b/src/Baballonia/Assets/Resources.nl-NL.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.pl-PL.resx b/src/Baballonia/Assets/Resources.pl-PL.resx index c615b2fa..b283676f 100644 --- a/src/Baballonia/Assets/Resources.pl-PL.resx +++ b/src/Baballonia/Assets/Resources.pl-PL.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.pt-BR.resx b/src/Baballonia/Assets/Resources.pt-BR.resx index 850ed7c8..35dc982c 100644 --- a/src/Baballonia/Assets/Resources.pt-BR.resx +++ b/src/Baballonia/Assets/Resources.pt-BR.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.pt-PT.resx b/src/Baballonia/Assets/Resources.pt-PT.resx index 17fa3154..60f0386c 100644 --- a/src/Baballonia/Assets/Resources.pt-PT.resx +++ b/src/Baballonia/Assets/Resources.pt-PT.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.resx b/src/Baballonia/Assets/Resources.resx index 174ac1da..08b2a79f 100644 --- a/src/Baballonia/Assets/Resources.resx +++ b/src/Baballonia/Assets/Resources.resx @@ -726,4 +726,10 @@ If you are using Baballonia with VRCFaceTracking, there is no need to change these ports + + Enable Face Autocalibration + + + Enables autocalibrations of values over time + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.ro-RO.resx b/src/Baballonia/Assets/Resources.ro-RO.resx index bda22288..62c3af89 100644 --- a/src/Baballonia/Assets/Resources.ro-RO.resx +++ b/src/Baballonia/Assets/Resources.ro-RO.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.ru-RU.resx b/src/Baballonia/Assets/Resources.ru-RU.resx index c1792e40..f01a6b97 100644 --- a/src/Baballonia/Assets/Resources.ru-RU.resx +++ b/src/Baballonia/Assets/Resources.ru-RU.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.sv-SE.resx b/src/Baballonia/Assets/Resources.sv-SE.resx index 45e222a8..4385e580 100644 --- a/src/Baballonia/Assets/Resources.sv-SE.resx +++ b/src/Baballonia/Assets/Resources.sv-SE.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.tr-TR.resx b/src/Baballonia/Assets/Resources.tr-TR.resx index 019ffd40..941190fe 100644 --- a/src/Baballonia/Assets/Resources.tr-TR.resx +++ b/src/Baballonia/Assets/Resources.tr-TR.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.uk-UA.resx b/src/Baballonia/Assets/Resources.uk-UA.resx index 52d3b1e5..937e9daf 100644 --- a/src/Baballonia/Assets/Resources.uk-UA.resx +++ b/src/Baballonia/Assets/Resources.uk-UA.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.vi-VN.resx b/src/Baballonia/Assets/Resources.vi-VN.resx index a93ab205..024296cf 100644 --- a/src/Baballonia/Assets/Resources.vi-VN.resx +++ b/src/Baballonia/Assets/Resources.vi-VN.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.zh-CN.resx b/src/Baballonia/Assets/Resources.zh-CN.resx index ee61fac4..f579eca5 100644 --- a/src/Baballonia/Assets/Resources.zh-CN.resx +++ b/src/Baballonia/Assets/Resources.zh-CN.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Assets/Resources.zh-TW.resx b/src/Baballonia/Assets/Resources.zh-TW.resx index 70b336a7..c26a1282 100644 --- a/src/Baballonia/Assets/Resources.zh-TW.resx +++ b/src/Baballonia/Assets/Resources.zh-TW.resx @@ -721,4 +721,10 @@ + + + + + + \ No newline at end of file diff --git a/src/Baballonia/Contracts/ICalibrationService.cs b/src/Baballonia/Contracts/ICalibrationService.cs index c30384ef..145f638e 100644 --- a/src/Baballonia/Contracts/ICalibrationService.cs +++ b/src/Baballonia/Contracts/ICalibrationService.cs @@ -1,5 +1,6 @@ using System.Threading.Tasks; using Baballonia.Services.Calibration; +using Baballonia.Services.Inference.Filters; namespace Baballonia.Contracts; @@ -9,6 +10,10 @@ public interface ICalibrationService CalibrationParameter GetExpressionSettings(string parameterName); + AutocalibOptimized? FaceAutocalib { get; set; } + + float ApplyCalibrationSetting(string expression, float value); + float[] ApplyFaceCalibration(float[] expression); float GetExpressionSetting(string expression); void ResetValues(); void ResetMinimums(); diff --git a/src/Baballonia/Services/CalibrationService.cs b/src/Baballonia/Services/CalibrationService.cs index a883775e..95ec5f54 100644 --- a/src/Baballonia/Services/CalibrationService.cs +++ b/src/Baballonia/Services/CalibrationService.cs @@ -1,8 +1,11 @@ using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.Threading.Tasks; using Baballonia.Contracts; +using Baballonia.Helpers; using Baballonia.Services.Calibration; +using Baballonia.Services.Inference.Filters; namespace Baballonia.Services; @@ -15,12 +18,12 @@ public class CalibrationService : ICalibrationService { "LeftEyeY", "/LeftEyeY" }, { "RightEyeX", "/RightEyeX" }, { "RightEyeY", "/RightEyeY" }, + { "LeftEyeLid", "/LeftEyeLid" }, + { "RightEyeLid", "/RightEyeLid" }, }; private readonly Dictionary _faceExpressionMap = new() { - { "LeftEyeLid", "/LeftEyeLid" }, - { "RightEyeLid", "/RightEyeLid" }, { "CheekPuffLeft", "/cheekPuffLeft" }, { "CheekPuffRight", "/cheekPuffRight" }, { "CheekSuckLeft", "/cheekSuckLeft" }, @@ -72,10 +75,12 @@ public class CalibrationService : ICalibrationService private readonly ILocalSettingsService _localSettingsService; + public AutocalibOptimized? FaceAutocalib { get; set; } + public CalibrationService(ILocalSettingsService localSettingsService) { - _localSettingsService = localSettingsService; + _localSettingsService = localSettingsService; Load(); } @@ -98,7 +103,7 @@ public void SetExpression(string expression, float value) var param = new CalibrationParameter(lower, upper, min, max); _expressionSettings[parameterName] = param; - SaveAsync(); + Save(); } public CalibrationParameter GetExpressionSettings(string parameterName) @@ -108,6 +113,32 @@ public CalibrationParameter GetExpressionSettings(string parameterName) new CalibrationParameter(); } + public float ApplyCalibrationSetting(string expression, float value) + { + var settings = GetExpressionSettings(expression); + return value.Remap(settings.Lower, settings.Upper, settings.Min, settings.Max); + } + + public float[] ApplyFaceCalibration(float[] expression) + { + Debug.Assert(expression.Length == Utils.FaceRawExpressions); + + if (FaceAutocalib != null) + { + return FaceAutocalib.Filter(expression); + } + + var res = new float[Utils.FaceRawExpressions]; + var i = 0; + foreach (var faceExp in _faceExpressionMap) + { + res[i] = ApplyCalibrationSetting(faceExp.Value, expression[i]); + i++; + } + + return res; + } + public float GetExpressionSetting(string expression) { if (!expression.EndsWith("Lower") && !expression.EndsWith("Upper")) return 0; @@ -123,7 +154,7 @@ public float GetExpressionSetting(string expression) return isUpper ? currentSettings.Upper : currentSettings.Lower; } - private void SaveAsync() + private void Save() { _localSettingsService.SaveSetting("CalibrationParams", _expressionSettings); } @@ -168,7 +199,7 @@ public void ResetValues() parameter.Lower = parameter.Min; parameter.Upper = parameter.Max; } - SaveAsync(); + Save(); } public void ResetMinimums() @@ -177,7 +208,7 @@ public void ResetMinimums() { parameter.Lower = parameter.Min; } - SaveAsync(); + Save(); } public void ResetMaximums() @@ -186,6 +217,6 @@ public void ResetMaximums() { parameter.Upper = parameter.Max; } - SaveAsync(); + Save(); } } diff --git a/src/Baballonia/Services/Inference/FaceProcessingPipeline.cs b/src/Baballonia/Services/Inference/FaceProcessingPipeline.cs index bb195bf0..9e271e59 100644 --- a/src/Baballonia/Services/Inference/FaceProcessingPipeline.cs +++ b/src/Baballonia/Services/Inference/FaceProcessingPipeline.cs @@ -2,23 +2,25 @@ using Baballonia.Contracts; using Baballonia.Services.Inference.Enums; using OpenCvSharp; +using SoundFlow.Components; namespace Baballonia.Services.Inference; public class FaceProcessingPipeline : DefaultProcessingPipeline { + public IFilter? AutocalibFilter; - public float[]? RunUpdate() + public float[]? RunUpdate() { var frame = VideoSource?.GetFrame(ColorType.Gray8); - if(frame == null) + if (frame == null) return null; InvokeNewFrameEvent(frame); var transformed = ImageTransformer?.Apply(frame); - if(transformed == null) + if (transformed == null) return null; InvokeTransformedFrameEvent(transformed); @@ -31,14 +33,16 @@ public class FaceProcessingPipeline : DefaultProcessingPipeline transformed.Dispose(); var inferenceResult = InferenceService?.Run(); - if(inferenceResult == null) + if (inferenceResult == null) return null; - if(Filter != null) + if (Filter != null) inferenceResult = Filter.Filter(inferenceResult); - InvokeFilteredResultEvent(inferenceResult); + if (AutocalibFilter != null) + inferenceResult = AutocalibFilter.Filter(inferenceResult); + InvokeFilteredResultEvent(inferenceResult); return inferenceResult; } diff --git a/src/Baballonia/Services/Inference/Filters/AutocalibOptimized.cs b/src/Baballonia/Services/Inference/Filters/AutocalibOptimized.cs new file mode 100644 index 00000000..f218639e --- /dev/null +++ b/src/Baballonia/Services/Inference/Filters/AutocalibOptimized.cs @@ -0,0 +1,216 @@ +using System; + +namespace Baballonia.Services.Inference.Filters; + +public class AutocalibOptimized : IFilter +{ + private int _expressionCount; + + private float _alpha; + private float _beta; + private float _multiplier; + private bool _isInitialized; + + private readonly float[] _neutralMask; + private readonly float[] _activeMask; + private readonly float[] _minDiff; + private readonly float[] _maxDiff; + private readonly float[] _diff; + private readonly float[] _std; + public float[] _min; + public float[] _max; + public float[] _mean; + public float[] _variance; + public float[] _threshold; + public float[] _confidence; + private float _sampleCount; + private float[] _sampleCounts; + private bool[] _calibratedFlags; + private int _warmupFrames; + private float _adaptationRate; + private float _minAdaptationRate; + private float _adaptationDecay; + private float _hysteresisThreshold; + private int[] _decayCounters; + private int _daceyThreshold; + private float[] _decayLevels; + + public float[] Filter(float[] input) + { + var (minVals, maxVals, calibratedFlags) = Update(input); + + // Reuse buffer for denominator & calibrated + var calibrated = new float[_expressionCount]; + for (int i = 0; i < _expressionCount; i++) + { + float denom = maxVals[i] - minVals[i]; + if (denom == 0) denom = 1e-6f; // avoid divide by zero + + if (calibratedFlags[i]) + { + calibrated[i] = (input[i] - minVals[i]) / denom; + calibrated[i] = Math.Min(Math.Max(calibrated[i], 0f), 1f); + } + } + return calibrated; + } + + + public AutocalibOptimized(int expressionCount, float alpha = 0.1f, float beta = 0.1f, + float thresholdMultiplier = 2.0f) + { + _expressionCount = expressionCount; + _alpha = alpha; + _beta = beta; + _multiplier = thresholdMultiplier; + _isInitialized = false; + + _neutralMask = new float[_expressionCount]; + _activeMask = new float[_expressionCount]; + _minDiff = new float[_expressionCount]; + _maxDiff = new float[_expressionCount]; + _diff = new float[_expressionCount]; + _std = new float[_expressionCount]; + _threshold = new float[_expressionCount]; + _confidence = new float[_expressionCount]; + + _min = new float[expressionCount]; + _max = new float[expressionCount]; + + _mean = new float[expressionCount]; + _variance = new float[expressionCount]; + _threshold = new float[expressionCount]; + _confidence = new float[expressionCount]; + _sampleCounts = new float[expressionCount]; + _calibratedFlags = new bool[expressionCount]; + _decayCounters = new int[expressionCount]; + _decayLevels = new float[expressionCount]; + + _sampleCount = 0f; + _warmupFrames = 300; + + _adaptationRate = 1f; + _minAdaptationRate = 0.01f; + _adaptationDecay = 0.99998f; + _hysteresisThreshold = 0.05f; + + Array.Fill(_min, float.PositiveInfinity); + Array.Fill(_max, float.NegativeInfinity); + + Array.Fill(_mean, 0f); + Array.Fill(_variance, 0f); + Array.Fill(_threshold, 0f); + Array.Fill(_confidence, 0f); + Array.Fill(_sampleCounts, 0f); + Array.Fill(_calibratedFlags, false); + + Array.Fill(_decayCounters, 0); + Array.Fill(_decayLevels, 0); + } + + private (float[] minVals, float[] maxVals, bool[] calibratedFlags) Update(float[] input) + { + if (input.Length != _expressionCount) + throw new ArgumentException( + $"Input length mismatch. Expected: {_expressionCount}, got: {input.Length}", + nameof(input)); + + if (_sampleCount > 0) + _adaptationRate = Math.Max(_minAdaptationRate, _adaptationRate * _adaptationDecay); + + if (!_isInitialized) + { + input.CopyTo(_min, 0); + input.CopyTo(_max, 0); + + for (int i = 0; i < _expressionCount; i++) + _mean[i] = _alpha * input[i] + (1 - _alpha) * _mean[i]; + + _sampleCount++; + Array.Fill(_confidence, 1f); + _isInitialized = true; + return (_min, _max, _calibratedFlags); + } + + const float neutralThreshold = 0.15f; + int neutralCount = 0; + for (int i = 0; i < _expressionCount; i++) + { + bool neutral = Math.Abs(input[i]) < neutralThreshold; + if (neutral) neutralCount++; + _neutralMask[i] = neutral ? 1f : 0f; + _activeMask[i] = neutral ? 0f : 1f; + + _minDiff[i] = _min[i] - input[i]; + _maxDiff[i] = input[i] - _max[i]; + } + + bool isNeutralPose = neutralCount >= (int)(0.8 * _expressionCount); + + if (isNeutralPose) + { + for (int i = 0; i < _expressionCount; i++) + { + if (_minDiff[i] > _hysteresisThreshold && _neutralMask[i] > 0) + _min[i] = _adaptationRate * input[i] + (1 - _adaptationRate) * _min[i]; + + if (_neutralMask[i] > 0) + _mean[i] = _alpha * input[i] + (1 - _alpha) * _mean[i]; + } + } + + for (int i = 0; i < _expressionCount; i++) + { + if (_maxDiff[i] > _hysteresisThreshold && _activeMask[i] > 0) + _max[i] = ((_decayLevels[i] > 0.1f) ? 0.5f : _adaptationRate) * input[i] + + (1 - ((_decayLevels[i] > 0.1f) ? 0.5f : _adaptationRate)) * _max[i]; + + if (input[i] < _max[i] * 0.9f) + _decayCounters[i]++; + + if (_decayCounters[i] > _daceyThreshold) + { + _decayLevels[i] += 0.01f; + _max[i] *= 0.999f; + } + else + { + _decayCounters[i] = 0; + _decayLevels[i] *= 0.95f; + } + + if (_min[i] > _max[i]) + { + float avg = (_min[i] + _max[i]) / 2f; + _min[i] = avg; + _max[i] = avg; + } + } + + for (int i = 0; i < _expressionCount; i++) + { + _diff[i] = input[i] - _mean[i]; + _variance[i] = _beta * (_diff[i] * _diff[i]) + (1 - _beta) * _variance[i]; + + _std[i] = (float)Math.Sqrt(_variance[i]) + 1e-6f; + _threshold[i] = _multiplier * _std[i]; + _confidence[i] = (float)Math.Exp(-Math.Abs(_diff[i]) / _threshold[i]); + + _sampleCounts[i]++; + } + + for (int i = 0; i < _expressionCount; i++) + { + if (!_calibratedFlags[i] && + (_max[i] - _min[i] > 1e-3f) && + (_sampleCounts[i] >= 300) && + (_variance[i] > 1e-4f)) + { + _calibratedFlags[i] = true; + } + } + + _sampleCount++; + return (_min, _max, _calibratedFlags); + } +} diff --git a/src/Baballonia/Services/ParameterSenderService.cs b/src/Baballonia/Services/ParameterSenderService.cs index 60e77320..63a85746 100644 --- a/src/Baballonia/Services/ParameterSenderService.cs +++ b/src/Baballonia/Services/ParameterSenderService.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -177,20 +178,17 @@ private void ProcessNativeVrcEyeTracking(float[] expressions) private void ProcessFaceExpressionData(float[] expressions) { - if (expressions == null) return; - if (expressions.Length == 0) return; + Debug.Assert(expressions != null); + Debug.Assert(expressions.Length == Utils.FaceRawExpressions); + + var calibrated = calibrationService.ApplyFaceCalibration(expressions); - for (int i = 0; i < Math.Min(expressions.Length, FaceExpressionMap.Count); i++) + for (int i = 0; i < Utils.FaceRawExpressions; i++) { - var weight = expressions[i]; + var weight = calibrated[i]; var faceElement = FaceExpressionMap.ElementAt(i); - var settings = calibrationService.GetExpressionSettings(faceElement.Key); - var msg = new OscMessage(prefix + faceElement.Value, - Math.Clamp( - weight.Remap(settings.Lower, settings.Upper, settings.Min, settings.Max), - settings.Min, - settings.Max)); + var msg = new OscMessage(prefix + faceElement.Value, weight); _sendQueue.Enqueue(msg); } } diff --git a/src/Baballonia/Services/ProcessingLoopService.cs b/src/Baballonia/Services/ProcessingLoopService.cs index f7f5e7fa..892c8f7c 100644 --- a/src/Baballonia/Services/ProcessingLoopService.cs +++ b/src/Baballonia/Services/ProcessingLoopService.cs @@ -64,6 +64,10 @@ public ProcessingLoopService( EyesProcessingPipeline.Filter = eyeFilter; LoadEyeStabilizationSetting(); + var UseAutocalib = _localSettingsService.ReadSetting("AppSettings_UseAutocalib", true); + if(UseAutocalib) + FaceProcessingPipeline.AutocalibFilter = new AutocalibOptimized(Utils.FaceRawExpressions); + _drawTimer.Tick += TimerEvent; _drawTimer.Start(); } diff --git a/src/Baballonia/Themes/Fluent/Controls/RangeSlider.axaml b/src/Baballonia/Themes/Fluent/Controls/RangeSlider.axaml index 68b2af0d..e4ab9500 100644 --- a/src/Baballonia/Themes/Fluent/Controls/RangeSlider.axaml +++ b/src/Baballonia/Themes/Fluent/Controls/RangeSlider.axaml @@ -94,6 +94,12 @@ + + + diff --git a/src/Baballonia/ViewModels/SplitViewPane/CalibrationViewModel.cs b/src/Baballonia/ViewModels/SplitViewPane/CalibrationViewModel.cs index 884af50d..6ae33507 100644 --- a/src/Baballonia/ViewModels/SplitViewPane/CalibrationViewModel.cs +++ b/src/Baballonia/ViewModels/SplitViewPane/CalibrationViewModel.cs @@ -10,6 +10,7 @@ using Avalonia.Threading; using Baballonia.Helpers; using Baballonia.Services; +using Baballonia.Services.Inference.Filters; using CommunityToolkit.Mvvm.Input; namespace Baballonia.ViewModels.SplitViewPane; @@ -27,6 +28,9 @@ public partial class CalibrationViewModel : ViewModelBase, IDisposable [property: SavedSetting("AppSettings_StabilizeEyes", false)] private bool _stabilizeEyes; + [property: SavedSetting("AppSettings_UseAutocalib", true)] + [ObservableProperty] private bool _useAutocalib; + private ILocalSettingsService _settingsService { get; } private readonly ICalibrationService _calibrationService; private readonly ParameterSenderService _parameterSenderService; @@ -159,17 +163,31 @@ public CalibrationViewModel() }; } + partial void OnUseAutocalibChanged(bool value) + { + var prev = _settingsService.ReadSetting("AppSettings_UseAutocalib", true); + if (prev == value) + return; + + _settingsService.SaveSetting("AppSettings_UseAutocalib", UseAutocalib); + _calibrationService.FaceAutocalib = value ? new AutocalibOptimized(Utils.FaceRawExpressions) : null; + } + private void ExpressionUpdateHandler(ProcessingLoopService.Expressions expressions) { - if(expressions.FaceExpression != null) + if (expressions.FaceExpression != null) + { Dispatcher.UIThread.Post(() => { - ApplyCurrentFaceExpressionValues(expressions.FaceExpression, CheekSettings); - ApplyCurrentFaceExpressionValues(expressions.FaceExpression, MouthSettings); - ApplyCurrentFaceExpressionValues(expressions.FaceExpression, JawSettings); - ApplyCurrentFaceExpressionValues(expressions.FaceExpression, NoseSettings); - ApplyCurrentFaceExpressionValues(expressions.FaceExpression, TongueSettings); + var expr = _calibrationService.ApplyFaceCalibration(expressions.FaceExpression); + ApplyCurrentFaceExpressionValues(expr, CheekSettings); + ApplyCurrentFaceExpressionValues(expr, MouthSettings); + ApplyCurrentFaceExpressionValues(expr, JawSettings); + ApplyCurrentFaceExpressionValues(expr, NoseSettings); + ApplyCurrentFaceExpressionValues(expr, TongueSettings); }); + } + if(expressions.EyeExpression != null) Dispatcher.UIThread.Post(() => { @@ -261,8 +279,10 @@ private void LoadInitialSettings(IEnumerable settings) } } + + public void Dispose() { - // _processingLoopService.ExpressionUpdateEvent -= ExpressionUpdateHandler; + _processingLoopService.ExpressionChangeEvent -= ExpressionUpdateHandler; } } diff --git a/src/Baballonia/Views/CalibrationView.axaml b/src/Baballonia/Views/CalibrationView.axaml index 9afb6977..87f89d07 100644 --- a/src/Baballonia/Views/CalibrationView.axaml +++ b/src/Baballonia/Views/CalibrationView.axaml @@ -4,6 +4,7 @@ xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:d="http://schemas.microsoft.com/expression/blend/2008" xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" + xmlns:controls="clr-namespace:Baballonia.Controls" xmlns:splitViewPane="clr-namespace:Baballonia.ViewModels.SplitViewPane" xmlns:models="clr-namespace:Baballonia.Models" xmlns:twoRangeSlider="clr-namespace:Baballonia.Controls" @@ -15,294 +16,297 @@ - - - -