# HG changeset patch
# User StephaneLenclud
# Date 1483379025 -3600
# Node ID e5f85a895a6208e06848d4ec666db6660e666332
# Parent 10de0c7c2fed751a0da5a6606a24c2e077cfdc58
Draft audio spectrum visualizer.
diff -r 10de0c7c2fed -r e5f85a895a62 Server/FormMain.cs
--- a/Server/FormMain.cs Mon Jan 02 15:50:50 2017 +0100
+++ b/Server/FormMain.cs Mon Jan 02 18:43:45 2017 +0100
@@ -37,8 +37,14 @@
using System.Runtime.InteropServices;
using System.Security;
//CSCore
+using CSCore;
+using CSCore.Win32;
+using CSCore.DSP;
+using CSCore.Streams;
using CSCore.CoreAudioAPI;
-
+using CSCore.SoundIn;
+// Visualization
+using Visualization;
// CEC
using CecSharp;
//Network
@@ -49,7 +55,7 @@
using MiniDisplayInterop;
using SharpLib.Display;
using Ear = SharpLib.Ear;
-using CSCore.Win32;
+
namespace SharpDisplayManager
{
@@ -105,9 +111,15 @@
//Function pointer for pixel Y coordinate intercept
CoordinateTranslationDelegate iScreenY;
//CSCore
+ // Volume management
private MMDeviceEnumerator iMultiMediaDeviceEnumerator;
private MMDevice iMultiMediaDevice;
private AudioEndpointVolume iAudioEndpointVolume;
+ // Audio visualization
+ private WasapiCapture iSoundIn;
+ private IWaveSource iWaveSource;
+ private LineSpectrum iLineSpectrum;
+
//Network
private NetworkManager iNetworkManager;
@@ -641,6 +653,106 @@
/// <summary>
///
/// </summary>
+ private void StartAudioVisualization()
+ {
+ StopAudioVisualization();
+ //Open the default device
+ iSoundIn = new WasapiLoopbackCapture();
+ //Our loopback capture opens the default render device by default so the following is not needed
+ //iSoundIn.Device = MMDeviceEnumerator.DefaultAudioEndpoint(DataFlow.Render, Role.Console);
+ iSoundIn.Initialize();
+
+ SoundInSource soundInSource = new SoundInSource(iSoundIn);
+ ISampleSource source = soundInSource.ToSampleSource();
+
+ const FftSize fftSize = FftSize.Fft4096;
+ //create a spectrum provider which provides fft data based on some input
+ BasicSpectrumProvider spectrumProvider = new BasicSpectrumProvider(source.WaveFormat.Channels, source.WaveFormat.SampleRate, fftSize);
+
+ //linespectrum and voiceprint3dspectrum used for rendering some fft data
+ //in oder to get some fft data, set the previously created spectrumprovider
+ iLineSpectrum = new LineSpectrum(fftSize)
+ {
+ SpectrumProvider = spectrumProvider,
+ UseAverage = true,
+ BarCount = 32,
+ BarSpacing = 0,
+ IsXLogScale = true,
+ ScalingStrategy = ScalingStrategy.Sqrt
+ };
+
+
+ //the SingleBlockNotificationStream is used to intercept the played samples
+ var notificationSource = new SingleBlockNotificationStream(source);
+ //pass the intercepted samples as input data to the spectrumprovider (which will calculate a fft based on them)
+ notificationSource.SingleBlockRead += (s, a) => spectrumProvider.Add(a.Left, a.Right);
+
+ iWaveSource = notificationSource.ToWaveSource(16);
+
+
+ // We need to read from our source otherwise SingleBlockRead is never called and our spectrum provider is not populated
+ byte[] buffer = new byte[iWaveSource.WaveFormat.BytesPerSecond / 2];
+ soundInSource.DataAvailable += (s, aEvent) =>
+ {
+ int read;
+ while ((read = iWaveSource.Read(buffer, 0, buffer.Length)) > 0) ;
+ };
+
+
+ //Start recording
+ iSoundIn.Start();
+ }
+
+ /// <summary>
+ ///
+ /// </summary>
+ private void StopAudioVisualization()
+ {
+
+ if (iSoundIn != null)
+ {
+ iSoundIn.Stop();
+ iSoundIn.Dispose();
+ iSoundIn = null;
+ }
+ if (iWaveSource != null)
+ {
+ iWaveSource.Dispose();
+ iWaveSource = null;
+ }
+
+ }
+
+
+ /// <summary>
+ ///
+ /// </summary>
+ private void GenerateAudioVisualization()
+ {
+ // For demo draft purposes just fetch the firt picture box control and update it with current audio spectrum
+ foreach (Control ctrl in iTableLayoutPanel.Controls)
+ {
+ if (ctrl is PictureBox)
+ {
+ PictureBox pb = (PictureBox)ctrl;
+ Image image = pb.Image;
+ var newImage = iLineSpectrum.CreateSpectrumLine(pb.Size, Color.Black, Color.Black, Color.White, false);
+ if (newImage != null)
+ {
+ pb.Image = newImage;
+ if (image != null)
+ image.Dispose();
+ }
+
+ break;
+ }
+ }
+ }
+
+
+ /// <summary>
+ ///
+ /// </summary>
private void UpdateAudioDeviceAndMasterVolumeThreadSafe()
{
if (this.InvokeRequired)
@@ -657,6 +769,7 @@
//Get our master volume
iMultiMediaDevice = iMultiMediaDeviceEnumerator.GetDefaultAudioEndpoint(DataFlow.Render, Role.Multimedia);
iAudioEndpointVolume = AudioEndpointVolume.FromDevice(iMultiMediaDevice);
+
//Update our label
labelDefaultAudioDevice.Text = iMultiMediaDevice.FriendlyName;
@@ -667,7 +780,9 @@
AudioEndpointVolumeCallback callback = new AudioEndpointVolumeCallback();
callback.NotifyRecived += OnVolumeNotificationThreadSafe;
// Do we need to unregister?
- iAudioEndpointVolume.RegisterControlChangeNotify(callback);
+ iAudioEndpointVolume.RegisterControlChangeNotify(callback);
+ //
+ StartAudioVisualization();
//
trackBarMasterVolume.Enabled = true;
}
@@ -1105,8 +1220,10 @@
}
}
+ GenerateAudioVisualization();
+
//Compute instant FPS
- toolStripStatusLabelFps.Text = (1.0/NewTickTime.Subtract(LastTickTime).TotalSeconds).ToString("F0") + " / " +
+ toolStripStatusLabelFps.Text = (1.0/NewTickTime.Subtract(LastTickTime).TotalSeconds).ToString("F0") + " / " +
(1000/iTimerDisplay.Interval).ToString() + " FPS";
LastTickTime = NewTickTime;
@@ -1441,6 +1558,8 @@
private void MainForm_FormClosing(object sender, FormClosingEventArgs e)
{
+ //TODO: discard other CSCore audio objects
+ StopAudioVisualization();
iCecManager.Stop();
iNetworkManager.Dispose();
CloseDisplayConnection();
diff -r 10de0c7c2fed -r e5f85a895a62 Server/SharpDisplayManager.csproj
--- a/Server/SharpDisplayManager.csproj Mon Jan 02 15:50:50 2017 +0100
+++ b/Server/SharpDisplayManager.csproj Mon Jan 02 18:43:45 2017 +0100
@@ -174,11 +174,15 @@
<Compile Include="Actions\ActionCecUserControlReleased.cs" />
<Compile Include="Actions\ActionDisplayMessage.cs" />
<Compile Include="Actions\ActionHarmonyCommand.cs" />
+ <Compile Include="Spectrum\BasicSpectrumProvider.cs" />
<Compile Include="CbtHook.cs" />
<Compile Include="CecClient.cs" />
<Compile Include="ConsumerElectronicControl.cs" />
<Compile Include="ClientData.cs" />
<Compile Include="EarManager.cs" />
+ <Compile Include="Spectrum\ISpectrumProvider.cs" />
+ <Compile Include="Spectrum\LineSpectrum.cs" />
+ <Compile Include="Spectrum\ScalingStrategy.cs" />
<Compile Include="Secure.cs" />
<Compile Include="Events\EventHid.cs" />
<Compile Include="FormEditObject.cs">
@@ -223,6 +227,7 @@
<Compile Include="RichTextBoxTraceListener.cs" />
<Compile Include="Session.cs" />
<Compile Include="Settings.cs" />
+ <Compile Include="Spectrum\SpectrumBase.cs" />
<Compile Include="StartupManager.cs" />
<Compile Include="TaskScheduler.cs" />
<Compile Include="Win32API.cs" />
diff -r 10de0c7c2fed -r e5f85a895a62 Server/Spectrum/BasicSpectrumProvider.cs
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/Server/Spectrum/BasicSpectrumProvider.cs Mon Jan 02 18:43:45 2017 +0100
@@ -0,0 +1,54 @@
+using System;
+using System.Collections.Generic;
+using CSCore.DSP;
+
+namespace Visualization
+{
+ /// <summary>
+ /// BasicSpectrumProvider
+ /// </summary>
+ public class BasicSpectrumProvider : FftProvider, ISpectrumProvider
+ {
+ private readonly int _sampleRate;
+ private readonly List<object> _contexts = new List<object>();
+
+ public BasicSpectrumProvider(int channels, int sampleRate, FftSize fftSize)
+ : base(channels, fftSize)
+ {
+ if (sampleRate <= 0)
+ throw new ArgumentOutOfRangeException("sampleRate");
+ _sampleRate = sampleRate;
+ }
+
+ public int GetFftBandIndex(float frequency)
+ {
+ int fftSize = (int)FftSize;
+ double f = _sampleRate / 2.0;
+ // ReSharper disable once PossibleLossOfFraction
+ return (int)((frequency / f) * (fftSize / 2));
+ }
+
+ public bool GetFftData(float[] fftResultBuffer, object context)
+ {
+ if (_contexts.Contains(context))
+ return false;
+
+ _contexts.Add(context);
+ GetFftData(fftResultBuffer);
+ return true;
+ }
+
+ public override void Add(float[] samples, int count)
+ {
+ base.Add(samples, count);
+ if (count > 0)
+ _contexts.Clear();
+ }
+
+ public override void Add(float left, float right)
+ {
+ base.Add(left, right);
+ _contexts.Clear();
+ }
+ }
+}
\ No newline at end of file
diff -r 10de0c7c2fed -r e5f85a895a62 Server/Spectrum/ISpectrumProvider.cs
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/Server/Spectrum/ISpectrumProvider.cs Mon Jan 02 18:43:45 2017 +0100
@@ -0,0 +1,8 @@
+namespace Visualization
+{
+ public interface ISpectrumProvider
+ {
+ bool GetFftData(float[] fftBuffer, object context);
+ int GetFftBandIndex(float frequency);
+ }
+}
\ No newline at end of file
diff -r 10de0c7c2fed -r e5f85a895a62 Server/Spectrum/LineSpectrum.cs
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/Server/Spectrum/LineSpectrum.cs Mon Jan 02 18:43:45 2017 +0100
@@ -0,0 +1,166 @@
+using System;
+using System.ComponentModel;
+using System.Drawing;
+using System.Drawing.Drawing2D;
+using System.Drawing.Text;
+using CSCore.DSP;
+
+namespace Visualization
+{
+ public class LineSpectrum : SpectrumBase
+ {
+ private int _barCount;
+ private double _barSpacing;
+ private double _barWidth;
+ private Size _currentSize;
+
+ public LineSpectrum(FftSize fftSize)
+ {
+ FftSize = fftSize;
+ }
+
+ [Browsable(false)]
+ public double BarWidth
+ {
+ get { return _barWidth; }
+ }
+
+ public double BarSpacing
+ {
+ get { return _barSpacing; }
+ set
+ {
+ if (value < 0)
+ throw new ArgumentOutOfRangeException("value");
+ _barSpacing = value;
+ UpdateFrequencyMapping();
+
+ RaisePropertyChanged("BarSpacing");
+ RaisePropertyChanged("BarWidth");
+ }
+ }
+
+ public int BarCount
+ {
+ get { return _barCount; }
+ set
+ {
+ if (value <= 0)
+ throw new ArgumentOutOfRangeException("value");
+ _barCount = value;
+ SpectrumResolution = value;
+ UpdateFrequencyMapping();
+
+ RaisePropertyChanged("BarCount");
+ RaisePropertyChanged("BarWidth");
+ }
+ }
+
+ [BrowsableAttribute(false)]
+ public Size CurrentSize
+ {
+ get { return _currentSize; }
+ protected set
+ {
+ _currentSize = value;
+ RaisePropertyChanged("CurrentSize");
+ }
+ }
+
+ public Bitmap CreateSpectrumLine(Size size, Brush brush, Color background, bool highQuality)
+ {
+ if (!UpdateFrequencyMappingIfNessesary(size))
+ return null;
+
+ var fftBuffer = new float[(int)FftSize];
+
+ //get the fft result from the spectrum provider
+ if (SpectrumProvider.GetFftData(fftBuffer, this))
+ {
+ using (var pen = new Pen(brush, (float)_barWidth))
+ {
+ var bitmap = new Bitmap(size.Width, size.Height);
+
+ using (Graphics graphics = Graphics.FromImage(bitmap))
+ {
+ PrepareGraphics(graphics, highQuality);
+ graphics.Clear(background);
+
+ CreateSpectrumLineInternal(graphics, pen, fftBuffer, size);
+ }
+
+ return bitmap;
+ }
+ }
+ return null;
+ }
+
+ public Bitmap CreateSpectrumLine(Size size, Color color1, Color color2, Color background, bool highQuality)
+ {
+ if (!UpdateFrequencyMappingIfNessesary(size))
+ return null;
+
+ using (
+ Brush brush = new LinearGradientBrush(new RectangleF(0, 0, (float)_barWidth, size.Height), color2,
+ color1, LinearGradientMode.Vertical))
+ {
+ return CreateSpectrumLine(size, brush, background, highQuality);
+ }
+ }
+
+ private void CreateSpectrumLineInternal(Graphics graphics, Pen pen, float[] fftBuffer, Size size)
+ {
+ int height = size.Height;
+ //prepare the fft result for rendering
+ SpectrumPointData[] spectrumPoints = CalculateSpectrumPoints(height, fftBuffer);
+
+ //connect the calculated points with lines
+ for (int i = 0; i < spectrumPoints.Length; i++)
+ {
+ SpectrumPointData p = spectrumPoints[i];
+ int barIndex = p.SpectrumPointIndex;
+ double xCoord = BarSpacing * (barIndex + 1) + (_barWidth * barIndex) + _barWidth / 2;
+
+ var p1 = new PointF((float)xCoord, height);
+ var p2 = new PointF((float)xCoord, height - (float)p.Value - 1);
+
+ graphics.DrawLine(pen, p1, p2);
+ }
+ }
+
+ protected override void UpdateFrequencyMapping()
+ {
+ _barWidth = Math.Max(((_currentSize.Width - (BarSpacing * (BarCount + 1))) / BarCount), 0.00001);
+ base.UpdateFrequencyMapping();
+ }
+
+ private bool UpdateFrequencyMappingIfNessesary(Size newSize)
+ {
+ if (newSize != CurrentSize)
+ {
+ CurrentSize = newSize;
+ UpdateFrequencyMapping();
+ }
+
+ return newSize.Width > 0 && newSize.Height > 0;
+ }
+
+ private void PrepareGraphics(Graphics graphics, bool highQuality)
+ {
+ if (highQuality)
+ {
+ graphics.SmoothingMode = SmoothingMode.AntiAlias;
+ graphics.CompositingQuality = CompositingQuality.AssumeLinear;
+ graphics.PixelOffsetMode = PixelOffsetMode.Default;
+ graphics.TextRenderingHint = TextRenderingHint.ClearTypeGridFit;
+ }
+ else
+ {
+ graphics.SmoothingMode = SmoothingMode.HighSpeed;
+ graphics.CompositingQuality = CompositingQuality.HighSpeed;
+ graphics.PixelOffsetMode = PixelOffsetMode.None;
+ graphics.TextRenderingHint = TextRenderingHint.SingleBitPerPixelGridFit;
+ }
+ }
+ }
+}
\ No newline at end of file
diff -r 10de0c7c2fed -r e5f85a895a62 Server/Spectrum/ScalingStrategy.cs
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/Server/Spectrum/ScalingStrategy.cs Mon Jan 02 18:43:45 2017 +0100
@@ -0,0 +1,9 @@
+namespace Visualization
+{
+ public enum ScalingStrategy
+ {
+ Decibel,
+ Linear,
+ Sqrt
+ }
+}
\ No newline at end of file
diff -r 10de0c7c2fed -r e5f85a895a62 Server/Spectrum/SpectrumBase.cs
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/Server/Spectrum/SpectrumBase.cs Mon Jan 02 18:43:45 2017 +0100
@@ -0,0 +1,228 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Diagnostics;
+using CSCore;
+using CSCore.DSP;
+
+namespace Visualization
+{
+ public class SpectrumBase : INotifyPropertyChanged
+ {
+ private const int ScaleFactorLinear = 9;
+ protected const int ScaleFactorSqr = 2;
+ protected const double MinDbValue = -90;
+ protected const double MaxDbValue = 0;
+ protected const double DbScale = (MaxDbValue - MinDbValue);
+
+ private int _fftSize;
+ private bool _isXLogScale;
+ private int _maxFftIndex;
+ private int _maximumFrequency = 20000;
+ private int _maximumFrequencyIndex;
+ private int _minimumFrequency = 20; //Default spectrum from 20Hz to 20kHz
+ private int _minimumFrequencyIndex;
+ private ScalingStrategy _scalingStrategy;
+ private int[] _spectrumIndexMax;
+ private int[] _spectrumLogScaleIndexMax;
+ private ISpectrumProvider _spectrumProvider;
+
+ protected int SpectrumResolution;
+ private bool _useAverage;
+
+ public int MaximumFrequency
+ {
+ get { return _maximumFrequency; }
+ set
+ {
+ if (value <= MinimumFrequency)
+ {
+ throw new ArgumentOutOfRangeException("value",
+ "Value must not be less or equal the MinimumFrequency.");
+ }
+ _maximumFrequency = value;
+ UpdateFrequencyMapping();
+
+ RaisePropertyChanged("MaximumFrequency");
+ }
+ }
+
+ public int MinimumFrequency
+ {
+ get { return _minimumFrequency; }
+ set
+ {
+ if (value < 0)
+ throw new ArgumentOutOfRangeException("value");
+ _minimumFrequency = value;
+ UpdateFrequencyMapping();
+
+ RaisePropertyChanged("MinimumFrequency");
+ }
+ }
+
+ [BrowsableAttribute(false)]
+ public ISpectrumProvider SpectrumProvider
+ {
+ get { return _spectrumProvider; }
+ set
+ {
+ if (value == null)
+ throw new ArgumentNullException("value");
+ _spectrumProvider = value;
+
+ RaisePropertyChanged("SpectrumProvider");
+ }
+ }
+
+ public bool IsXLogScale
+ {
+ get { return _isXLogScale; }
+ set
+ {
+ _isXLogScale = value;
+ UpdateFrequencyMapping();
+ RaisePropertyChanged("IsXLogScale");
+ }
+ }
+
+ public ScalingStrategy ScalingStrategy
+ {
+ get { return _scalingStrategy; }
+ set
+ {
+ _scalingStrategy = value;
+ RaisePropertyChanged("ScalingStrategy");
+ }
+ }
+
+ public bool UseAverage
+ {
+ get { return _useAverage; }
+ set
+ {
+ _useAverage = value;
+ RaisePropertyChanged("UseAverage");
+ }
+ }
+
+ [BrowsableAttribute(false)]
+ public FftSize FftSize
+ {
+ get { return (FftSize) _fftSize; }
+ protected set
+ {
+ if ((int) Math.Log((int) value, 2) % 1 != 0)
+ throw new ArgumentOutOfRangeException("value");
+
+ _fftSize = (int) value;
+ _maxFftIndex = _fftSize / 2 - 1;
+
+ RaisePropertyChanged("FFTSize");
+ }
+ }
+
+ public event PropertyChangedEventHandler PropertyChanged;
+
+ protected virtual void UpdateFrequencyMapping()
+ {
+ _maximumFrequencyIndex = Math.Min(_spectrumProvider.GetFftBandIndex(MaximumFrequency) + 1, _maxFftIndex);
+ _minimumFrequencyIndex = Math.Min(_spectrumProvider.GetFftBandIndex(MinimumFrequency), _maxFftIndex);
+
+ int actualResolution = SpectrumResolution;
+
+ int indexCount = _maximumFrequencyIndex - _minimumFrequencyIndex;
+ double linearIndexBucketSize = Math.Round(indexCount / (double) actualResolution, 3);
+
+ _spectrumIndexMax = _spectrumIndexMax.CheckBuffer(actualResolution, true);
+ _spectrumLogScaleIndexMax = _spectrumLogScaleIndexMax.CheckBuffer(actualResolution, true);
+
+ double maxLog = Math.Log(actualResolution, actualResolution);
+ for (int i = 1; i < actualResolution; i++)
+ {
+ int logIndex =
+ (int) ((maxLog - Math.Log((actualResolution + 1) - i, (actualResolution + 1))) * indexCount) +
+ _minimumFrequencyIndex;
+
+ _spectrumIndexMax[i - 1] = _minimumFrequencyIndex + (int) (i * linearIndexBucketSize);
+ _spectrumLogScaleIndexMax[i - 1] = logIndex;
+ }
+
+ if (actualResolution > 0)
+ {
+ _spectrumIndexMax[_spectrumIndexMax.Length - 1] =
+ _spectrumLogScaleIndexMax[_spectrumLogScaleIndexMax.Length - 1] = _maximumFrequencyIndex;
+ }
+ }
+
+ protected virtual SpectrumPointData[] CalculateSpectrumPoints(double maxValue, float[] fftBuffer)
+ {
+ var dataPoints = new List<SpectrumPointData>();
+
+ double value0 = 0, value = 0;
+ double lastValue = 0;
+ double actualMaxValue = maxValue;
+ int spectrumPointIndex = 0;
+
+ for (int i = _minimumFrequencyIndex; i <= _maximumFrequencyIndex; i++)
+ {
+ switch (ScalingStrategy)
+ {
+ case ScalingStrategy.Decibel:
+ value0 = (((20 * Math.Log10(fftBuffer[i])) - MinDbValue) / DbScale) * actualMaxValue;
+ break;
+ case ScalingStrategy.Linear:
+ value0 = (fftBuffer[i] * ScaleFactorLinear) * actualMaxValue;
+ break;
+ case ScalingStrategy.Sqrt:
+ value0 = ((Math.Sqrt(fftBuffer[i])) * ScaleFactorSqr) * actualMaxValue;
+ break;
+ }
+
+ bool recalc = true;
+
+ value = Math.Max(0, Math.Max(value0, value));
+
+ while (spectrumPointIndex <= _spectrumIndexMax.Length - 1 &&
+ i ==
+ (IsXLogScale
+ ? _spectrumLogScaleIndexMax[spectrumPointIndex]
+ : _spectrumIndexMax[spectrumPointIndex]))
+ {
+ if (!recalc)
+ value = lastValue;
+
+ if (value > maxValue)
+ value = maxValue;
+
+ if (_useAverage && spectrumPointIndex > 0)
+ value = (lastValue + value) / 2.0;
+
+ dataPoints.Add(new SpectrumPointData {SpectrumPointIndex = spectrumPointIndex, Value = value});
+
+ lastValue = value;
+ value = 0.0;
+ spectrumPointIndex++;
+ recalc = false;
+ }
+
+ //value = 0;
+ }
+
+ return dataPoints.ToArray();
+ }
+
+ protected void RaisePropertyChanged(string propertyName)
+ {
+ if (PropertyChanged != null && !String.IsNullOrEmpty(propertyName))
+ PropertyChanged(this, new PropertyChangedEventArgs(propertyName));
+ }
+
+ [DebuggerDisplay("{Value}")]
+ protected struct SpectrumPointData
+ {
+ public int SpectrumPointIndex;
+ public double Value;
+ }
+ }
+}
\ No newline at end of file