# HG changeset patch # User StephaneLenclud # Date 1483379025 -3600 # Node ID e5f85a895a6208e06848d4ec666db6660e666332 # Parent 10de0c7c2fed751a0da5a6606a24c2e077cfdc58 Draft audio spectrum visualizer. diff -r 10de0c7c2fed -r e5f85a895a62 Server/FormMain.cs --- a/Server/FormMain.cs Mon Jan 02 15:50:50 2017 +0100 +++ b/Server/FormMain.cs Mon Jan 02 18:43:45 2017 +0100 @@ -37,8 +37,14 @@ using System.Runtime.InteropServices; using System.Security; //CSCore +using CSCore; +using CSCore.Win32; +using CSCore.DSP; +using CSCore.Streams; using CSCore.CoreAudioAPI; - +using CSCore.SoundIn; +// Visualization +using Visualization; // CEC using CecSharp; //Network @@ -49,7 +55,7 @@ using MiniDisplayInterop; using SharpLib.Display; using Ear = SharpLib.Ear; -using CSCore.Win32; + namespace SharpDisplayManager { @@ -105,9 +111,15 @@ //Function pointer for pixel Y coordinate intercept CoordinateTranslationDelegate iScreenY; //CSCore + // Volume management private MMDeviceEnumerator iMultiMediaDeviceEnumerator; private MMDevice iMultiMediaDevice; private AudioEndpointVolume iAudioEndpointVolume; + // Audio visualization + private WasapiCapture iSoundIn; + private IWaveSource iWaveSource; + private LineSpectrum iLineSpectrum; + //Network private NetworkManager iNetworkManager; @@ -641,6 +653,106 @@ /// /// /// + private void StartAudioVisualization() + { + StopAudioVisualization(); + //Open the default device + iSoundIn = new WasapiLoopbackCapture(); + //Our loopback capture opens the default render device by default so the following is not needed + //iSoundIn.Device = MMDeviceEnumerator.DefaultAudioEndpoint(DataFlow.Render, Role.Console); + iSoundIn.Initialize(); + + SoundInSource soundInSource = new SoundInSource(iSoundIn); + ISampleSource source = soundInSource.ToSampleSource(); + + const FftSize fftSize = FftSize.Fft4096; + //create a spectrum provider which provides fft data based on some input + BasicSpectrumProvider spectrumProvider = new BasicSpectrumProvider(source.WaveFormat.Channels, source.WaveFormat.SampleRate, fftSize); + + //linespectrum and voiceprint3dspectrum used for rendering some fft data + //in oder to get some fft data, set the previously created spectrumprovider + iLineSpectrum = new LineSpectrum(fftSize) + { + SpectrumProvider = spectrumProvider, + UseAverage = true, + BarCount = 32, + BarSpacing = 0, + IsXLogScale = true, + ScalingStrategy = ScalingStrategy.Sqrt + }; + + + //the SingleBlockNotificationStream is used to intercept the played samples + var notificationSource = new SingleBlockNotificationStream(source); + //pass the intercepted samples as input data to the spectrumprovider (which will calculate a fft based on them) + notificationSource.SingleBlockRead += (s, a) => spectrumProvider.Add(a.Left, a.Right); + + iWaveSource = notificationSource.ToWaveSource(16); + + + // We need to read from our source otherwise SingleBlockRead is never called and our spectrum provider is not populated + byte[] buffer = new byte[iWaveSource.WaveFormat.BytesPerSecond / 2]; + soundInSource.DataAvailable += (s, aEvent) => + { + int read; + while ((read = iWaveSource.Read(buffer, 0, buffer.Length)) > 0) ; + }; + + + //Start recording + iSoundIn.Start(); + } + + /// + /// + /// + private void StopAudioVisualization() + { + + if (iSoundIn != null) + { + iSoundIn.Stop(); + iSoundIn.Dispose(); + iSoundIn = null; + } + if (iWaveSource != null) + { + iWaveSource.Dispose(); + iWaveSource = null; + } + + } + + + /// + /// + /// + private void GenerateAudioVisualization() + { + // For demo draft purposes just fetch the firt picture box control and update it with current audio spectrum + foreach (Control ctrl in iTableLayoutPanel.Controls) + { + if (ctrl is PictureBox) + { + PictureBox pb = (PictureBox)ctrl; + Image image = pb.Image; + var newImage = iLineSpectrum.CreateSpectrumLine(pb.Size, Color.Black, Color.Black, Color.White, false); + if (newImage != null) + { + pb.Image = newImage; + if (image != null) + image.Dispose(); + } + + break; + } + } + } + + + /// + /// + /// private void UpdateAudioDeviceAndMasterVolumeThreadSafe() { if (this.InvokeRequired) @@ -657,6 +769,7 @@ //Get our master volume iMultiMediaDevice = iMultiMediaDeviceEnumerator.GetDefaultAudioEndpoint(DataFlow.Render, Role.Multimedia); iAudioEndpointVolume = AudioEndpointVolume.FromDevice(iMultiMediaDevice); + //Update our label labelDefaultAudioDevice.Text = iMultiMediaDevice.FriendlyName; @@ -667,7 +780,9 @@ AudioEndpointVolumeCallback callback = new AudioEndpointVolumeCallback(); callback.NotifyRecived += OnVolumeNotificationThreadSafe; // Do we need to unregister? - iAudioEndpointVolume.RegisterControlChangeNotify(callback); + iAudioEndpointVolume.RegisterControlChangeNotify(callback); + // + StartAudioVisualization(); // trackBarMasterVolume.Enabled = true; } @@ -1105,8 +1220,10 @@ } } + GenerateAudioVisualization(); + //Compute instant FPS - toolStripStatusLabelFps.Text = (1.0/NewTickTime.Subtract(LastTickTime).TotalSeconds).ToString("F0") + " / " + + toolStripStatusLabelFps.Text = (1.0/NewTickTime.Subtract(LastTickTime).TotalSeconds).ToString("F0") + " / " + (1000/iTimerDisplay.Interval).ToString() + " FPS"; LastTickTime = NewTickTime; @@ -1441,6 +1558,8 @@ private void MainForm_FormClosing(object sender, FormClosingEventArgs e) { + //TODO: discard other CSCore audio objects + StopAudioVisualization(); iCecManager.Stop(); iNetworkManager.Dispose(); CloseDisplayConnection(); diff -r 10de0c7c2fed -r e5f85a895a62 Server/SharpDisplayManager.csproj --- a/Server/SharpDisplayManager.csproj Mon Jan 02 15:50:50 2017 +0100 +++ b/Server/SharpDisplayManager.csproj Mon Jan 02 18:43:45 2017 +0100 @@ -174,11 +174,15 @@ + + + + @@ -223,6 +227,7 @@ + diff -r 10de0c7c2fed -r e5f85a895a62 Server/Spectrum/BasicSpectrumProvider.cs --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Server/Spectrum/BasicSpectrumProvider.cs Mon Jan 02 18:43:45 2017 +0100 @@ -0,0 +1,54 @@ +using System; +using System.Collections.Generic; +using CSCore.DSP; + +namespace Visualization +{ + /// + /// BasicSpectrumProvider + /// + public class BasicSpectrumProvider : FftProvider, ISpectrumProvider + { + private readonly int _sampleRate; + private readonly List _contexts = new List(); + + public BasicSpectrumProvider(int channels, int sampleRate, FftSize fftSize) + : base(channels, fftSize) + { + if (sampleRate <= 0) + throw new ArgumentOutOfRangeException("sampleRate"); + _sampleRate = sampleRate; + } + + public int GetFftBandIndex(float frequency) + { + int fftSize = (int)FftSize; + double f = _sampleRate / 2.0; + // ReSharper disable once PossibleLossOfFraction + return (int)((frequency / f) * (fftSize / 2)); + } + + public bool GetFftData(float[] fftResultBuffer, object context) + { + if (_contexts.Contains(context)) + return false; + + _contexts.Add(context); + GetFftData(fftResultBuffer); + return true; + } + + public override void Add(float[] samples, int count) + { + base.Add(samples, count); + if (count > 0) + _contexts.Clear(); + } + + public override void Add(float left, float right) + { + base.Add(left, right); + _contexts.Clear(); + } + } +} \ No newline at end of file diff -r 10de0c7c2fed -r e5f85a895a62 Server/Spectrum/ISpectrumProvider.cs --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Server/Spectrum/ISpectrumProvider.cs Mon Jan 02 18:43:45 2017 +0100 @@ -0,0 +1,8 @@ +namespace Visualization +{ + public interface ISpectrumProvider + { + bool GetFftData(float[] fftBuffer, object context); + int GetFftBandIndex(float frequency); + } +} \ No newline at end of file diff -r 10de0c7c2fed -r e5f85a895a62 Server/Spectrum/LineSpectrum.cs --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Server/Spectrum/LineSpectrum.cs Mon Jan 02 18:43:45 2017 +0100 @@ -0,0 +1,166 @@ +using System; +using System.ComponentModel; +using System.Drawing; +using System.Drawing.Drawing2D; +using System.Drawing.Text; +using CSCore.DSP; + +namespace Visualization +{ + public class LineSpectrum : SpectrumBase + { + private int _barCount; + private double _barSpacing; + private double _barWidth; + private Size _currentSize; + + public LineSpectrum(FftSize fftSize) + { + FftSize = fftSize; + } + + [Browsable(false)] + public double BarWidth + { + get { return _barWidth; } + } + + public double BarSpacing + { + get { return _barSpacing; } + set + { + if (value < 0) + throw new ArgumentOutOfRangeException("value"); + _barSpacing = value; + UpdateFrequencyMapping(); + + RaisePropertyChanged("BarSpacing"); + RaisePropertyChanged("BarWidth"); + } + } + + public int BarCount + { + get { return _barCount; } + set + { + if (value <= 0) + throw new ArgumentOutOfRangeException("value"); + _barCount = value; + SpectrumResolution = value; + UpdateFrequencyMapping(); + + RaisePropertyChanged("BarCount"); + RaisePropertyChanged("BarWidth"); + } + } + + [BrowsableAttribute(false)] + public Size CurrentSize + { + get { return _currentSize; } + protected set + { + _currentSize = value; + RaisePropertyChanged("CurrentSize"); + } + } + + public Bitmap CreateSpectrumLine(Size size, Brush brush, Color background, bool highQuality) + { + if (!UpdateFrequencyMappingIfNessesary(size)) + return null; + + var fftBuffer = new float[(int)FftSize]; + + //get the fft result from the spectrum provider + if (SpectrumProvider.GetFftData(fftBuffer, this)) + { + using (var pen = new Pen(brush, (float)_barWidth)) + { + var bitmap = new Bitmap(size.Width, size.Height); + + using (Graphics graphics = Graphics.FromImage(bitmap)) + { + PrepareGraphics(graphics, highQuality); + graphics.Clear(background); + + CreateSpectrumLineInternal(graphics, pen, fftBuffer, size); + } + + return bitmap; + } + } + return null; + } + + public Bitmap CreateSpectrumLine(Size size, Color color1, Color color2, Color background, bool highQuality) + { + if (!UpdateFrequencyMappingIfNessesary(size)) + return null; + + using ( + Brush brush = new LinearGradientBrush(new RectangleF(0, 0, (float)_barWidth, size.Height), color2, + color1, LinearGradientMode.Vertical)) + { + return CreateSpectrumLine(size, brush, background, highQuality); + } + } + + private void CreateSpectrumLineInternal(Graphics graphics, Pen pen, float[] fftBuffer, Size size) + { + int height = size.Height; + //prepare the fft result for rendering + SpectrumPointData[] spectrumPoints = CalculateSpectrumPoints(height, fftBuffer); + + //connect the calculated points with lines + for (int i = 0; i < spectrumPoints.Length; i++) + { + SpectrumPointData p = spectrumPoints[i]; + int barIndex = p.SpectrumPointIndex; + double xCoord = BarSpacing * (barIndex + 1) + (_barWidth * barIndex) + _barWidth / 2; + + var p1 = new PointF((float)xCoord, height); + var p2 = new PointF((float)xCoord, height - (float)p.Value - 1); + + graphics.DrawLine(pen, p1, p2); + } + } + + protected override void UpdateFrequencyMapping() + { + _barWidth = Math.Max(((_currentSize.Width - (BarSpacing * (BarCount + 1))) / BarCount), 0.00001); + base.UpdateFrequencyMapping(); + } + + private bool UpdateFrequencyMappingIfNessesary(Size newSize) + { + if (newSize != CurrentSize) + { + CurrentSize = newSize; + UpdateFrequencyMapping(); + } + + return newSize.Width > 0 && newSize.Height > 0; + } + + private void PrepareGraphics(Graphics graphics, bool highQuality) + { + if (highQuality) + { + graphics.SmoothingMode = SmoothingMode.AntiAlias; + graphics.CompositingQuality = CompositingQuality.AssumeLinear; + graphics.PixelOffsetMode = PixelOffsetMode.Default; + graphics.TextRenderingHint = TextRenderingHint.ClearTypeGridFit; + } + else + { + graphics.SmoothingMode = SmoothingMode.HighSpeed; + graphics.CompositingQuality = CompositingQuality.HighSpeed; + graphics.PixelOffsetMode = PixelOffsetMode.None; + graphics.TextRenderingHint = TextRenderingHint.SingleBitPerPixelGridFit; + } + } + } +} \ No newline at end of file diff -r 10de0c7c2fed -r e5f85a895a62 Server/Spectrum/ScalingStrategy.cs --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Server/Spectrum/ScalingStrategy.cs Mon Jan 02 18:43:45 2017 +0100 @@ -0,0 +1,9 @@ +namespace Visualization +{ + public enum ScalingStrategy + { + Decibel, + Linear, + Sqrt + } +} \ No newline at end of file diff -r 10de0c7c2fed -r e5f85a895a62 Server/Spectrum/SpectrumBase.cs --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Server/Spectrum/SpectrumBase.cs Mon Jan 02 18:43:45 2017 +0100 @@ -0,0 +1,228 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using CSCore; +using CSCore.DSP; + +namespace Visualization +{ + public class SpectrumBase : INotifyPropertyChanged + { + private const int ScaleFactorLinear = 9; + protected const int ScaleFactorSqr = 2; + protected const double MinDbValue = -90; + protected const double MaxDbValue = 0; + protected const double DbScale = (MaxDbValue - MinDbValue); + + private int _fftSize; + private bool _isXLogScale; + private int _maxFftIndex; + private int _maximumFrequency = 20000; + private int _maximumFrequencyIndex; + private int _minimumFrequency = 20; //Default spectrum from 20Hz to 20kHz + private int _minimumFrequencyIndex; + private ScalingStrategy _scalingStrategy; + private int[] _spectrumIndexMax; + private int[] _spectrumLogScaleIndexMax; + private ISpectrumProvider _spectrumProvider; + + protected int SpectrumResolution; + private bool _useAverage; + + public int MaximumFrequency + { + get { return _maximumFrequency; } + set + { + if (value <= MinimumFrequency) + { + throw new ArgumentOutOfRangeException("value", + "Value must not be less or equal the MinimumFrequency."); + } + _maximumFrequency = value; + UpdateFrequencyMapping(); + + RaisePropertyChanged("MaximumFrequency"); + } + } + + public int MinimumFrequency + { + get { return _minimumFrequency; } + set + { + if (value < 0) + throw new ArgumentOutOfRangeException("value"); + _minimumFrequency = value; + UpdateFrequencyMapping(); + + RaisePropertyChanged("MinimumFrequency"); + } + } + + [BrowsableAttribute(false)] + public ISpectrumProvider SpectrumProvider + { + get { return _spectrumProvider; } + set + { + if (value == null) + throw new ArgumentNullException("value"); + _spectrumProvider = value; + + RaisePropertyChanged("SpectrumProvider"); + } + } + + public bool IsXLogScale + { + get { return _isXLogScale; } + set + { + _isXLogScale = value; + UpdateFrequencyMapping(); + RaisePropertyChanged("IsXLogScale"); + } + } + + public ScalingStrategy ScalingStrategy + { + get { return _scalingStrategy; } + set + { + _scalingStrategy = value; + RaisePropertyChanged("ScalingStrategy"); + } + } + + public bool UseAverage + { + get { return _useAverage; } + set + { + _useAverage = value; + RaisePropertyChanged("UseAverage"); + } + } + + [BrowsableAttribute(false)] + public FftSize FftSize + { + get { return (FftSize) _fftSize; } + protected set + { + if ((int) Math.Log((int) value, 2) % 1 != 0) + throw new ArgumentOutOfRangeException("value"); + + _fftSize = (int) value; + _maxFftIndex = _fftSize / 2 - 1; + + RaisePropertyChanged("FFTSize"); + } + } + + public event PropertyChangedEventHandler PropertyChanged; + + protected virtual void UpdateFrequencyMapping() + { + _maximumFrequencyIndex = Math.Min(_spectrumProvider.GetFftBandIndex(MaximumFrequency) + 1, _maxFftIndex); + _minimumFrequencyIndex = Math.Min(_spectrumProvider.GetFftBandIndex(MinimumFrequency), _maxFftIndex); + + int actualResolution = SpectrumResolution; + + int indexCount = _maximumFrequencyIndex - _minimumFrequencyIndex; + double linearIndexBucketSize = Math.Round(indexCount / (double) actualResolution, 3); + + _spectrumIndexMax = _spectrumIndexMax.CheckBuffer(actualResolution, true); + _spectrumLogScaleIndexMax = _spectrumLogScaleIndexMax.CheckBuffer(actualResolution, true); + + double maxLog = Math.Log(actualResolution, actualResolution); + for (int i = 1; i < actualResolution; i++) + { + int logIndex = + (int) ((maxLog - Math.Log((actualResolution + 1) - i, (actualResolution + 1))) * indexCount) + + _minimumFrequencyIndex; + + _spectrumIndexMax[i - 1] = _minimumFrequencyIndex + (int) (i * linearIndexBucketSize); + _spectrumLogScaleIndexMax[i - 1] = logIndex; + } + + if (actualResolution > 0) + { + _spectrumIndexMax[_spectrumIndexMax.Length - 1] = + _spectrumLogScaleIndexMax[_spectrumLogScaleIndexMax.Length - 1] = _maximumFrequencyIndex; + } + } + + protected virtual SpectrumPointData[] CalculateSpectrumPoints(double maxValue, float[] fftBuffer) + { + var dataPoints = new List(); + + double value0 = 0, value = 0; + double lastValue = 0; + double actualMaxValue = maxValue; + int spectrumPointIndex = 0; + + for (int i = _minimumFrequencyIndex; i <= _maximumFrequencyIndex; i++) + { + switch (ScalingStrategy) + { + case ScalingStrategy.Decibel: + value0 = (((20 * Math.Log10(fftBuffer[i])) - MinDbValue) / DbScale) * actualMaxValue; + break; + case ScalingStrategy.Linear: + value0 = (fftBuffer[i] * ScaleFactorLinear) * actualMaxValue; + break; + case ScalingStrategy.Sqrt: + value0 = ((Math.Sqrt(fftBuffer[i])) * ScaleFactorSqr) * actualMaxValue; + break; + } + + bool recalc = true; + + value = Math.Max(0, Math.Max(value0, value)); + + while (spectrumPointIndex <= _spectrumIndexMax.Length - 1 && + i == + (IsXLogScale + ? _spectrumLogScaleIndexMax[spectrumPointIndex] + : _spectrumIndexMax[spectrumPointIndex])) + { + if (!recalc) + value = lastValue; + + if (value > maxValue) + value = maxValue; + + if (_useAverage && spectrumPointIndex > 0) + value = (lastValue + value) / 2.0; + + dataPoints.Add(new SpectrumPointData {SpectrumPointIndex = spectrumPointIndex, Value = value}); + + lastValue = value; + value = 0.0; + spectrumPointIndex++; + recalc = false; + } + + //value = 0; + } + + return dataPoints.ToArray(); + } + + protected void RaisePropertyChanged(string propertyName) + { + if (PropertyChanged != null && !String.IsNullOrEmpty(propertyName)) + PropertyChanged(this, new PropertyChangedEventArgs(propertyName)); + } + + [DebuggerDisplay("{Value}")] + protected struct SpectrumPointData + { + public int SpectrumPointIndex; + public double Value; + } + } +} \ No newline at end of file