对python中Librosa的mfcc步骤详解

Python  /  管理员 发布于 7年前   482

1.对语音数据归一化

如16000hz的数据，会将每个点/32768

2.计算窗函数：（*注意librosa中不进行预处理）

3.进行数据扩展填充，他进行的是镜像填充（"reflect")

如原数据为 12345 -》 填充为4的，左右各填充4 即：5432123454321 即：5432-12345-4321

4.分帧

5.加窗：对每一帧进行加窗，

6.进行fft傅里叶变换

librosa中fft计算，可以使用.net中的System.Numerics

MathNet.Numerics.IntegralTransforms.Fourier.Forward(FFT_frame, FourierOptions.Matlab) 计算，结果相同

7.mel计算（每一帧取20个特征点）

Imports System.NumericsImports MathNet.NumericsImports MathNet.Numerics.IntegralTransforms Module mfcc_module   Public Class Librosa   End Class   Dim pi As Double = 3.1415926535897931       Public Function spectrum(fft_data(,) As Complex) As Double(,)    Dim new_data(fft_data.GetLength(0) - 1, fft_data.GetLength(1) - 1) As Double      For n = 0 To fft_data.GetLength(0) - 1      ' Debug.Print("////////////////////////spectrum//////////////////")      ' Debug.Print("////////////////////////spectrum//////////////////")      For i = 0 To fft_data.GetLength(1) - 1        new_data(n, i) = fft_data(n, i).MagnitudeSquared        ' Debug.Write(new_data(n, i) & "  ")      Next    Next     Return new_data   End Function    Public Function FFT(data As Double(,)) As Complex(,)     Dim result(data.GetLength(0) - 1, 1024) As Complex    '2049 加了一个 数组类型 0 开始    Dim FFT_frame As Complex() = New Complex(data.GetLength(1) - 1) {}     For n = 0 To data.GetLength(0) - 1      For i As Integer = 0 To data.GetLength(1) - 1        FFT_frame(i) = data(n, i)      Next      MathNet.Numerics.IntegralTransforms.Fourier.Forward(FFT_frame, FourierOptions.Matlab)       For k = 0 To 1024        result(n, k) = FFT_frame(k)      Next       'Debug.Print("fft **************")      'For Each mem In FFT_frame      '  Debug.Print(mem.ToString & "  ")      'Next      Next n       Return result    End Function   Public Function _mfcc(dct_ As Double(,）, power_to_db_ As Double(,)) As Double(,)    'dct 20,128    'power_to_db 5,128    'result = 20,5    Dim result(dct_.GetLength(0) - 1, power_to_db_.GetLength(1) - 1) As Double    Dim r1, r2 As Double    For n = 0 To dct_.GetLength(0) - 1 '20      For i = 0 To power_to_db_.GetLength(1) - 1 '5        r2 = 0        For k = 0 To dct_.GetLength(1) - 1 '128          r1 = dct_(n, k) * power_to_db_(k, i)          r2 = r2 + r1         Next        result(n, i) = r2      Next    Next     Return result  End Function   Public Function Dct(n_filters As Integer, n_input As Integer) As Double(,)     Dim t1 As Double = 2 * n_input     Dim samples(n_input - 1) As Double    Dim basis(n_filters - 1, n_input - 1） As Double     Dim n As Integer = 1    For i = 0 To n_input - 1      samples(i) = n * pi / (2 * n_input)      n = n + 2    Next i     For i = 0 To n_input - 1      basis(0, i) = 1 / Math.Sqrt(n_input)    Next    For n = 1 To n_filters - 1      For i = 0 To n_input - 1        basis(n, i) = Math.Cos(n * samples(i)) * Math.Sqrt(2 / n_input)       Next    Next     Return basis  End Function    '1e-10 = 0.0000000001  Public Function power_to_db(S As Double(,), Optional ref As Double = 1, Optional admin As Double = 0.0000000001, Optional top_db As Double = 80) As Double(,)     Dim result(S.GetLength(0) - 1, S.GetLength(1) - 1） As Double     Dim log_spec As Double      For n = 0 To S.GetLength(0) - 1      For i = 0 To S.GetLength(1) - 1         log_spec = 10 * Math.Log10(Math.Max(admin, S(n, i)))        result(n, i) = log_spec - 10 * Math.Log10(Math.Max(admin, ref))      Next    Next       'If top_db <> 0 Then    '  For n = 0 To S.GetLength(0) - 1    '    For i = 0 To S.GetLength(1) - 1    '      'result(n, i) = Math.Max(result(n, i), result(n, i) - top_db)    '    Next    '  Next     'End If     Return result     End Function     Public Function melspectrogram(mel_basis(,) As Double, s(,) As Double) As Double(,)    'mel_basis 128,1025    's 5 ,1025 -> 1025,5    ' result 128,5    Dim result(mel_basis.GetLength(0) - 3, s.GetLength(0) - 1) As Double    Dim r1, r2 As Double     For n = 0 To mel_basis.GetLength(0) - 3       For i = 0 To s.GetLength(0) - 1        For k = 0 To mel_basis.GetLength(1) - 1r1 = mel_basis(n, k) * s(i, k)          r2 = r2 + r1        Next        result(n, i) = r2        r2 = 0      Next     Next    Return result   End Function   Public Function normal(mel_f As Double(), weights(,) As Double) As Double(,)    Dim enorm(mel_f.Length - 2) As Double     ' Debug.Print("*************normal//////////////")    ' Debug.Print("*************normal//////////////")    For i = 0 To mel_f.Length - 3      enorm(i) = 2 / (mel_f(2 + i) - mel_f(i))    Next      For i = 0 To weights.GetLength(1) - 1      For n = 0 To weights.GetLength(0) - 2        weights(n, i) = weights(n, i) * enorm(n)      Next    Next    Return weights  End Function    Public Function weight(a As Double(,), fdiff As Double()) As Double(,)    Dim lower, upper As Double     Dim data(a.GetLength(0) - 1, a.GetLength(1) - 1) As Double     For n = 0 To a.GetLength(0) - 3      For i = 0 To a.GetLength(1) - 1        lower = -(a(n, i) / fdiff(n))        upper = a(n + 2, i) / fdiff(n + 1)        data(n, i) = Math.Max(0, Math.Min(lower, upper))      Next    Next    Return data  End Function   Public Function ramps(A As Double(), B As Double()) As Double(,)    Dim data(A.Length - 1, B.Length - 1) As Double     ' Debug.Print("ramps*********************")    For n = 0 To A.Length - 1      'Debug.Print("******")      'Debug.Print("------")      For i = 0 To B.Length - 1        data(n, i) = A(n) - B(i)        'Debug.Write(data(n, i) & "  ")      Next    Next    Return data   End Function  Public Function diff(arr As Double()) As Double()    Dim data(arr.Length - 2) As Double    For i = 1 To arr.Length - 1      data(i - 1) = arr(i) - arr(i - 1)      'Debug.Print(data(i - 1))    Next     Return data  End Function   '分帧 算法2  Public Function Frame2(y As Double(), Optional n_ftt As Integer = 2048, Optional hop As Integer = 512) As Double(,)    Dim tim As Integer = Math.Floor((y.Length - n_ftt) / hop) + 1    Dim new_buff(tim - 1, n_ftt - 1) As Double    Dim copypos As Integer = 0    For i = 0 To tim - 1      For k = 0 To n_ftt - 1        new_buff(i, k) = y(copypos + k)      Next      copypos = copypos + hop    Next     'For k = 0 To tim - 1    '  Debug.Print("//////////////////////////////////////")    '  Debug.Print("///////////////fram2///////////////////////" & k)    '  For i = 0 To n_ftt - 1    '    Debug.Print(new_buff(k, i) & " ")    '  Next    'Next k     Return new_buff    End Function   '  Public Function Frame(y As Double(), Optional n_ftt As Integer = 2048, Optional hop As Integer = 512) As Double()    Dim tim As Integer = Math.Floor((y.Length - n_ftt) / hop) + 1    Dim new_buff(tim * n_ftt) As Double    Dim pos As Integer = 0    Dim copypos As Integer = 0    For i = 0 To tim - 1      Array.Copy(y, copypos, new_buff, pos, n_ftt)      'Buffer.BlockCopy(y, 0, new_buff, pos, n_ftt)      copypos = copypos + hop      pos = pos + n_ftt    Next     For k = 0 To tim - 1      'Debug.Print("//////////////////////////////////////")      'Debug.Print("//////////////////////////////////////")      For i = 0 To n_ftt - 1        Debug.Write(new_buff(k * n_ftt + i) & " ")      Next    Next k     Return new_buff   End Function    Public Function MelFilter() As Double()    Dim filter_points（128 + 1） As Integer '40个滤波器，需要41点    Const sampleRate As Integer = 16000 '采样频率 16000    Const filterNum As Integer = 128 '滤波器数量 取40个    Const frameSize As Integer = 512 '帧长512     Dim freMax As Double = sampleRate / 2  '实际最大频率     Dim freMin As Double = 0  '实际最小频率     Dim melFremax As Double = hz_to_mel(freMax)   '将实际频率转换成梅尔频率     Dim melFremin As Double = 1125 * Math.Log(1 + freMin / 700)     Dim k As Double = (melFremax - melFremin) / (filterNum + 1)     Dim m As Double() = New Double(filterNum + 1) {}    Dim h As Double() = New Double(filterNum + 1) {}     For i As Integer = 0 To filterNum + 1      m(i) = melFremin + k * i      'h(i) = 700 * (Math.Exp(m(i) / 1125) - 1)      '将梅尔频率转换成实际频率       filter_points(i) = mel_to_hz(m(i))       'Debug.Print(m(i))    Next     Dim hzs As Double() = mel_to_hz2(m)    'For i = 0 To filterNum + 1    '  ' Debug.Print(hzs(i))    'Next    Return hzs    End Function   Public Function hz_to_mel(frequencies As Double, Optional htk As Boolean = False) As Double     Dim mels As Double     If htk Then      mels = 1125 * Math.Log(1 + frequencies / 700)    Else      Dim f_min As Double = 0.0      Dim f_sp As Double = 200.0 / 3      Dim min_log_hz As Double = 1000.0 ' beginning of log region (Hz)      Dim min_log_mel As Double = (min_log_hz - f_min) / f_sp  ' same (Mels)      Dim logstep As Double = Math.Log(6.4) / 27.0        ' step size for log region      mels = min_log_mel + Math.Log(frequencies / min_log_hz) / logstep    End If    Return mels  End Function   Public Function mel_to_hz2(mel() As Double, Optional htk As Boolean = False) As Double()    Dim hz(mel.Length - 1) As Double     Dim f_min As Double = 0.0    Dim f_sp As Double = 200.0 / 3    Dim freqs(mel.Length - 1) As Double     For i = 0 To mel.Length - 1      freqs(i) = f_min + f_sp * mel(i)    Next i     Dim min_log_hz As Double = 1000.0 ' beginning of log region (Hz)    Dim min_log_mel As Double = (min_log_hz - f_min) / f_sp  ' same (Mels)    Dim logstep As Double = Math.Log(6.4) / 27.0     For i = 0 To mel.Length - 1      If (mel(i) > min_log_mel) Then        freqs(i) = min_log_hz * Math.Exp(logstep * (mel(i) - min_log_mel))      End If     Next    'hz = min_log_hz * Math.Exp(logstep * (mel - min_log_mel))      Return freqs  End Function   Public Function mel_to_hz(mel As Double, Optional htk As Boolean = False) As Double    Dim hz As Double    If htk Then      hz = 700 * (Math.Exp(mel) / 1125) - 1    Else      Dim f_min As Double = 0.0      Dim f_sp As Double = 200.0 / 3      Dim freqs = f_min + f_sp * mel       Dim min_log_hz As Double = 1000.0 ' beginning of log region (Hz)      Dim min_log_mel As Double = (min_log_hz - f_min) / f_sp  ' same (Mels)      Dim logstep As Double = Math.Log(6.4) / 27.0      hz = min_log_hz * Math.Exp(logstep * (mel - min_log_mel))      'hz = min_log_hz * Math.Exp(logstep * (mel - min_log_mel))     End If    Return hz  End Function    Public Function fft_frequencies(sr As Integer, n_fft As Integer) As Double()    Dim fft_data(n_fft / 2) As Double    For i = 0 To n_fft / 2      fft_data(i) = i * sr / n_fft    Next    Return fft_data  End Function     '左右填充，优化  Public Function PadReflect2(data() As Double, num As Integer)    'pad 10 ,10     Dim tim(data.Length - 3) As Double    For i = 0 To data.Length - 3      tim(i) = data(data.Length - 2 - i)    Next     Dim dump() As Double = data.Concat(tim).ToArray()     'For Each i In dump    '  Debug.Write(i)  End Function   Public Function PadReflect(data() As Double, num As Integer)     'pad 10 ,10     Dim tim(data.Length - 3) As Double    For i = 0 To data.Length - 3      tim(i) = data(data.Length - 2 - i)    Next     Dim dump() As Double = data.Concat(tim).ToArray()     'For Each i In dump    '  Debug.Write(i)    'Next     'left_edge     ' Debug.Print("***************************")    Dim left_edge(num - 1) As Double    _CopyDup(left_edge, dump, True)    'For i = 0 To num - 1    '  Debug.Write(left_edge(i))    'Next     'right_edge    'Debug.Print("***************************")    Dim right_edge(num + data.Length) As Double    _CopyDup(right_edge, dump, False)    'For i = 0 To num - 1    '  Debug.Write(right_edge(i))    'Next    'Debug.Print("***************************")    Dim result As Double() = left_edge.Concat(right_edge).ToArray()    Return result   End Function   'copy tim to data dumply  Public Function _CopyDup(data() As Double, tim() As Double, Optional left As Boolean = True)    Dim last As Integer = data.Length Mod tim.Length    Dim times As Integer = Math.Floor(data.Length / tim.Length)    Dim pos As Integer    If left Then      Array.Copy(tim, tim.Length - last, data, 0, last)      pos = last      For i = 0 To times - 1        Array.Copy(tim, 0, data, pos, tim.Length)        pos = pos + tim.Length      Next     Else       'RIGHT      pos = 0      For i = 0 To times - 1        Array.Copy(tim, 0, data, pos, tim.Length)        pos = pos + tim.Length      Next       Array.Copy(tim, 0, data, pos, last)     End If    End Function     Public Function General_cosine(M As Integer, alpha As Double(), sym As Boolean) As Double()     If Not sym Then      M = M + 1    End If      Dim tim As Double = (2 * pi) / (M - 1)    Dim x(M) As Double    Dim w(M) As Double     'Debug.Print("ine")    For i = 0 To M - 1      x(i) = -pi + tim * i      'Debug.Write(x(i) & "  ")    Next    'Debug.Print("******")    For i = 0 To alpha.GetLength(0) - 1      For k = 0 To M - 1        w(k) = w(k) + alpha(i) * Math.Cos(i * x(k))        'Debug.Write(w(k) & "  ")      Next     Next     Return w   End Function   ''' <summary>  ''' 汉明窗  ''' </summary>  ''' <param name="M"> 窗长</param>  ''' <returns></returns>  Public Function General_hamming(M As Integer) As Double()    Dim db As Double() = {0.5, 1 - 0.5}    Return General_cosine(M, db, False)  '进行加1 ，若sys为false  End Function   Public Function Get_window(M As Integer) As Double()    Return General_hamming(M)   End Function   End Module

 Imports System.IOImports System.NumericsImports TensorFlow 'Install-Package TensorFlowSharp Public Class KeyWordDetect   Dim graph As TFGraph  Dim session As TFSession   '加载模型  Public Sub New()    Dim model As Byte() = File.ReadAllBytes("f:\graph1.pb")    '导入GraphDef     graph = New TFGraph()    graph.Import(model, "")     session = New TFSession(graph)     ' Threading.ThreadPool.SetMaxThreads(5, 5)  End Sub   Protected Overrides Sub finalize()     session.CloseSession()    End Sub   '将声音数据变为mfcc byte数据  Public Function DataBToMFCC(dataB() As Byte) As Double(,)    Dim buff16(dataB.Length / 2 - 1) As Int16    Buffer.BlockCopy(dataB, 0, buff16, 0, dataB.Length - 1)     Dim result(,) As Double = MFCC(buff16)    Return result  End Function    '将声音数据变为mfcc  Public Function DataToMFCC(dataI() As Int16) As Double(,)     Dim result(,) As Double = MFCC(dataI)    Return result  End Function    '将mfcc变为输入数据格式  Public Function MFCCToVect(mfcc As Double(,)) As Double(,,)    Dim data(0, 1, 129) As Double     Dim n As Integer = 0, m As Integer = 0    For i = 0 To mfcc.GetLength(0) - 1      For k = 0 To mfcc.GetLength(1) - 1        data(0, m, n) = mfcc(i, k)        n = n + 1      Next      If n = 130 Then         m = 1        n = 0      End If    Next    Return data  End Function   Dim output  Dim runner As TFSession.Runner  Dim result  Dim rshape   '关键字检测  Public Function Detected(Data(,,) As Double) As Double     ' Dim tensor As TFTensor = New TFTensor(Data)    runner = session.GetRunner()     runner.AddInput(graph("input")(0), Data).Fetch(graph("out")(0))     output = runner.Run()      result = output(0)    rshape = result.Shape    Dim rt As Double    rt = result.GetValue(True)(0)(0)    'For k = 0 To rshape.GetValue(0) - 1    '  rt = result.GetValue(True)(k)(0)    '  'Debug.Print(rt)    '  If (rt > 0.8) Then    '    Debug.Print("-----------recogxili")    '    ' MsgBox("recgo")    '  End If    'Next     Return RT   End Function     'Public Function RunB(dataB() As Byte)  '  Dim mfccd As Double(,) = DataBToMFCC(dataB)  '  Dim inputx As Double(,,) = MFCCToVect(mfccd)  '  Detected(inputx)  'End Function    'Public Function ThreadPoolRun(dataI() As Int16)   '  Threading.ThreadPool.QueueUserWorkItem(Run(dataI), dataI)  '  '  Dim thrd1 As New Threading.Thread(New Threading.ParameterizedThreadStart(AddressOf Run))  '  ' thrd1.Start(dataI)  'End Function  'Delegate Function DelgRun(dataI() As Int16)  'Public Function ThreadRun(dataI() As Int16)  '  ' Dim drun As New DelgRun(AddressOf Run)   '  Dim thrd1 As New Threading.Thread(New Threading.ParameterizedThreadStart(AddressOf Run))  '  thrd1.Start(dataI)   'End Function    Public Function Run(dataI() As Int16) As Double    ' Debug.Print("thread *****1")    Dim mfccd As Double(,) = DataToMFCC(dataI)    Dim inputx As Double(,,) = MFCCToVect(mfccd)    Return Detected(inputx)  End Function   Public Function MFCC(buff16() As Int16) As Double(,)    Dim datalen As Integer = buff16.Length * 2    Dim double_buff(datalen / 2 - 1) As Double    Dim len As Integer = datalen / 2    Array.Copy(buff16, double_buff, len)     '******************    For i = 0 To double_buff.Length - 1      double_buff(i) = double_buff(i) / 32768      ' Debug.Print(double_buff(i))    Next      '汉明窗create    Dim hann_window As Double() = Get_window(2048)    'Debug.Print("--------------------------")    'Debug.Print("hann_window**************")    For Each i In hann_window      'Debug.Print(i & "  ")    Next     'Debug.Print("--------------------------")    'Debug.Print("*************pad reflect**************")    Dim y As Double() = PadReflect(double_buff, 1024)    ' Dim y As Double() = double_buff    'For Each i In y    '  'Debug.Print(i & "  ")    'Next     'Debug.Print("--------------------------")    'Debug.Print("***************frame************")    Dim frams As Double(,) = Frame2(y)     Dim tim As Integer = frams.GetLength(0)     'Debug.Print("--------------------------")    'Debug.Print("**********hann * data**************")    Dim hannData(tim - 1, 2047) As Double       For n = 0 To tim - 1      For i = 0 To 2048 - 1        hannData(n, i) = frams(n, i) * hann_window(i)        ' Debug.Print(hannData(i) & "  ")      Next     Next n      '\\\\\\\\\\\\\\\\melspecture     Dim specturm1(,) As Complex = FFT(hannData)      'For i = 0 To specturm1.GetLength(0) - 1    '  Debug.Print("--------------------------------------")    '  Debug.Print("--------------------------------------")    '  For k = 0 To specturm1.GetLength(1) - 1    '    Debug.Print(specturm1（i, k).Real & "  " & specturm1(i, k).Imaginary)    '  Next    'Next     Dim s As Double(,) = spectrum(specturm1)     Dim fftfreqs() As Double = fft_frequencies(16000, 2048)    'Debug.Print("***************fftfreqs*****************")    'Debug.Print("***************fftfreqs*****************")    'Debug.Print("fftfreqs.shape", fftfreqs.Length)    'For i = 0 To fftfreqs.Length - 1    '  'Debug.Write(fftfreqs(i) & "  ")    'Next     ''''''''''''''''mel * specturm1    'Debug.Print("**************")    'Debug.Print("****滤波器创建**********")    Dim mel_f As Double() = MelFilter()      'Debug.Print("--------------------------")    'Debug.Print("hann_window**************")    'Debug.Print("diff")    Dim fdiff As Double() = diff(mel_f)     Dim ramps_ As Double(,) = ramps(mel_f, fftfreqs)     Dim weights(,) As Double = weight(ramps_, fdiff)     normal(mel_f, weights)     'S*WEIGHT = melspectrogram    'weight 128,1025    's 5 ,1025    Dim melspectrogram_(,) As Double = melspectrogram(weights, s)    Dim power_to_db_ As Double（,) = power_to_db(melspectrogram_)     Dim dct_ As Double(,) = Dct(20, 128)     Return _mfcc(dct_, power_to_db_)  End Function End Class

以上这篇对python中Librosa的mfcc步骤详解就是小编分享给大家的全部内容了，希望能给大家一个参考，也希望大家多多支持。