├── README.md
├── Metal
    ├── _MetalSpectrogram.hpp
    ├── MetalPerformer.hpp
    ├── MetalPerformer.mm
    ├── shader.metal
    ├── _MetalSpectrogram.mm
    └── MetalSpectrogram.mm
├── Base.h
├── LICENSE
├── MetalSpectrogram.hpp
├── vDSP
    ├── vDSPSpectrogram.h
    └── vDSPSpectrogram.c
├── Tensor.hpp
├── NSObjectCPPProxy.hpp
├── .gitignore
├── BaseSpectrogram.hpp
└── vDSPSpectrogram.hpp


/README.md:
--------------------------------------------------------------------------------
1 | # AppleSpectrogram
2 | Spectrogram based on Apple's Accelerate and Metal frameworks.
3 | 
4 | BNNS like C filter in this repo: https://github.com/techpro-studio/NNToollkitCore
5 | 
6 | If you need MPS like kernel it is here https://github.com/techpro-studio/MetalAudioShaders
7 | 


--------------------------------------------------------------------------------
/Metal/_MetalSpectrogram.hpp:
--------------------------------------------------------------------------------
 1 | //
 2 | //  BaseMetalSpectrogram.h
 3 | //  audio_test
 4 | //
 5 | //  Created by Alex on 25.04.2020.
 6 | //  Copyright © 2020 Alex. All rights reserved.
 7 | //
 8 | 
 9 | #import <Foundation/Foundation.h>
10 | #import "Base.h"
11 | #import "MetalPerformer.hpp"
12 | 
13 | NS_ASSUME_NONNULL_BEGIN
14 | 
15 | @interface _MetalSpectrogram : MetalPerformer
16 | 
17 | - (instancetype) initWithDevice: (id<MTLDevice>) device inputType: (SpectrogramInputType) inputType andSCP:(SpectrogramComputationParameters) parameters;
18 | - (void) compute: (void*) input output: (float *)output;
19 | 
20 | @end
21 | 
22 | NS_ASSUME_NONNULL_END
23 | 


--------------------------------------------------------------------------------
/Base.h:
--------------------------------------------------------------------------------
 1 | 
 2 | //
 3 | //  Common..h
 4 | //  audio_test
 5 | //
 6 | //  Created by Alex on 26.04.2020.
 7 | //  Copyright © 2020 Alex. All rights reserved.
 8 | //
 9 | 
10 | #ifndef Common__h
11 | #define Common__h
12 | 
13 | #if defined __cplusplus
14 | extern "C" {
15 | #endif
16 | 
17 | typedef struct {
18 |     const unsigned int nfft;
19 |     const unsigned int step;
20 |     const unsigned int inputSize;
21 |     const unsigned int outputRow;
22 |     const unsigned int outputColumn;
23 | } SpectrogramComputationParameters;
24 | #if defined __cplusplus
25 | }
26 | #endif
27 | 
28 | enum SpectrogramInputType { real, complex };
29 | 
30 | #endif
31 | 
32 | 


--------------------------------------------------------------------------------
/Metal/MetalPerformer.hpp:
--------------------------------------------------------------------------------
 1 | //
 2 | //  MetalPerformer.h
 3 | //  audio_test
 4 | //
 5 | //  Created by Alex on 27.04.2020.
 6 | //  Copyright © 2020 Alex. All rights reserved.
 7 | //
 8 | 
 9 | #import <Foundation/Foundation.h>
10 | #import <Metal/Metal.h>
11 | 
12 | NS_ASSUME_NONNULL_BEGIN
13 | 
14 | typedef void(^CommandEncoder)(id<MTLComputeCommandEncoder>);
15 | 
16 | @interface MetalPerformer : NSObject
17 | {
18 |     @protected id<MTLDevice> device;
19 |     @protected id<MTLComputePipelineState> function;
20 |     @protected id<MTLCommandQueue> commandQueue;
21 |     @protected MTLSize countOfThreadGroups;
22 |     @protected MTLSize threadPerGroup;
23 | }
24 | 
25 | - (NSError *)setupMetalFunction: (NSString*) functionName;
26 | - (void) computeWithEncoding: (CommandEncoder) encoder;
27 | - (void) calculateThreadParameters: (unsigned int) length;
28 | 
29 | @end
30 | 
31 | NS_ASSUME_NONNULL_END
32 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2020 techprostudio
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/MetalSpectrogram.hpp:
--------------------------------------------------------------------------------
 1 | //
 2 | //  gpu_spectrogram.h
 3 | //  audio_test
 4 | //
 5 | //  Created by Alex on 25.04.2020.
 6 | //  Copyright © 2020 Alex. All rights reserved.
 7 | //
 8 | 
 9 | #ifndef gpu_spectrogram_h
10 | #define gpu_spectrogram_h
11 | 
12 | #include "BaseSpectrogram.hpp"
13 | #include "_MetalSpectrogram.hpp"
14 | #include "NSObjectCPPProxy.hpp"
15 | #include <complex>
16 | 
17 | 
18 | template <typename Input, unsigned int size, unsigned int nfft, unsigned int noverlap>
19 | class MetalSpectrogram: public BaseSpectrogram<Input, size, nfft, noverlap>, protected NSObjectCPPProxy{
20 |     typedef BaseSpectrogram<Input, size, nfft, noverlap> Base;
21 |     using typename Base::Output;
22 | public:
23 |     MetalSpectrogram(const void* gpu): BaseSpectrogram<Input, size, nfft, noverlap>(), NSObjectCPPProxy((__bridge void *)[[_MetalSpectrogram alloc] initWithDevice:(__bridge id<MTLDevice>)gpu inputType: Base::inputType andSCP: Base::scp]){}
24 | 
25 |     Output calculate(const Tensor<Input, size>& input) override{
26 |         Output output;
27 |         auto spectrogram = (__bridge _MetalSpectrogram*)ptr;
28 |         [spectrogram compute: input.value output:output.value];
29 |         return output;
30 |     }
31 | };
32 | 
33 | #endif /* gpu_spectrogram_h */
34 | 


--------------------------------------------------------------------------------
/vDSP/vDSPSpectrogram.h:
--------------------------------------------------------------------------------
 1 | //
 2 | //  vDSPSpectrogram.h
 3 | //  audio_test
 4 | //
 5 | //  Created by Alex on 25.04.2020.
 6 | //  Copyright © 2020 Alex. All rights reserved.
 7 | //
 8 | 
 9 | #ifndef vDSPSpectrogram_h
10 | #define vDSPSpectrogram_h
11 | 
12 | #include <Accelerate/Accelerate.h>
13 | #include "Base.h"
14 | 
15 | #if defined __cplusplus
16 | extern "C" {
17 | #endif
18 | 
19 | // this should be calculated one time somewhere; its const parameters;
20 | typedef struct {
21 |     SpectrogramComputationParameters parameters;
22 |     vDSP_DFT_Setup setup;
23 | } vDSP_Spectrogram_Setup;
24 | 
25 | 
26 | vDSP_Spectrogram_Setup vDSP_Spectrogram_Setup_Init(SpectrogramComputationParameters parameters);
27 | 
28 | void vDSP_Spectrogram_Setup_Destroy(vDSP_Spectrogram_Setup setup);
29 | 
30 | void CalculateMagnitude(DSPSplitComplex *split, float * columnPtr, const int vectorSize);
31 | 
32 | void vDSP_Spectrogram_Real(vDSP_Spectrogram_Setup setup, float* input, float* output);
33 | 
34 | void vDSP_Spectrogram_Real_P(vDSP_Spectrogram_Setup setup, float* input, float* output);
35 | 
36 | void vDSP_Spectrogram_Complex(vDSP_Spectrogram_Setup setup, DSPComplex* input, float* output);
37 | 
38 | void vDSP_Spectrogram_Complex_P(vDSP_Spectrogram_Setup setup, DSPComplex* input, float* output);
39 | 
40 | 
41 | #if defined __cplusplus
42 | }
43 | #endif
44 | 
45 | #endif /* vDSPSpectrogram_h */
46 | 


--------------------------------------------------------------------------------
/Tensor.hpp:
--------------------------------------------------------------------------------
 1 | //
 2 | //  Tensor.h
 3 | //  audio_test
 4 | //
 5 | //  Created by Alex on 19.04.2020.
 6 | //  Copyright © 2020 Alex. All rights reserved.
 7 | //
 8 | #include <algorithm>
 9 | 
10 | #ifndef TENSOR_H
11 | #define TENSOR_H
12 | 
13 | 
14 | 
15 | template <typename T, unsigned int ...side>
16 | class Tensor {
17 | 
18 | public:
19 | 
20 |     T *value;
21 | 
22 |     static constexpr unsigned int size = []{
23 |         int result = 1;
24 |         for (auto && a:  {side...}){
25 |             result *= a;
26 |         }
27 |         return result;
28 |     }();
29 | 
30 |     Tensor(): value(new T[size]) {}
31 | 
32 |     explicit Tensor(T init): value(new T[size]){
33 |         std::fill(value, value + size, init);
34 |     }
35 | 
36 |     Tensor(Tensor && other): value(other.value){
37 |         other.value = nullptr;
38 |     }
39 | 
40 |     Tensor(const Tensor& other): value(new T[size]){
41 |         memcpy(value, other.value, sizeof(T) * size);
42 |     }
43 | 
44 |     Tensor& operator = (Tensor& other) {
45 |         memcpy(value, other.value, sizeof(T) * size);
46 |         return *this;
47 |     }
48 | 
49 |     Tensor& operator = (Tensor && other) {
50 |         value = other.value;
51 |         other.value = nullptr;
52 |         return *this;
53 |     }
54 | 
55 |     ~Tensor(){
56 |         delete[] value;
57 |     }
58 | };
59 | 
60 | 
61 | 
62 | 
63 | #endif //TENSOR_H
64 | 


--------------------------------------------------------------------------------
/NSObjectCPPProxy.hpp:
--------------------------------------------------------------------------------
 1 | //
 2 | //  RealGPUSpectrogramBridge.hpp
 3 | //  audio_test
 4 | //
 5 | //  Created by Alex on 25.04.2020.
 6 | //  Copyright © 2020 Alex. All rights reserved.
 7 | //
 8 | 
 9 | #ifndef object_hpp
10 | #define object_hpp
11 | 
12 | #include <stdio.h>
13 | #include <CoreFoundation/CoreFoundation.h>
14 | 
15 | class NSObjectCPPProxy
16 | {
17 | public:
18 |     const void* GetPtr() const { return ptr; }
19 |     inline operator bool() const { return ptr != nullptr; }
20 | 
21 | protected:
22 |     NSObjectCPPProxy(): ptr(nullptr){}
23 | 
24 |     NSObjectCPPProxy(const void* ptr): ptr(ptr) {
25 |         if (ptr)
26 |             CFRetain(ptr);
27 |     }
28 | 
29 |     NSObjectCPPProxy(const NSObjectCPPProxy& rhs): ptr(rhs.ptr) {
30 |         if (ptr)
31 |             CFRetain(ptr);
32 |     }
33 |     
34 |     NSObjectCPPProxy(NSObjectCPPProxy&& rhs):  ptr(rhs.ptr){
35 |         rhs.ptr = nullptr;
36 |     }
37 | 
38 |     virtual ~NSObjectCPPProxy() {
39 |         if (ptr)
40 |             CFRelease(ptr);
41 |     }
42 | 
43 |     NSObjectCPPProxy& operator=(const NSObjectCPPProxy& rhs) {
44 |         if (rhs.ptr == ptr)
45 |             return *this;
46 |         if (rhs.ptr)
47 |             CFRetain(rhs.ptr);
48 |         if (ptr)
49 |             CFRelease(ptr);
50 |         ptr = rhs.ptr;
51 |         return *this;
52 |     }
53 | 
54 |     NSObjectCPPProxy& operator=(NSObjectCPPProxy&& rhs) {
55 |         if (rhs.ptr == ptr)
56 |             return *this;
57 |         if (ptr)
58 |             CFRelease(ptr);
59 |         ptr = rhs.ptr;
60 |         rhs.ptr = nullptr;
61 |         return *this;
62 |     }
63 | 
64 | 
65 |     const void* ptr = nullptr;
66 | };
67 | 
68 | #endif
69 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # Xcode
 2 | #
 3 | # gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore
 4 | 
 5 | ## User settings
 6 | xcuserdata/
 7 | 
 8 | ## compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9)
 9 | *.xcscmblueprint
10 | *.xccheckout
11 | 
12 | ## compatibility with Xcode 3 and earlier (ignoring not required starting Xcode 4)
13 | build/
14 | DerivedData/
15 | *.moved-aside
16 | *.pbxuser
17 | !default.pbxuser
18 | *.mode1v3
19 | !default.mode1v3
20 | *.mode2v3
21 | !default.mode2v3
22 | *.perspectivev3
23 | !default.perspectivev3
24 | 
25 | ## Obj-C/Swift specific
26 | *.hmap
27 | 
28 | ## App packaging
29 | *.ipa
30 | *.dSYM.zip
31 | *.dSYM
32 | 
33 | # CocoaPods
34 | #
35 | # We recommend against adding the Pods directory to your .gitignore. However
36 | # you should judge for yourself, the pros and cons are mentioned at:
37 | # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
38 | #
39 | # Pods/
40 | #
41 | # Add this line if you want to avoid checking in source code from the Xcode workspace
42 | # *.xcworkspace
43 | 
44 | # Carthage
45 | #
46 | # Add this line if you want to avoid checking in source code from Carthage dependencies.
47 | # Carthage/Checkouts
48 | 
49 | Carthage/Build/
50 | 
51 | # fastlane
52 | #
53 | # It is recommended to not store the screenshots in the git repo.
54 | # Instead, use fastlane to re-generate the screenshots whenever they are needed.
55 | # For more information about the recommended setup visit:
56 | # https://docs.fastlane.tools/best-practices/source-control/#source-control
57 | 
58 | fastlane/report.xml
59 | fastlane/Preview.html
60 | fastlane/screenshots/**/*.png
61 | fastlane/test_output
62 | 
63 | # Code Injection
64 | #
65 | # After new code Injection tools there's a generated folder /iOSInjectionProject
66 | # https://github.com/johnno1962/injectionforxcode
67 | 
68 | iOSInjectionProject/
69 | 


--------------------------------------------------------------------------------
/BaseSpectrogram.hpp:
--------------------------------------------------------------------------------
 1 | //
 2 | //  spectro_impl.h
 3 | //  audio_test
 4 | //
 5 | //  Created by Alex on 25.04.2020.
 6 | //  Copyright © 2020 Alex. All rights reserved.
 7 | //
 8 | 
 9 | #ifndef spectro_impl_h
10 | #define spectro_impl_h
11 | 
12 | #include <Accelerate/Accelerate.h>
13 | #include "Tensor.hpp"
14 | #include <complex>
15 | #include <memory>
16 | #include "Base.h"
17 | #include <type_traits>
18 | 
19 | 
20 | 
21 | template<typename Input, uint size, uint nfft, uint noverlap>
22 | class BaseSpectrogram {
23 | public:
24 | 
25 |     static constexpr bool isFloat = std::is_same<Input, float>::value;
26 |     static_assert(isFloat || std::is_same<Input, std::complex<float>>::value, "Input should be float or complex");
27 |     
28 |     static constexpr SpectrogramInputType inputType = []{
29 |         if constexpr(isFloat){
30 |             return SpectrogramInputType::real;
31 |         }
32 |         return SpectrogramInputType::complex;
33 |     }();
34 | 
35 |     static constexpr unsigned int column = inputType == real ?  nfft / 2 + 1 : nfft;
36 |     static constexpr auto step = nfft - noverlap;
37 | 
38 |     static_assert(nfft != noverlap, "nfft should not be equal to noverlap");
39 |     static_assert(nfft > 0, "nfft should not be more than 0");
40 |     static_assert(nfft % step == 0, "nfft should divide on delta");
41 |     static_assert(size % step == 0, "size should divide on delta");
42 |     static_assert(size >= nfft, "size should be greater or equal than nftt");
43 | 
44 |     static constexpr unsigned int row = (size - noverlap) /  step;
45 |     static constexpr SpectrogramComputationParameters scp = { .nfft = nfft, .step = step , .outputRow = row, .outputColumn = column, .inputSize = size };
46 |     
47 |     static constexpr unsigned int outputSize = row * column;
48 | 
49 |     typedef Tensor<float, outputSize> Output;
50 | 
51 |     virtual Output calculate(const Tensor<Input, size>& input) = 0;
52 | 
53 | };
54 | 
55 | 
56 | 
57 | 
58 | 
59 | 
60 | #endif /* spectro_impl_h */
61 | 


--------------------------------------------------------------------------------
/Metal/MetalPerformer.mm:
--------------------------------------------------------------------------------
 1 | //
 2 | //  MetalPerformer.m
 3 | //  audio_test
 4 | //
 5 | //  Created by Alex on 27.04.2020.
 6 | //  Copyright © 2020 Alex. All rights reserved.
 7 | //
 8 | 
 9 | #import "MetalPerformer.hpp"
10 | 
11 | @implementation MetalPerformer
12 | 
13 | - (NSError *) createError: (NSString *) description code: (NSInteger) code
14 | {
15 |     return [NSError errorWithDomain:@"com.metal.performer" code:code userInfo:@{NSLocalizedDescriptionKey: description}];
16 | }
17 | 
18 | - (NSError *)setupMetalFunction: (NSString*) functionName {
19 | 
20 |     auto defaultLibrary = [device newDefaultLibrary];
21 |     if (defaultLibrary == nil)
22 |     {
23 |         return [self createError:@"Failed to find the default library." code: 0];
24 |     }
25 |     auto spectrogram = [defaultLibrary newFunctionWithName:functionName];
26 |     if (spectrogram == nil)
27 |     {
28 |         return [self createError:@"Failed to find the function." code: 1];
29 |     }
30 |     NSError* error = nil;
31 |     function = [device newComputePipelineStateWithFunction: spectrogram error:&error];
32 |     if (function == nil)
33 |     {
34 |         return [self createError:[NSString stringWithFormat:@"Failed to created pipeline state object, error %@.", error.localizedDescription] code:2];
35 |     }
36 |     commandQueue = [device newCommandQueue];
37 |     if (commandQueue == nil)
38 |     {
39 |         return [self createError:@"Failed to find the command queue." code:3];
40 |     }
41 |     return nil;
42 | }
43 | 
44 | - (void) calculateThreadParameters: (unsigned int) length
45 | {
46 |     NSUInteger threadGroupSize = function.maxTotalThreadsPerThreadgroup;
47 |     NSUInteger countOfThreadGroups = length / threadGroupSize + (length % threadGroupSize == 0 ? 0 : 1);
48 |     self->countOfThreadGroups = MTLSizeMake(countOfThreadGroups, 1, 1);
49 |     self->threadPerGroup = MTLSizeMake(threadGroupSize, 1, 1);
50 | }
51 | 
52 | - (void) computeWithEncoding: (CommandEncoder) encoder
53 | {
54 |     auto commandBuffer = [commandQueue commandBuffer];
55 |     assert(commandBuffer != nil);
56 |     auto computeEncoder = [commandBuffer computeCommandEncoder];
57 |     assert(computeEncoder != nil);
58 |     encoder(computeEncoder);
59 |     [computeEncoder endEncoding];
60 |     [commandBuffer commit];
61 |     [commandBuffer waitUntilCompleted];
62 | }
63 | 
64 | @end
65 | 


--------------------------------------------------------------------------------
/vDSPSpectrogram.hpp:
--------------------------------------------------------------------------------
 1 | //
 2 | //  vdsp_spectrogram.h
 3 | //  audio_test
 4 | //
 5 | //  Created by Alex on 25.04.2020.
 6 | //  Copyright © 2020 Alex. All rights reserved.
 7 | //
 8 | 
 9 | #ifndef vdsp_spectrogram_h
10 | #define vdsp_spectrogram_h
11 | 
12 | #include "BaseSpectrogram.hpp"
13 | #include "vDSPSpectrogram.h"
14 | 
15 | template <typename Input, unsigned int size, unsigned int nfft, unsigned int noverlap, bool concurrent>
16 | class BasevDSPSpectrogram: public BaseSpectrogram<Input, size, nfft, noverlap>{
17 |     static_assert(powerof2(nfft) || powerof2(nfft / 3) || powerof2(nfft / 5) || powerof2(nfft / 15), "nfft should be power of 2 or 2 ^ n * 3 * 5 where n >= 3");
18 | protected:
19 |     vDSP_Spectrogram_Setup setup;
20 | public:
21 |     BasevDSPSpectrogram(): setup(vDSP_Spectrogram_Setup_Init(BaseSpectrogram<Input, size, nfft, noverlap>::scp)) {}
22 |     virtual ~BasevDSPSpectrogram() {
23 |         vDSP_Spectrogram_Setup_Destroy(setup);
24 |     }
25 | };
26 | 
27 | template <unsigned int size, unsigned int nfft, unsigned int noverlap, bool concurrent>
28 | class RealvDSPSpectrogram: public BasevDSPSpectrogram<float, size, nfft, noverlap, concurrent> {
29 |     static constexpr auto calc = concurrent ? vDSP_Spectrogram_Real_P : vDSP_Spectrogram_Real;
30 | public:
31 |     using typename BaseSpectrogram<float, size, nfft, noverlap>::Output;
32 | 
33 |     Output calculate(const Tensor<float, size>& input) override {
34 |         Output output;
35 |         calc(BasevDSPSpectrogram<float, size, nfft, noverlap, concurrent>::setup, input.value, output.value);
36 |         return output;
37 |     }
38 | };
39 | 
40 | 
41 | template <unsigned int size, unsigned int nfft, unsigned int noverlap, bool concurrent>
42 | class ComplexvDSPSpectrogram: public BasevDSPSpectrogram<std::complex<float>, size, nfft, noverlap, concurrent> {
43 |     static constexpr auto calc = concurrent ? vDSP_Spectrogram_Complex_P : vDSP_Spectrogram_Complex;
44 | public:
45 |     using typename BaseSpectrogram<std::complex<float>, size, nfft, noverlap>::Output;
46 | 
47 |     Output calculate(const Tensor<std::complex<float>, size>& input) override {
48 |         Output output;
49 |         calc(BasevDSPSpectrogram<std::complex<float>, size, nfft, noverlap, concurrent>::setup, (DSPComplex *)input.value, output.value);
50 |         return output;
51 |     }
52 | };
53 | 
54 | 
55 | #endif /* vdsp_spectrogram_h */
56 | 


--------------------------------------------------------------------------------
/Metal/shader.metal:
--------------------------------------------------------------------------------
 1 | //
 2 | //  spectrogram.metal
 3 | //  audio_test
 4 | //
 5 | //  Created by Alex on 20.04.2020.
 6 | //  Copyright © 2020 Alex. All rights reserved.
 7 | //
 8 | 
 9 | #include <metal_stdlib>
10 | using namespace metal;
11 | 
12 | struct config {
13 |     uint height;
14 |     uint nfft;
15 |     uint delta;
16 |     uint ouputLength;
17 | };
18 | 
19 | // this was moved to function, because it was an idea to put differeent implementations of Fourier transform;
20 | float2 real_dft_step(device const float *input, uint k, uint nfft){
21 |     float2 value {0.f, 0.f};
22 |     for (uint n = 0; n < nfft; ++n) {
23 |         float angle = 2 * M_PI_F * n * k / nfft;
24 |         value += { *(input + n) * cos(angle), -1 * *(input + n) * sin(angle) };
25 |     }
26 |     return value;
27 | }
28 | 
29 | float2 complex_dft_step(device const float2 *input, uint k, uint nfft){
30 |     float2 value {0.f, 0.f};
31 |     for (uint n = 0; n < nfft; ++n) {
32 |         float angle = 2 * M_PI_F * n * k / nfft;
33 |         float2 inputN = *(input + n);
34 |         value += { inputN[0] * cos(angle) + inputN[1] * sin(angle) , -1 * inputN[0] * sin(angle) + inputN[1] * cos(angle) };
35 |     }
36 |     return value;
37 | }
38 | 
39 | float calculate_magnitude(float2 complex) {
40 |     float magnitude = sqrt(complex[0] * complex[0] + complex[1] * complex[1]);
41 |     magnitude += 1.5849e-13;
42 |     float result = 10.0f * log10(magnitude);
43 |     return result;
44 | }
45 | 
46 | 
47 | kernel void spectrogram_real(device const float *input, device const config *config, device float *output, uint index[[thread_position_in_grid]])
48 | {
49 |     if (index >= config->ouputLength) { return; }
50 |     uint index_inside = index % config->height;
51 |     uint number_of_slice = (index - index_inside) / config->height;
52 |     device const float *input_begin = input + number_of_slice * config->delta;
53 |     float2 dft = real_dft_step(input_begin, index_inside, config->nfft);
54 |     output[index] = calculate_magnitude(dft);
55 | }
56 | 
57 | kernel void spectrogram_complex(device const float2 *input, device const config *config, device float *output, uint index[[thread_position_in_grid]])
58 | {
59 |     if (index >= config->ouputLength) { return; }
60 |     uint index_inside = index % config->height;
61 |     uint number_of_slice = (index - index_inside) / config->height;
62 |     device const float2 *input_begin = input + number_of_slice * config->delta;
63 |     float2 dft = complex_dft_step(input_begin, index_inside, config->nfft);
64 |     output[index] = calculate_magnitude(dft);
65 | }
66 | 
67 | 
68 | 
69 | 
70 | 


--------------------------------------------------------------------------------
/Metal/_MetalSpectrogram.mm:
--------------------------------------------------------------------------------
 1 | //
 2 | //  BaseMetalSpectrogram.m
 3 | //  audio_test
 4 | //
 5 | //  Created by Alex on 25.04.2020.
 6 | //  Copyright © 2020 Alex. All rights reserved.
 7 | //
 8 | 
 9 | #import "_MetalSpectrogram.hpp"
10 | 
11 | struct GPUConfig {
12 |     uint height;
13 |     uint nfft;
14 |     uint delta;
15 |     uint ouputLength;
16 | };
17 | 
18 | @implementation _MetalSpectrogram
19 | {
20 |     id<MTLBuffer> configBuffer;
21 |     id<MTLBuffer> inputBuffer;
22 |     id<MTLBuffer> outputBuffer;
23 | 
24 |     unsigned int outputLength;
25 |     unsigned int inputBufferSize;
26 |     unsigned int outputBufferSize;
27 | }
28 | 
29 | - (instancetype) initWithDevice: (id<MTLDevice>) device inputType: (SpectrogramInputType) inputType andSCP:(SpectrogramComputationParameters) parameters;
30 | {
31 |     self = [super init];
32 |     if (self)
33 |     {
34 |         self->device = device;
35 |         NSString* functionName = inputType == real ?  @"spectrogram_real" : @"spectrogram_complex";
36 |         unsigned int inputSize = inputType == real ? sizeof(float) : 2 * sizeof(float);
37 |         auto error = [super setupMetalFunction: functionName];
38 |         if (error != nil){
39 |             NSLog(@"%@", error.localizedDescription);
40 |             return nil;
41 |         }
42 |         outputLength = parameters.outputRow * parameters.outputColumn;
43 |         GPUConfig gpuConfig { .height = parameters.outputColumn , .nfft=parameters.nfft, .delta=parameters.step, .ouputLength=outputLength };
44 |         configBuffer = [device newBufferWithBytes:&gpuConfig length:sizeof(struct GPUConfig) options:MTLResourceStorageModeShared];
45 |         inputBufferSize = parameters.inputSize * inputSize;
46 |         outputBufferSize = outputLength * sizeof(float);
47 |         inputBuffer = [device newBufferWithLength:inputBufferSize options:MTLResourceStorageModeShared];
48 |         outputBuffer = [device newBufferWithLength:outputBufferSize options:MTLResourceStorageModeShared];
49 |         [super calculateThreadParameters:outputLength];
50 |     }
51 | 
52 |     return self;
53 | }
54 | 
55 | 
56 | - (void) compute: (void*) input output: (float *)output
57 | {
58 |     memcpy(inputBuffer.contents, input, inputBufferSize);;
59 |     __weak auto weakSelf = self;
60 |     [super computeWithEncoding:^(id<MTLComputeCommandEncoder> encoder) {
61 |         __strong auto strong = weakSelf;
62 |         [encoder setComputePipelineState: strong->function];
63 |         [encoder setBuffer: strong->inputBuffer offset:0 atIndex:0];
64 |         [encoder setBuffer: strong->configBuffer offset:0 atIndex:1];
65 |         [encoder setBuffer: strong->outputBuffer offset:0 atIndex:2];
66 |         [encoder dispatchThreadgroups: strong->countOfThreadGroups threadsPerThreadgroup:strong->threadPerGroup];
67 |     }];
68 |     memcpy(output, outputBuffer.contents, outputBufferSize);
69 | }
70 | 
71 | @end
72 | 


--------------------------------------------------------------------------------
/vDSP/vDSPSpectrogram.c:
--------------------------------------------------------------------------------
 1 | //
 2 | //  vDSPSpectrogram.c
 3 | //  audio_test
 4 | //
 5 | //  Created by Alex on 25.04.2020.
 6 | //  Copyright © 2020 Alex. All rights reserved.
 7 | //
 8 | 
 9 | #include "vDSPSpectrogram.h"
10 | 
11 | void CalculateMagnitude(DSPSplitComplex *split, float * columnPtr, const int vectorSize)
12 | {
13 |     vDSP_zvmags(split, 1, columnPtr, 1, vectorSize);
14 |     vvsqrtf(columnPtr, columnPtr, &vectorSize);
15 |     const Float32 kAdjust0DB = 1.5849e-13;
16 |     vDSP_vsadd(columnPtr, 1, &kAdjust0DB, columnPtr, 1, vectorSize);
17 |     Float32 one = 1;
18 |     vDSP_vdbcon(columnPtr, 1, &one, columnPtr, 1, vectorSize, 0);
19 | }
20 | 
21 | vDSP_Spectrogram_Setup vDSP_Spectrogram_Setup_Init(SpectrogramComputationParameters parameters) {
22 |     vDSP_DFT_Setup dftSetup = vDSP_DFT_zop_CreateSetup(NULL, parameters.nfft, vDSP_DFT_FORWARD);
23 |     return (vDSP_Spectrogram_Setup){ .parameters = parameters, .setup = dftSetup };
24 | }
25 | 
26 | void vDSP_Spectrogram_Setup_Destroy(vDSP_Spectrogram_Setup setup) {
27 |     vDSP_DFT_DestroySetup(setup.setup);
28 | }
29 | 
30 | void vDSP_Spectrogram_Real(vDSP_Spectrogram_Setup setup, float* input, float* output) {
31 |     float outputMemory[setup.parameters.nfft * 2];
32 |     float inputImag [setup.parameters.nfft];
33 |     memset(inputImag, 0.0f, setup.parameters.nfft*sizeof(float));
34 |     DSPSplitComplex outputSplit = {outputMemory, outputMemory + setup.parameters.nfft};
35 |     for (int r = 0; r < setup.parameters.outputRow; ++r) {
36 |         vDSP_DFT_Execute(setup.setup,
37 |                          input + r * setup.parameters.step, inputImag,
38 |                          outputSplit.realp, outputSplit.imagp);
39 |         CalculateMagnitude(&outputSplit, output + r * setup.parameters.outputColumn, setup.parameters.outputColumn);
40 |     }
41 | }
42 | 
43 | void vDSP_Spectrogram_Real_P(vDSP_Spectrogram_Setup setup, float* input, float* output){
44 |     __block float* inputImag = calloc(setup.parameters.nfft, sizeof(float));
45 |     memset(inputImag, 0.0f, setup.parameters.nfft*sizeof(float));
46 |     dispatch_apply(setup.parameters.outputRow, DISPATCH_APPLY_AUTO, ^(size_t r) {
47 |         float outputMemory[setup.parameters.nfft * 2];
48 |         DSPSplitComplex outputSplit = {outputMemory, outputMemory + setup.parameters.nfft};
49 |         vDSP_DFT_Execute(setup.setup,
50 |                          input + r * setup.parameters.step, inputImag,
51 |                          outputSplit.realp, outputSplit.imagp);
52 |         CalculateMagnitude(&outputSplit, output + r * setup.parameters.outputColumn, setup.parameters.outputColumn);;
53 |     });
54 |     free(inputImag);
55 | }
56 | 
57 | void vDSP_Spectrogram_Complex(vDSP_Spectrogram_Setup setup, DSPComplex* input, float* output) {
58 |     float outputMemory[setup.parameters.nfft * 2];
59 |     float inputMemory[setup.parameters.nfft * 2];
60 |     DSPSplitComplex inputSplit = {inputMemory, inputMemory + setup.parameters.nfft};
61 |     DSPSplitComplex outputSplit = {outputMemory, outputMemory + setup.parameters.nfft};
62 |     for (int r = 0; r < setup.parameters.outputRow; ++r) {
63 |         vDSP_ctoz(input + r * setup.parameters.step, 2, &inputSplit, 1, setup.parameters.nfft);
64 |         vDSP_DFT_Execute(setup.setup,
65 |                          inputSplit.realp, inputSplit.imagp,
66 |                          outputSplit.realp, outputSplit.imagp);
67 |         CalculateMagnitude(&outputSplit, output + r * setup.parameters.outputColumn, setup.parameters.outputColumn);
68 |     }
69 | }
70 | 
71 | void vDSP_Spectrogram_Complex_P(vDSP_Spectrogram_Setup setup, DSPComplex* input, float* output) {
72 |     dispatch_apply(setup.parameters.outputRow, DISPATCH_APPLY_AUTO, ^(size_t r) {
73 |         float outputMemory[setup.parameters.nfft * 2];
74 |         float inputMemory[setup.parameters.nfft * 2];
75 |         DSPSplitComplex inputSplit = {inputMemory, inputMemory + setup.parameters.nfft};
76 |         DSPSplitComplex outputSplit = {outputMemory, outputMemory + setup.parameters.nfft};
77 |         vDSP_ctoz(input + r * setup.parameters.step, 2, &inputSplit, 1, setup.parameters.nfft);
78 |         vDSP_DFT_Execute(setup.setup,
79 |                          inputSplit.realp, inputSplit.imagp,
80 |                          outputSplit.realp, outputSplit.imagp);
81 |         CalculateMagnitude(&outputSplit, output + r * setup.parameters.outputColumn, setup.parameters.outputColumn);
82 |     });
83 | }
84 | 


--------------------------------------------------------------------------------
/Metal/MetalSpectrogram.mm:
--------------------------------------------------------------------------------
  1 | //
  2 | //  BaseMetalSpectrogram.m
  3 | //  audio_test
  4 | //
  5 | //  Created by Alex on 25.04.2020.
  6 | //  Copyright © 2020 Alex. All rights reserved.
  7 | //
  8 | 
  9 | #import "_MetalSpectrogram.h"
 10 | 
 11 | struct GPUConfig {
 12 |     uint height;
 13 |     uint nfft;
 14 |     uint delta;
 15 |     uint ouputLength;
 16 | };
 17 | 
 18 | 
 19 | @implementation _MetalSpectrogram
 20 | {
 21 |     id<MTLDevice> device;
 22 |     id<MTLComputePipelineState> spectrogramFunctionPipState;
 23 |     id<MTLCommandQueue> commanQueue;
 24 |     id<MTLBuffer> configBuffer;
 25 |     unsigned int outputLength;
 26 |     MTLSize countOfThreadGroups;
 27 |     MTLSize threadPerGroup;
 28 | }
 29 | 
 30 | - (instancetype) initWithDevice: (id<MTLDevice>) device inputType: (MetalSpectrogramInputType) inputType andConfig: (SpectrogramConfig) config;
 31 | {
 32 |     self = [super init];
 33 |     if (self)
 34 |     {
 35 |         self->device = device;
 36 |         NSError* error = nil;
 37 |         auto defaultLibrary = [device newDefaultLibrary];
 38 |         if (defaultLibrary == nil)
 39 |          {
 40 |              NSLog(@"Failed to find the default library.");
 41 |              return nil;
 42 |          }
 43 |         NSString* functionName = inputType == real ?  @"spectrogram_real" : @"spectrogram_complex";
 44 |         unsigned int inputSize = inputType == real ? sizeof(float) : 2 * sizeof(float);
 45 | 
 46 |         auto spectrogram = [defaultLibrary newFunctionWithName:functionName];
 47 |         if (spectrogram == nil)
 48 |         {
 49 |             NSLog(@"Failed to find the adder function.");
 50 |             return nil;
 51 |         }
 52 |         spectrogramFunctionPipState = [device newComputePipelineStateWithFunction: spectrogram error:&error];
 53 |         if (spectrogramFunctionPipState == nil)
 54 |         {
 55 |             NSLog(@"Failed to created pipeline state object, error %@.", error);
 56 |             return nil;
 57 |         }
 58 |         commanQueue = [device newCommandQueue];
 59 |         if (commanQueue == nil)
 60 |         {
 61 |             NSLog(@"Failed to find the command queue.");
 62 |             return nil;
 63 |         }
 64 |         if (config.nfft % 2 != 0) {
 65 |             NSLog(@"nfft should be divided by 2");
 66 |             return nil;
 67 |         }
 68 |         auto delta = config.nfft - config.noverlap;
 69 |         if (delta == 0) {
 70 |             NSLog(@"Failed to noverlap the command queue.");
 71 |             return nil;
 72 |         }
 73 |         auto height = config.nfft / 2 + 1;
 74 |         if ((config.size - config.noverlap) % delta != 0) {
 75 |             NSLog(@"Failed size or noverlap");
 76 |             return nil;
 77 |         }
 78 |         auto width = (config.size - config.noverlap) / delta;
 79 |         outputLength = width * height;
 80 |         GPUConfig gpuConfig { .height = height , .nfft=config.nfft, .delta=delta, .ouputLength=outputLength };
 81 |         configBuffer = [device newBufferWithBytes:&gpuConfig length:sizeof(struct GPUConfig) options:MTLResourceStorageModeShared];
 82 |         _inputBufferSize = config.size * inputSize;
 83 |         _outputBufferSize = outputLength * sizeof(float);
 84 |         _inputBuffer = [device newBufferWithLength:_inputBufferSize options:MTLResourceStorageModeShared];
 85 |         _outputBuffer = [device newBufferWithLength:_outputBufferSize options:MTLResourceStorageModeShared];
 86 |         NSUInteger threadGroupSize = spectrogramFunctionPipState.maxTotalThreadsPerThreadgroup;
 87 |         NSUInteger countOfThreadGroups = outputLength / threadGroupSize + (outputLength % threadGroupSize == 0 ? 0 : 1);
 88 |         self->countOfThreadGroups = MTLSizeMake(countOfThreadGroups, 1, 1);
 89 |         self->threadPerGroup = MTLSizeMake(threadGroupSize, 1, 1);
 90 |     }
 91 | 
 92 |     return self;
 93 | }
 94 | 
 95 | - (void) compute
 96 | {
 97 |     auto commandBuffer = [commanQueue commandBuffer];
 98 |     assert(commandBuffer != nil);
 99 |     auto computeEncoder = [commandBuffer computeCommandEncoder];
100 |     assert(computeEncoder != nil);
101 |     [self encodeSpectrogramCommand:computeEncoder];
102 |     [computeEncoder endEncoding];
103 |     [commandBuffer commit];
104 |     [commandBuffer waitUntilCompleted];
105 | }
106 | 
107 | - (void)encodeSpectrogramCommand:(id<MTLComputeCommandEncoder>)computeEncoder {
108 |     [computeEncoder setComputePipelineState:spectrogramFunctionPipState];
109 |     [computeEncoder setBuffer:_inputBuffer offset:0 atIndex:0];
110 |     [computeEncoder setBuffer:configBuffer offset:0 atIndex:1];
111 |     [computeEncoder setBuffer:_outputBuffer offset:0 atIndex:2];
112 |     [computeEncoder dispatchThreadgroups:countOfThreadGroups threadsPerThreadgroup:threadPerGroup];
113 | }
114 | 
115 | @end
116 | 


--------------------------------------------------------------------------------