├── .bazelrc
├── .bazelversion
├── .clang-format
├── .github
├── actions
│ ├── lyra-builder
│ │ └── action.yml
│ └── setup-lyra-deps
│ │ └── action.yml
└── workflows
│ └── ci.yml
├── .gitignore
├── BUILD
├── CONTRIBUTING.md
├── Doxyfile
├── LICENSE
├── README.md
├── WORKSPACE
├── android_configure.bzl
├── external
├── eigen.BUILD
└── fft2d.BUILD
├── lyra
├── BUILD
├── android_example
│ ├── AndroidManifest.xml
│ ├── BUILD
│ ├── LibraryManifest.xml
│ ├── java
│ │ └── com
│ │ │ └── example
│ │ │ └── android
│ │ │ └── lyra
│ │ │ ├── BUILD
│ │ │ └── MainActivity.java
│ ├── jni_lyra_benchmark_lib.cc
│ └── res
│ │ ├── drawable-v24
│ │ └── ic_launcher_foreground.xml
│ │ ├── drawable
│ │ └── ic_launcher_background.xml
│ │ ├── layout
│ │ └── activity_main.xml
│ │ ├── mipmap-anydpi-v26
│ │ ├── ic_launcher.xml
│ │ └── ic_launcher_round.xml
│ │ ├── mipmap-hdpi
│ │ ├── ic_launcher.png
│ │ └── ic_launcher_round.png
│ │ ├── mipmap-mdpi
│ │ ├── ic_launcher.png
│ │ └── ic_launcher_round.png
│ │ ├── mipmap-xhdpi
│ │ ├── ic_launcher.png
│ │ └── ic_launcher_round.png
│ │ ├── mipmap-xxhdpi
│ │ ├── ic_launcher.png
│ │ └── ic_launcher_round.png
│ │ ├── mipmap-xxxhdpi
│ │ ├── ic_launcher.png
│ │ └── ic_launcher_round.png
│ │ └── values
│ │ ├── colors.xml
│ │ ├── strings.xml
│ │ └── styles.xml
├── architecture_utils.h
├── buffered_filter_interface.h
├── buffered_resampler.cc
├── buffered_resampler.h
├── buffered_resampler_test.cc
├── cli_example
│ ├── BUILD
│ ├── decoder_main.cc
│ ├── decoder_main_lib.cc
│ ├── decoder_main_lib.h
│ ├── decoder_main_lib_test.cc
│ ├── encoder_main.cc
│ ├── encoder_main_lib.cc
│ ├── encoder_main_lib.h
│ └── encoder_main_lib_test.cc
├── comfort_noise_generator.cc
├── comfort_noise_generator.h
├── comfort_noise_generator_test.cc
├── dsp_utils.cc
├── dsp_utils.h
├── dsp_utils_test.cc
├── feature_estimator_interface.h
├── feature_extractor_interface.h
├── fixed_packet_loss_model.cc
├── fixed_packet_loss_model.h
├── fixed_packet_loss_model_test.cc
├── generative_model_interface.h
├── gilbert_model.cc
├── gilbert_model.h
├── gilbert_model_test.cc
├── log_mel_spectrogram_extractor_impl.cc
├── log_mel_spectrogram_extractor_impl.h
├── log_mel_spectrogram_extractor_impl_benchmark.cc
├── log_mel_spectrogram_extractor_impl_test.cc
├── lyra_benchmark.cc
├── lyra_benchmark_lib.cc
├── lyra_benchmark_lib.h
├── lyra_components.cc
├── lyra_components.h
├── lyra_config.cc
├── lyra_config.h
├── lyra_config.proto
├── lyra_config_test.cc
├── lyra_decoder.cc
├── lyra_decoder.h
├── lyra_decoder_interface.h
├── lyra_decoder_test.cc
├── lyra_encoder.cc
├── lyra_encoder.h
├── lyra_encoder_interface.h
├── lyra_encoder_test.cc
├── lyra_gan_model.cc
├── lyra_gan_model.h
├── lyra_gan_model_test.cc
├── lyra_integration_test.cc
├── model_coeffs
│ ├── lyra_config.binarypb
│ ├── lyragan.tflite
│ ├── quantizer.tflite
│ ├── soundstream_encoder.tflite
│ └── test_playback.wav
├── no_op_preprocessor.h
├── no_op_preprocessor_test.cc
├── noise_estimator.cc
├── noise_estimator.h
├── noise_estimator_interface.h
├── noise_estimator_test.cc
├── packet.h
├── packet_interface.h
├── packet_loss_model_interface.h
├── packet_test.cc
├── preprocessor_interface.h
├── resampler.cc
├── resampler.h
├── resampler_interface.h
├── resampler_test.cc
├── residual_vector_quantizer.cc
├── residual_vector_quantizer.h
├── residual_vector_quantizer_test.cc
├── soundstream_encoder.cc
├── soundstream_encoder.h
├── soundstream_encoder_test.cc
├── testdata
│ ├── BUILD
│ ├── incomplete_encoded_packet.lyra
│ ├── invalid.wav
│ ├── no_encoded_packet.lyra
│ ├── one_encoded_packet_16khz.lyra
│ ├── sample1_16kHz.wav
│ ├── sample1_32kHz.wav
│ ├── sample1_48kHz.wav
│ ├── sample1_8kHz.wav
│ ├── sample2_16kHz.wav
│ ├── sample2_32kHz.wav
│ ├── sample2_48kHz.wav
│ ├── sample2_8kHz.wav
│ └── two_encoded_packets_16khz.lyra
├── testing
│ ├── BUILD
│ ├── mock_feature_extractor.h
│ ├── mock_generative_model.h
│ ├── mock_lyra_decoder.h
│ ├── mock_lyra_encoder.h
│ ├── mock_noise_estimator.h
│ ├── mock_resampler.h
│ └── mock_vector_quantizer.h
├── tflite_model_wrapper.cc
├── tflite_model_wrapper.h
├── tflite_model_wrapper_test.cc
├── vector_quantizer_interface.h
├── wav_utils.cc
├── wav_utils.h
├── wav_utils_test.cc
└── zero_feature_estimator.h
├── patches
├── BUILD
└── com_google_absl_f863b622fe13612433fdf43f76547d5edda0c93001.diff
└── toolchain
├── BUILD
├── README.md
└── cc_toolchain_config.bzl
/.bazelversion:
--------------------------------------------------------------------------------
1 | 5.3.2
2 |
--------------------------------------------------------------------------------
/.clang-format:
--------------------------------------------------------------------------------
1 | Language: Cpp
2 | BasedOnStyle: Google
3 | DerivePointerAlignment: false
4 | PointerAlignment: Left
5 |
--------------------------------------------------------------------------------
/.github/actions/lyra-builder/action.yml:
--------------------------------------------------------------------------------
1 | name: lyra-builder
2 |
3 | description: Build lyra binary on different platforms
4 |
5 | inputs:
6 | platform:
7 | description: Platform to build
8 | required: true
9 | architecture:
10 | description: Architecture to build
11 | required: true
12 | build-options:
13 | description: Options in bazel build command
14 | required: false
15 | default: ""
16 |
17 | runs:
18 | using: composite
19 | steps:
20 | - shell: bash
21 | run: |
22 | export ANDROID_NDK_HOME="$ANDROID_HOME/ndk/21.4.7075529"
23 | if [[ "${{inputs.platform}}" != "android" ]]; then
24 | bazelisk test -c opt lyra/cli_example:encoder_main_lib_test ${{ inputs.build-options }}
25 | fi
26 | bazelisk build -c opt lyra/cli_example:encoder_main ${{ inputs.build-options }}
27 | - shell: bash
28 | run: |
29 | export ANDROID_NDK_HOME="$ANDROID_HOME/ndk/21.4.7075529"
30 | if [[ "${{inputs.platform}}" != "android" ]]; then
31 | bazelisk test -c opt lyra/cli_example:decoder_main_lib_test ${{ inputs.build-options }}
32 | fi
33 | bazelisk build -c opt lyra/cli_example:decoder_main ${{ inputs.build-options }}
34 | - shell: bash
35 | run: |
36 | mkdir action-product
37 | cp -r lyra/model_coeffs action-product/
38 | cp bazel-bin/lyra/cli_example/encoder_main action-product/lyra-encoder
39 | cp bazel-bin/lyra/cli_example/decoder_main action-product/lyra-decoder
40 | - uses: actions/upload-artifact@v2
41 | with:
42 | name: lyra-${{ inputs.platform }}-${{ inputs.architecture }}
43 | path: action-product
44 |
--------------------------------------------------------------------------------
/.github/actions/setup-lyra-deps/action.yml:
--------------------------------------------------------------------------------
1 | name: setup-lyra-deps
2 |
3 | description: Setup NDK and python for lyra build
4 |
5 | runs:
6 | using: composite
7 | steps:
8 | - shell: bash
9 | run: |
10 | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --install "ndk;21.4.7075529"
11 | python -m pip install --upgrade pip
12 | pip install numpy
13 |
--------------------------------------------------------------------------------
/.github/workflows/ci.yml:
--------------------------------------------------------------------------------
1 | name: build
2 |
3 | on: [push, pull_request]
4 |
5 | jobs:
6 | android-example:
7 | runs-on: ubuntu-latest
8 | steps:
9 | - name: Checkout repo
10 | uses: actions/checkout@v2
11 | - name: Set up Python 3.9
12 | uses: actions/setup-python@v4
13 | with:
14 | python-version: 3.9
15 | - name: Setup Lyra dependencies
16 | uses: ./.github/actions/setup-lyra-deps
17 | - name: Build Android App
18 | shell: bash
19 | run: |
20 | export ANDROID_NDK_HOME="$ANDROID_HOME/ndk/21.4.7075529"
21 | bazelisk build lyra/android_example:lyra_android_example --config=android_arm64 --copt=-DBENCHMARK
22 | - name: Copy artifacts
23 | shell: bash
24 | run: |
25 | mkdir action-product
26 | cp bazel-bin/lyra/android_example/lyra_android_example.apk action-product/lyra_example.apk
27 | - name: Upload artifact
28 | uses: actions/upload-artifact@v2
29 | with:
30 | name: lyra-android-example
31 | path: action-product
32 |
33 | android-arm64:
34 | runs-on: ubuntu-latest
35 | steps:
36 | - name: Checkout repo
37 | uses: actions/checkout@v2
38 | - name: Set up Python 3.9
39 | uses: actions/setup-python@v4
40 | with:
41 | python-version: 3.9
42 | - name: Setup Lyra dependencies
43 | uses: ./.github/actions/setup-lyra-deps
44 | - name: Build and upload
45 | uses: ./.github/actions/lyra-builder
46 | with:
47 | platform: android
48 | architecture: arm64
49 | build-options: --config=android_arm64
50 |
51 | linux-amd64:
52 | runs-on: ubuntu-latest
53 | steps:
54 | - name: Checkout repo
55 | uses: actions/checkout@v2
56 | - name: Set up Python 3.9
57 | uses: actions/setup-python@v4
58 | with:
59 | python-version: 3.9
60 | - name: Setup Lyra dependencies
61 | uses: ./.github/actions/setup-lyra-deps
62 | - name: Build and upload
63 | uses: ./.github/actions/lyra-builder
64 | with:
65 | platform: linux
66 | architecture: amd64
67 |
68 | macos-amd64:
69 | runs-on: macos-latest
70 | steps:
71 | - name: Checkout repo
72 | uses: actions/checkout@v2
73 | - name: Set up Python 3.9
74 | uses: actions/setup-python@v4
75 | with:
76 | python-version: 3.9
77 | - name: Setup Lyra dependencies
78 | uses: ./.github/actions/setup-lyra-deps
79 | - name: Build and upload
80 | uses: ./.github/actions/lyra-builder
81 | with:
82 | platform: macos
83 | architecture: amd64
84 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *~
--------------------------------------------------------------------------------
/BUILD:
--------------------------------------------------------------------------------
1 | package(default_visibility = [":__subpackages__"])
2 |
3 | licenses(["notice"])
4 |
5 | exports_files(["LICENSE"])
6 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # How to Contribute
2 |
3 | We'd love to accept your patches and contributions to this project. There are
4 | just a few small guidelines you need to follow.
5 |
6 | ## Contributor License Agreement
7 |
8 | Contributions to this project must be accompanied by a Contributor License
9 | Agreement (CLA). You (or your employer) retain the copyright to your
10 | contribution; this simply gives us permission to use and redistribute your
11 | contributions as part of the project. Head over to
12 | to see your current agreements on file or
13 | to sign a new one.
14 |
15 | You generally only need to submit a CLA once, so if you've already submitted one
16 | (even if it was for a different project), you probably don't need to do it
17 | again.
18 |
19 | ## Code Reviews
20 |
21 | All submissions, including submissions by project members, require review. We
22 | use GitHub pull requests for this purpose. Consult
23 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
24 | information on using pull requests.
25 |
26 | ## Community Guidelines
27 |
28 | This project follows
29 | [Google's Open Source Community Guidelines](https://opensource.google/conduct/).
30 |
--------------------------------------------------------------------------------
/android_configure.bzl:
--------------------------------------------------------------------------------
1 | """Repository rule for Android SDK and NDK autoconfiguration.
2 | This rule is a no-op unless the required android environment variables are set.
3 | """
4 |
5 | # Based on https://github.com/envoyproxy/envoy-mobile/pull/2039
6 | # Workaround for https://github.com/bazelbuild/bazel/issues/14260
7 |
8 | def _android_autoconf_impl(repository_ctx):
9 | sdk_rule = ""
10 | if repository_ctx.os.environ.get("ANDROID_HOME"):
11 | sdk_rule = """
12 | native.android_sdk_repository(
13 | name="androidsdk",
14 | api_level=30,
15 | build_tools_version="30.0.3",
16 | )
17 | """
18 |
19 | ndk_rule = ""
20 | if repository_ctx.os.environ.get("ANDROID_NDK_HOME"):
21 | ndk_rule = """
22 | native.android_ndk_repository(
23 | name="androidndk",
24 | api_level=30,
25 | )
26 | """
27 |
28 | if ndk_rule == "" and sdk_rule == "":
29 | sdk_rule = "pass"
30 |
31 | repository_ctx.file("BUILD.bazel", "")
32 | repository_ctx.file("android_configure.bzl", """
33 | def android_workspace():
34 | {}
35 | {}
36 | """.format(sdk_rule, ndk_rule))
37 |
38 | android_configure = repository_rule(
39 | implementation = _android_autoconf_impl,
40 | )
41 |
--------------------------------------------------------------------------------
/external/eigen.BUILD:
--------------------------------------------------------------------------------
1 | # Description:
2 | # Eigen is a C++ template library for linear algebra: vectors,
3 | # matrices, and related algorithms.
4 | #
5 | # Note that this build file is mostly stolen from
6 | # https://github.com/tensorflow/tensorflow/blob/master/third_party/eigen.BUILD
7 |
8 | licenses([
9 | # Note: Eigen is an MPL2 library that includes GPL v3 and LGPL v2.1+ code.
10 | # We've taken special care to not reference any restricted code.
11 | "reciprocal", # MPL2
12 | "notice", # Portions BSD
13 | ])
14 |
15 | exports_files(["COPYING.MPL2"])
16 |
17 | # License-restricted (i.e. not reciprocal or notice) files inside Eigen/...
18 | EIGEN_RESTRICTED_FILES = [
19 | "Eigen/src/OrderingMethods/Amd.h",
20 | "Eigen/src/SparseCholesky/**",
21 | ]
22 |
23 | # Notable transitive dependencies of restricted files inside Eigen/...
24 | EIGEN_RESTRICTED_DEPS = [
25 | "Eigen/Eigen",
26 | "Eigen/IterativeLinearSolvers",
27 | "Eigen/MetisSupport",
28 | "Eigen/Sparse",
29 | "Eigen/SparseCholesky",
30 | "Eigen/SparseLU",
31 | ]
32 |
33 | EIGEN_FILES = [
34 | "Eigen/**",
35 | ]
36 |
37 | # List of files picked up by glob but actually part of another target.
38 | EIGEN_EXCLUDE_FILES = [
39 | "Eigen/src/Core/arch/AVX/PacketMathGoogleTest.cc",
40 | ]
41 |
42 | # Files known to be under MPL2 license.
43 | EIGEN_MPL2_HEADER_FILES = glob(
44 | EIGEN_FILES,
45 | exclude = EIGEN_EXCLUDE_FILES +
46 | EIGEN_RESTRICTED_FILES +
47 | EIGEN_RESTRICTED_DEPS + [
48 | # Guarantees any file missed by excludes above will not compile.
49 | "Eigen/src/Core/util/NonMPL2.h",
50 | "Eigen/**/CMakeLists.txt",
51 | ],
52 | )
53 |
54 | cc_library(
55 | name = "eigen",
56 | hdrs = EIGEN_MPL2_HEADER_FILES,
57 | defines = [
58 | # This define (mostly) guarantees we don't link any problematic
59 | # code. We use it, but we do not rely on it, as evidenced above.
60 | "EIGEN_MPL2_ONLY",
61 | ],
62 | includes = ["."],
63 | visibility = ["//visibility:public"],
64 | )
65 |
--------------------------------------------------------------------------------
/external/fft2d.BUILD:
--------------------------------------------------------------------------------
1 | package(default_visibility = ["//visibility:public"])
2 |
3 | # Unrestricted use; can only distribute original package.
4 | # See fft2d/readme2d.txt
5 | licenses(["notice"])
6 |
7 | exports_files(["LICENSE"])
8 |
9 | # This is the main 2D FFT library. The 2D FFTs in this library call
10 | # 1D FFTs. In addition, fast DCTs are provided for the special case
11 | # of 8x8 and 16x16. This code in this library is referred to as
12 | # "Version II" on http://momonga.t.u-tokyo.ac.jp/~ooura/fft.html.
13 | cc_library(
14 | name = "fft2d",
15 | srcs = [
16 | "fft2d/alloc.c",
17 | "fft2d/fftsg.c",
18 | "fft2d/fftsg2d.c",
19 | "fft2d/shrtdct.c",
20 | ],
21 | textual_hdrs = [
22 | "fft2d/alloc.h",
23 | ],
24 | linkopts = ["-lm"],
25 | )
26 |
--------------------------------------------------------------------------------
/lyra/android_example/AndroidManifest.xml:
--------------------------------------------------------------------------------
1 |
2 |
17 |
18 |
21 |
22 |
23 |
30 |
31 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
--------------------------------------------------------------------------------
/lyra/android_example/BUILD:
--------------------------------------------------------------------------------
1 | load("@rules_android//android:rules.bzl", "android_binary", "android_library")
2 |
3 | # Placeholder for jni import
4 |
5 | package(default_visibility = ["//visibility:public"])
6 |
7 | licenses(["notice"])
8 |
9 | # Android App rules
10 | cc_library(
11 | name = "jni_lyra_benchmark_lib",
12 | srcs = [
13 | "jni_lyra_benchmark_lib.cc",
14 | "@org_tensorflow//tensorflow/lite:libtensorflowlite.so",
15 | ],
16 | linkopts = [
17 | "-landroid",
18 | "-ldl",
19 | "-llog",
20 | ],
21 | deps = [
22 | "//lyra:lyra_benchmark_lib",
23 | "//lyra:lyra_config",
24 | "//lyra/cli_example:decoder_main_lib",
25 | "//lyra/cli_example:encoder_main_lib",
26 | "@com_google_absl//absl/random",
27 | ],
28 | alwayslink = True,
29 | )
30 |
31 | android_library(
32 | name = "lyra_android_lib",
33 | srcs = ["//lyra/android_example/java/com/example/android/lyra:MainActivity.java"],
34 | custom_package = "com.example.android.lyra",
35 | manifest = "LibraryManifest.xml",
36 | resource_files = glob(["res/**/*"]),
37 | deps = [
38 | ":jni_lyra_benchmark_lib",
39 | "@maven//:androidx_annotation_annotation",
40 | "@maven//:androidx_appcompat_appcompat",
41 | "@maven//:androidx_constraintlayout_constraintlayout",
42 | "@maven//:androidx_core_core",
43 | "@org_tensorflow//tensorflow/lite:libtensorflowlite.so",
44 | ],
45 | )
46 |
47 | filegroup(
48 | name = "assets",
49 | srcs = ["//lyra:android_example_assets"],
50 | )
51 |
52 | android_binary(
53 | name = "lyra_android_example",
54 | assets = [
55 | ":assets",
56 | ],
57 | assets_dir = "model_coeffs",
58 | custom_package = "com.example.android.lyra",
59 | manifest = "AndroidManifest.xml",
60 | multidex = "native",
61 | deps = [":lyra_android_lib"],
62 | )
63 |
--------------------------------------------------------------------------------
/lyra/android_example/LibraryManifest.xml:
--------------------------------------------------------------------------------
1 |
2 |
17 |
18 |
20 |
21 |
22 |
23 |
30 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
--------------------------------------------------------------------------------
/lyra/android_example/java/com/example/android/lyra/BUILD:
--------------------------------------------------------------------------------
1 | licenses(["notice"])
2 |
3 | exports_files(
4 | ["MainActivity.java"],
5 | visibility = ["//lyra/android_example:__subpackages__"],
6 | )
7 |
--------------------------------------------------------------------------------
/lyra/android_example/jni_lyra_benchmark_lib.cc:
--------------------------------------------------------------------------------
1 | // Copyright 2021 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | #include
16 |
17 | #include
18 | #include
19 |
20 | #include "absl/random/random.h"
21 | #include "lyra/cli_example/decoder_main_lib.h"
22 | #include "lyra/cli_example/encoder_main_lib.h"
23 | #include "lyra/lyra_benchmark_lib.h"
24 | #include "lyra/lyra_config.h"
25 |
26 | extern "C" JNIEXPORT jshortArray JNICALL
27 | Java_com_example_android_lyra_MainActivity_encodeAndDecodeSamples(
28 | JNIEnv* env, jobject this_obj, jshortArray samples, jint sample_length,
29 | jint bitrate, jstring model_base_path) {
30 | std::vector samples_vector(sample_length);
31 | std::vector features;
32 | std::vector decoded_audio;
33 | jshortArray java_decoded_audio = nullptr;
34 | env->GetShortArrayRegion(samples, jsize{0}, sample_length,
35 | &samples_vector[0]);
36 |
37 | const char* cpp_model_base_path = env->GetStringUTFChars(model_base_path, 0);
38 | std::unique_ptr decoder =
39 | chromemedia::codec::LyraDecoder::Create(
40 | 16000, chromemedia::codec::kNumChannels, cpp_model_base_path);
41 |
42 | absl::BitGen gen;
43 | if (chromemedia::codec::EncodeWav(
44 | samples_vector, chromemedia::codec::kNumChannels, 16000, bitrate,
45 | false, false, cpp_model_base_path, &features) &&
46 | chromemedia::codec::DecodeFeatures(
47 | features, chromemedia::codec::BitrateToPacketSize(bitrate),
48 | /*randomize_num_samples_requested=*/false, gen, decoder.get(),
49 | nullptr, &decoded_audio)) {
50 | java_decoded_audio = env->NewShortArray(decoded_audio.size());
51 | env->SetShortArrayRegion(java_decoded_audio, 0, decoded_audio.size(),
52 | &decoded_audio[0]);
53 | }
54 |
55 | env->ReleaseStringUTFChars(model_base_path, cpp_model_base_path);
56 |
57 | return java_decoded_audio;
58 | }
59 |
60 | extern "C" JNIEXPORT int JNICALL
61 | Java_com_example_android_lyra_MainActivity_lyraBenchmark(
62 | JNIEnv* env, jobject this_obj, jint num_cond_vectors,
63 | jstring model_base_path) {
64 | const char* cpp_model_base_path = env->GetStringUTFChars(model_base_path, 0);
65 | int ret =
66 | chromemedia::codec::lyra_benchmark(num_cond_vectors, cpp_model_base_path,
67 | /*benchmark_feature_extraction=*/true,
68 | /*benchmark_quantizer=*/true,
69 | /*benchmark_generative_model=*/true);
70 | env->ReleaseStringUTFChars(model_base_path, cpp_model_base_path);
71 | return ret;
72 | }
73 |
--------------------------------------------------------------------------------
/lyra/android_example/res/drawable-v24/ic_launcher_foreground.xml:
--------------------------------------------------------------------------------
1 |
16 |
17 |
23 |
28 |
29 |
35 |
38 |
41 |
42 |
43 |
44 |
50 |
51 |
--------------------------------------------------------------------------------
/lyra/android_example/res/layout/activity_main.xml:
--------------------------------------------------------------------------------
1 |
2 |
17 |
18 |
24 |
25 |
34 |
44 |
54 |
63 |
71 |
81 |
82 |
--------------------------------------------------------------------------------
/lyra/android_example/res/mipmap-anydpi-v26/ic_launcher.xml:
--------------------------------------------------------------------------------
1 |
2 |
17 |
18 |
19 |
20 |
21 |
--------------------------------------------------------------------------------
/lyra/android_example/res/mipmap-anydpi-v26/ic_launcher_round.xml:
--------------------------------------------------------------------------------
1 |
2 |
17 |
18 |
19 |
20 |
21 |
--------------------------------------------------------------------------------
/lyra/android_example/res/mipmap-hdpi/ic_launcher.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/lyra/47698dadf0010abff6a848e02642f55f806d4842/lyra/android_example/res/mipmap-hdpi/ic_launcher.png
--------------------------------------------------------------------------------
/lyra/android_example/res/mipmap-hdpi/ic_launcher_round.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/lyra/47698dadf0010abff6a848e02642f55f806d4842/lyra/android_example/res/mipmap-hdpi/ic_launcher_round.png
--------------------------------------------------------------------------------
/lyra/android_example/res/mipmap-mdpi/ic_launcher.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/lyra/47698dadf0010abff6a848e02642f55f806d4842/lyra/android_example/res/mipmap-mdpi/ic_launcher.png
--------------------------------------------------------------------------------
/lyra/android_example/res/mipmap-mdpi/ic_launcher_round.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/lyra/47698dadf0010abff6a848e02642f55f806d4842/lyra/android_example/res/mipmap-mdpi/ic_launcher_round.png
--------------------------------------------------------------------------------
/lyra/android_example/res/mipmap-xhdpi/ic_launcher.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/lyra/47698dadf0010abff6a848e02642f55f806d4842/lyra/android_example/res/mipmap-xhdpi/ic_launcher.png
--------------------------------------------------------------------------------
/lyra/android_example/res/mipmap-xhdpi/ic_launcher_round.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/lyra/47698dadf0010abff6a848e02642f55f806d4842/lyra/android_example/res/mipmap-xhdpi/ic_launcher_round.png
--------------------------------------------------------------------------------
/lyra/android_example/res/mipmap-xxhdpi/ic_launcher.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/lyra/47698dadf0010abff6a848e02642f55f806d4842/lyra/android_example/res/mipmap-xxhdpi/ic_launcher.png
--------------------------------------------------------------------------------
/lyra/android_example/res/mipmap-xxhdpi/ic_launcher_round.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/lyra/47698dadf0010abff6a848e02642f55f806d4842/lyra/android_example/res/mipmap-xxhdpi/ic_launcher_round.png
--------------------------------------------------------------------------------
/lyra/android_example/res/mipmap-xxxhdpi/ic_launcher.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/lyra/47698dadf0010abff6a848e02642f55f806d4842/lyra/android_example/res/mipmap-xxxhdpi/ic_launcher.png
--------------------------------------------------------------------------------
/lyra/android_example/res/mipmap-xxxhdpi/ic_launcher_round.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/google/lyra/47698dadf0010abff6a848e02642f55f806d4842/lyra/android_example/res/mipmap-xxxhdpi/ic_launcher_round.png
--------------------------------------------------------------------------------
/lyra/android_example/res/values/colors.xml:
--------------------------------------------------------------------------------
1 |
2 |
17 |
18 |
19 | #3F51B5
20 | #303F9F
21 | #FF4081
22 |
23 |
--------------------------------------------------------------------------------
/lyra/android_example/res/values/strings.xml:
--------------------------------------------------------------------------------
1 |
16 |
17 |
18 | Lyra Example App
20 | Record from microphone
22 | Benchmark
24 | Encode/decode to speaker
26 | Stop recording
28 | Benchmark in progress. See logcat output for progress.
30 | Finished benchmarking. See logcat for results.
32 |
33 |
--------------------------------------------------------------------------------
/lyra/android_example/res/values/styles.xml:
--------------------------------------------------------------------------------
1 |
16 |
17 |
18 |
19 |
20 |
26 |
27 |
28 |
--------------------------------------------------------------------------------
/lyra/architecture_utils.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2021 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | #ifndef LYRA_ARCHITECTURE_UTILS_H_
18 | #define LYRA_ARCHITECTURE_UTILS_H_
19 |
20 | // Placeholder for get runfiles header.
21 | #include "include/ghc/filesystem.hpp"
22 |
23 | namespace chromemedia {
24 | namespace codec {
25 |
26 | ghc::filesystem::path GetCompleteArchitecturePath(
27 | const ghc::filesystem::path& model_path) {
28 | return model_path;
29 | }
30 |
31 | } // namespace codec
32 | } // namespace chromemedia
33 |
34 | #endif // LYRA_ARCHITECTURE_UTILS_H_
35 |
--------------------------------------------------------------------------------
/lyra/buffered_filter_interface.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2022 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | #ifndef LYRA_BUFFERED_FILTER_INTERFACE_H_
18 | #define LYRA_BUFFERED_FILTER_INTERFACE_H_
19 |
20 | #include
21 | #include
22 | #include
23 | #include
24 |
25 | namespace chromemedia {
26 | namespace codec {
27 |
28 | // Interface for filters which can not output an arbitrary number of samples.
29 | class BufferedFilterInterface {
30 | public:
31 | virtual ~BufferedFilterInterface() {}
32 |
33 | virtual std::optional> FilterAndBuffer(
34 | const std::function>(int)>&
35 | sample_generator,
36 | int num_samples) = 0;
37 | };
38 |
39 | } // namespace codec
40 | } // namespace chromemedia
41 |
42 | #endif // LYRA_BUFFERED_FILTER_INTERFACE_H_
43 |
--------------------------------------------------------------------------------
/lyra/buffered_resampler.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2021 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | #ifndef LYRA_BUFFERED_RESAMPLER_H_
18 | #define LYRA_BUFFERED_RESAMPLER_H_
19 |
20 | #include
21 | #include
22 | #include
23 | #include
24 | #include
25 |
26 | #include "lyra/buffered_filter_interface.h"
27 | #include "lyra/resampler_interface.h"
28 |
29 | namespace chromemedia {
30 | namespace codec {
31 |
32 | // A class to buffer samples at the external sample rate and reuse them to
33 | // produce an arbitrary number of samples. Time domain samples can only be
34 | // generated in multiples of |external_sample_rate| / |internal_sample_rate|,
35 | // if this ratio is greater than 1.
36 | class BufferedResampler : public BufferedFilterInterface {
37 | public:
38 | static std::unique_ptr Create(int internal_sample_rate,
39 | int external_sample_rate);
40 |
41 | // Buffer the newly generated samples and resample them to produce
42 | // |num_external_samples_requested| samples.
43 | std::optional> FilterAndBuffer(
44 | const std::function>(int)>&
45 | sample_generator,
46 | int num_external_samples_requested) override;
47 |
48 | private:
49 | explicit BufferedResampler(std::unique_ptr resampler);
50 |
51 | // Helper function to inform the generative model how many samples need to
52 | // be generated if a total of |num_external_samples_requested| are requested
53 | // upstream. Computed based on the number of leftover samples from previous
54 | // calls and the external to internal resample ratio.
55 | int GetInternalNumSamplesToGenerate(int num_external_samples_requested) const;
56 |
57 | // Use at most |num_external_samples_requested| from |leftover_samples_| to
58 | // fill the beginning of |samples|.
59 | int UseLeftoverSamples(int num_external_samples_requested,
60 | std::vector* samples);
61 |
62 | std::vector Resample(const std::vector& internal_samples);
63 |
64 | void CopyNewSamples(const std::vector& external_samples,
65 | int num_external_samples_requested, int num_leftover_used,
66 | std::vector* samples);
67 |
68 | // If the resample ratio is greater than 1, buffer at most
69 | // |external_sample_rate| / |internal_sample_rate_hz|/ - 1 leftover samples
70 | // from the last run. Otherwise this is unused.
71 | std::vector leftover_samples_;
72 |
73 | std::unique_ptr resampler_;
74 |
75 | friend class BufferedResamplerPeer;
76 | };
77 |
78 | } // namespace codec
79 | } // namespace chromemedia
80 |
81 | #endif // LYRA_BUFFERED_RESAMPLER_H_
82 |
--------------------------------------------------------------------------------
/lyra/cli_example/BUILD:
--------------------------------------------------------------------------------
1 | package(default_visibility = ["//visibility:public"])
2 |
3 | exports_files(
4 | srcs = [
5 | "decoder_main.cc",
6 | "decoder_main_lib.cc",
7 | "decoder_main_lib.h",
8 | "encoder_main.cc",
9 | "encoder_main_lib.cc",
10 | "encoder_main_lib.h",
11 | ],
12 | )
13 |
14 | cc_library(
15 | name = "encoder_main_lib",
16 | srcs = [
17 | "encoder_main_lib.cc",
18 | ],
19 | hdrs = [
20 | "encoder_main_lib.h",
21 | ],
22 | deps = [
23 | "//lyra:lyra_config",
24 | "//lyra:lyra_encoder",
25 | "//lyra:no_op_preprocessor",
26 | "//lyra:wav_utils",
27 | "@com_google_absl//absl/status",
28 | "@com_google_absl//absl/status:statusor",
29 | "@com_google_absl//absl/strings",
30 | "@com_google_absl//absl/time",
31 | "@com_google_absl//absl/types:span",
32 | "@com_google_glog//:glog",
33 | "@gulrak_filesystem//:filesystem",
34 | ],
35 | )
36 |
37 | cc_library(
38 | name = "decoder_main_lib",
39 | srcs = [
40 | "decoder_main_lib.cc",
41 | ],
42 | hdrs = [
43 | "decoder_main_lib.h",
44 | ],
45 | deps = [
46 | "//lyra:fixed_packet_loss_model",
47 | "//lyra:gilbert_model",
48 | "//lyra:lyra_config",
49 | "//lyra:lyra_decoder",
50 | "//lyra:packet_loss_model_interface",
51 | "//lyra:wav_utils",
52 | "@com_google_absl//absl/flags:marshalling",
53 | "@com_google_absl//absl/random",
54 | "@com_google_absl//absl/random:bit_gen_ref",
55 | "@com_google_absl//absl/status",
56 | "@com_google_absl//absl/strings",
57 | "@com_google_absl//absl/time",
58 | "@com_google_absl//absl/types:span",
59 | "@com_google_glog//:glog",
60 | "@gulrak_filesystem//:filesystem",
61 | ],
62 | )
63 |
64 | cc_test(
65 | name = "encoder_main_lib_test",
66 | size = "small",
67 | srcs = ["encoder_main_lib_test.cc"],
68 | data = [
69 | "//lyra:tflite_testdata",
70 | "//lyra/testdata:sample1_16kHz.wav",
71 | "//lyra/testdata:sample1_32kHz.wav",
72 | "//lyra/testdata:sample1_48kHz.wav",
73 | "//lyra/testdata:sample1_8kHz.wav",
74 | ],
75 | deps = [
76 | ":encoder_main_lib",
77 | "@com_google_absl//absl/flags:flag",
78 | "@com_google_absl//absl/strings",
79 | "@com_google_googletest//:gtest_main",
80 | "@gulrak_filesystem//:filesystem",
81 | ],
82 | )
83 |
84 | cc_test(
85 | name = "decoder_main_lib_test",
86 | size = "large",
87 | srcs = ["decoder_main_lib_test.cc"],
88 | data = [
89 | "//lyra:tflite_testdata",
90 | "//lyra/testdata:incomplete_encoded_packet.lyra",
91 | "//lyra/testdata:no_encoded_packet.lyra",
92 | "//lyra/testdata:one_encoded_packet_16khz.lyra",
93 | "//lyra/testdata:two_encoded_packets_16khz.lyra",
94 | ],
95 | shard_count = 4,
96 | deps = [
97 | ":decoder_main_lib",
98 | "//lyra:lyra_config",
99 | "//lyra:wav_utils",
100 | "@com_google_absl//absl/flags:flag",
101 | "@com_google_absl//absl/status:statusor",
102 | "@com_google_absl//absl/strings",
103 | "@com_google_googletest//:gtest_main",
104 | "@gulrak_filesystem//:filesystem",
105 | ],
106 | )
107 |
108 | cc_binary(
109 | name = "encoder_main",
110 | srcs = [
111 | "encoder_main.cc",
112 | ],
113 | data = ["//lyra:tflite_testdata"],
114 | linkopts = select({
115 | "//lyra:android_config": ["-landroid"],
116 | "//conditions:default": [],
117 | }),
118 | deps = [
119 | ":encoder_main_lib",
120 | "//lyra:architecture_utils",
121 | "@com_google_absl//absl/flags:flag",
122 | "@com_google_absl//absl/flags:parse",
123 | "@com_google_absl//absl/flags:usage",
124 | "@com_google_absl//absl/strings",
125 | "@com_google_glog//:glog",
126 | "@gulrak_filesystem//:filesystem",
127 | ],
128 | )
129 |
130 | cc_binary(
131 | name = "decoder_main",
132 | srcs = [
133 | "decoder_main.cc",
134 | ],
135 | data = ["//lyra:tflite_testdata"],
136 | linkopts = select({
137 | "//lyra:android_config": ["-landroid"],
138 | "//conditions:default": [],
139 | }),
140 | deps = [
141 | ":decoder_main_lib",
142 | "//lyra:architecture_utils",
143 | "@com_google_absl//absl/flags:flag",
144 | "@com_google_absl//absl/flags:parse",
145 | "@com_google_absl//absl/flags:usage",
146 | "@com_google_absl//absl/strings",
147 | "@com_google_glog//:glog",
148 | "@gulrak_filesystem//:filesystem",
149 | ],
150 | )
151 |
--------------------------------------------------------------------------------
/lyra/cli_example/decoder_main_lib.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2021 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | #ifndef LYRA_CLI_EXAMPLE_DECODER_MAIN_LIB_H_
18 | #define LYRA_CLI_EXAMPLE_DECODER_MAIN_LIB_H_
19 |
20 | #include
21 | #include
22 | #include
23 |
24 | #include "absl/random/bit_gen_ref.h"
25 | #include "absl/strings/string_view.h"
26 | #include "include/ghc/filesystem.hpp"
27 | #include "lyra/lyra_decoder.h"
28 | #include "lyra/packet_loss_model_interface.h"
29 |
30 | namespace chromemedia {
31 | namespace codec {
32 |
33 | // Used for custom command line flag in decoder_main.
34 | struct PacketLossPattern {
35 | explicit PacketLossPattern(const std::vector& starts,
36 | const std::vector& durations)
37 | : starts_(starts), durations_(durations) {}
38 |
39 | std::vector starts_;
40 | std::vector durations_;
41 | };
42 |
43 | std::string AbslUnparseFlag(chromemedia::codec::PacketLossPattern pattern);
44 |
45 | bool AbslParseFlag(absl::string_view text,
46 | chromemedia::codec::PacketLossPattern* p,
47 | std::string* error);
48 |
49 | // Decodes a vector of bytes into wav data.
50 | // If |packet_loss_model| is nullptr no packets will be lost.
51 | bool DecodeFeatures(const std::vector& packet_stream, int packet_size,
52 | bool randomize_num_samples_requested, absl::BitGenRef gen,
53 | LyraDecoder* decoder,
54 | PacketLossModelInterface* packet_loss_model,
55 | std::vector* decoded_audio);
56 |
57 | // Decodes an encoded features file into a wav file.
58 | // Uses the model and quant files located under |model_path|.
59 | // Given the file /tmp/lyra/file1.lyra exists and is a valid encoded file. For:
60 | // |encoded_path| = "/tmp/lyra/file1.lyra"
61 | // |output_path| = "/tmp/lyra/file1_decoded.lyra"
62 | // Then successful decoding will write out the file
63 | // /tmp/lyra/encoded/file1_decoded.wav
64 | bool DecodeFile(const ghc::filesystem::path& encoded_path,
65 | const ghc::filesystem::path& output_path, int sample_rate_hz,
66 | int bitrate, bool randomize_num_samples_requested,
67 | float packet_loss_rate, float average_burst_length,
68 | const PacketLossPattern& fixed_packet_loss_pattern,
69 | const ghc::filesystem::path& model_path);
70 |
71 | } // namespace codec
72 | } // namespace chromemedia
73 |
74 | #endif // LYRA_CLI_EXAMPLE_DECODER_MAIN_LIB_H_
75 |
--------------------------------------------------------------------------------
/lyra/cli_example/encoder_main.cc:
--------------------------------------------------------------------------------
1 | // Copyright 2021 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | #include
16 | #include // NOLINT(build/c++11)
17 |
18 | #include "absl/flags/flag.h"
19 | #include "absl/flags/parse.h"
20 | #include "absl/flags/usage.h"
21 | #include "absl/strings/string_view.h"
22 | #include "glog/logging.h" // IWYU pragma: keep
23 | #include "include/ghc/filesystem.hpp"
24 | #include "lyra/architecture_utils.h"
25 | #include "lyra/cli_example/encoder_main_lib.h"
26 |
27 | ABSL_FLAG(std::string, input_path, "",
28 | "Complete path to the WAV file to be encoded.");
29 | ABSL_FLAG(std::string, output_dir, "",
30 | "The dir for the encoded file to be written out. Recursively "
31 | "creates dir if it does not exist. Output files use the same "
32 | "name as the wav file they come from with a '.lyra' postfix. Will "
33 | "overwrite existing files.");
34 | ABSL_FLAG(int, bitrate, 3200,
35 | "The bitrate in bps with which to quantize the file. The "
36 | "bitrate options can be seen in lyra_encoder.h");
37 | ABSL_FLAG(bool, enable_preprocessing, false,
38 | "If enabled runs the input signal through the preprocessing "
39 | "module before encoding.");
40 | ABSL_FLAG(bool, enable_dtx, false,
41 | "Enables discontinuous transmission (DTX). DTX does not send packets "
42 | "when noise is detected.");
43 | ABSL_FLAG(std::string, model_path, "lyra/model_coeffs",
44 | "Path to directory containing TFLite files. For mobile this is the "
45 | "absolute path, like "
46 | "'/data/local/tmp/lyra/model_coeffs/'."
47 | " For desktop this is the path relative to the binary.");
48 |
49 | int main(int argc, char** argv) {
50 | absl::SetProgramUsageMessage(argv[0]);
51 | absl::ParseCommandLine(argc, argv);
52 |
53 | const ghc::filesystem::path input_path(absl::GetFlag(FLAGS_input_path));
54 | const ghc::filesystem::path output_dir(absl::GetFlag(FLAGS_output_dir));
55 | const ghc::filesystem::path model_path =
56 | chromemedia::codec::GetCompleteArchitecturePath(
57 | absl::GetFlag(FLAGS_model_path));
58 | const int bitrate = absl::GetFlag(FLAGS_bitrate);
59 | const bool enable_preprocessing = absl::GetFlag(FLAGS_enable_preprocessing);
60 | const bool enable_dtx = absl::GetFlag(FLAGS_enable_dtx);
61 |
62 | if (input_path.empty()) {
63 | LOG(ERROR) << "Flag --input_path not set.";
64 | return -1;
65 | }
66 | if (output_dir.empty()) {
67 | LOG(ERROR) << "Flag --output_dir not set.";
68 | return -1;
69 | }
70 |
71 | std::error_code error_code;
72 | if (!ghc::filesystem::is_directory(output_dir, error_code)) {
73 | LOG(INFO) << "Creating non existent output dir " << output_dir;
74 | if (!ghc::filesystem::create_directories(output_dir, error_code)) {
75 | LOG(ERROR) << "Tried creating output dir " << output_dir
76 | << " but failed.";
77 | return -1;
78 | }
79 | }
80 | const auto output_path =
81 | ghc::filesystem::path(output_dir) / input_path.stem().concat(".lyra");
82 |
83 | if (!chromemedia::codec::EncodeFile(input_path, output_path, bitrate,
84 | enable_preprocessing, enable_dtx,
85 | model_path)) {
86 | LOG(ERROR) << "Failed to encode " << input_path;
87 | return -1;
88 | }
89 | return 0;
90 | }
91 |
--------------------------------------------------------------------------------
/lyra/cli_example/encoder_main_lib.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2021 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | #ifndef LYRA_CLI_EXAMPLE_ENCODER_MAIN_LIB_H_
18 | #define LYRA_CLI_EXAMPLE_ENCODER_MAIN_LIB_H_
19 |
20 | #include
21 | #include
22 |
23 | #include "include/ghc/filesystem.hpp"
24 |
25 | namespace chromemedia {
26 | namespace codec {
27 |
28 | // Encodes a vector of wav_data into encoded_features.
29 | // Uses the quant files located under |model_path|.
30 | bool EncodeWav(const std::vector& wav_data, int num_channels,
31 | int sample_rate_hz, int bitrate, bool enable_preprocessing,
32 | bool enable_dtx, const ghc::filesystem::path& model_path,
33 | std::vector* encoded_features);
34 |
35 | // Encodes a wav file into an encoded feature file. Encodes num_samples from the
36 | // file at |wav_path| and writes the encoded features out to |output_path|.
37 | // Uses the quant files located under |model_path|.
38 | bool EncodeFile(const ghc::filesystem::path& wav_path,
39 | const ghc::filesystem::path& output_path, int bitrate,
40 | bool enable_preprocessing, bool enable_dtx,
41 | const ghc::filesystem::path& model_path);
42 |
43 | } // namespace codec
44 | } // namespace chromemedia
45 |
46 | #endif // LYRA_CLI_EXAMPLE_ENCODER_MAIN_LIB_H_
47 |
--------------------------------------------------------------------------------
/lyra/cli_example/encoder_main_lib_test.cc:
--------------------------------------------------------------------------------
1 | // Copyright 2021 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | #include "lyra/cli_example/encoder_main_lib.h"
16 |
17 | #include
18 | #include // NOLINT(build/c++11)
19 |
20 | // Placeholder for get runfiles header.
21 | // Placeholder for testing header.
22 | #include "absl/flags/flag.h"
23 | #include "absl/strings/string_view.h"
24 | #include "gtest/gtest.h"
25 | #include "include/ghc/filesystem.hpp"
26 |
27 | namespace chromemedia {
28 | namespace codec {
29 | namespace {
30 |
31 | static constexpr absl::string_view kWavFiles[] = {
32 | "sample1_8kHz", "sample1_16kHz", "sample1_32kHz", "sample1_48kHz"};
33 |
34 | static constexpr absl::string_view kTestdataDir = "lyra/testdata";
35 |
36 | class EncoderMainLibTest : public testing::Test {
37 | protected:
38 | EncoderMainLibTest()
39 | : output_dir_(ghc::filesystem::path(testing::TempDir()) / "output"),
40 | testdata_dir_(ghc::filesystem::current_path() / kTestdataDir),
41 | model_path_(ghc::filesystem::current_path() / "lyra/model_coeffs") {}
42 |
43 | void SetUp() override {
44 | std::error_code error_code;
45 | ghc::filesystem::create_directories(output_dir_, error_code);
46 | ASSERT_FALSE(error_code);
47 | }
48 |
49 | void TearDown() override {
50 | std::error_code error_code;
51 | ghc::filesystem::remove_all(output_dir_, error_code);
52 | ASSERT_FALSE(error_code);
53 | }
54 |
55 | const ghc::filesystem::path output_dir_;
56 | const ghc::filesystem::path testdata_dir_;
57 | const ghc::filesystem::path model_path_;
58 | };
59 |
60 | TEST_F(EncoderMainLibTest, WavFileNotFound) {
61 | const ghc::filesystem::path kNonExistentWav("should/not/exist.wav");
62 | const ghc::filesystem::path kOutputEncoded(output_dir_ / "exists.lyra");
63 |
64 | EXPECT_FALSE(EncodeFile(kNonExistentWav, kOutputEncoded, /*bitrate=*/3200,
65 | /*enable_preprocessing=*/false,
66 | /*enable_dtx=*/false, model_path_));
67 |
68 | std::error_code error_code;
69 | EXPECT_FALSE(ghc::filesystem::is_regular_file(kOutputEncoded, error_code));
70 | }
71 |
72 | TEST_F(EncoderMainLibTest, EncodeSingleWavFiles) {
73 | for (const auto wav_file : kWavFiles) {
74 | const auto kInputWavepath = (testdata_dir_ / wav_file).concat(".wav");
75 | const auto kOutputEncoded = (output_dir_ / wav_file).concat(".lyra");
76 | EXPECT_TRUE(EncodeFile(kInputWavepath, kOutputEncoded, /*bitrate=*/3200,
77 | /*enable_preprocessing=*/false,
78 | /*enable_dtx=*/false, model_path_));
79 | }
80 | }
81 |
82 | } // namespace
83 | } // namespace codec
84 | } // namespace chromemedia
85 |
--------------------------------------------------------------------------------
/lyra/comfort_noise_generator.cc:
--------------------------------------------------------------------------------
1 | // Copyright 2021 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | #include "lyra/comfort_noise_generator.h"
16 |
17 | #include
18 | #include
19 | #include
20 | #include
21 | #include
22 | #include
23 | #include
24 |
25 | #include "absl/memory/memory.h"
26 | #include "absl/random/random.h"
27 | #include "absl/types/span.h"
28 | #include "audio/dsp/mfcc/mel_filterbank.h"
29 | #include "audio/dsp/number_util.h"
30 | #include "audio/dsp/spectrogram/inverse_spectrogram.h"
31 | #include "glog/logging.h" // IWYU pragma: keep
32 | #include "lyra/dsp_utils.h"
33 | #include "lyra/log_mel_spectrogram_extractor_impl.h"
34 |
35 | namespace chromemedia {
36 | namespace codec {
37 |
38 | std::unique_ptr ComfortNoiseGenerator::Create(
39 | int sample_rate_hz, int num_samples_per_hop, int window_length_samples,
40 | int num_mel_bins) {
41 | const int kFftSize = static_cast(
42 | audio_dsp::NextPowerOfTwo(static_cast(window_length_samples)));
43 | const int kNumFftBins = kFftSize / 2 + 1;
44 | auto mel_filterbank = std::make_unique();
45 | if (!mel_filterbank->Initialize(
46 | kNumFftBins, static_cast(sample_rate_hz), num_mel_bins,
47 | LogMelSpectrogramExtractorImpl::GetLowerFreqLimit(),
48 | LogMelSpectrogramExtractorImpl::GetUpperFreqLimit(sample_rate_hz))) {
49 | LOG(ERROR) << "Could not initialize mel filterbank.";
50 | return nullptr;
51 | }
52 |
53 | auto inverse_spectrogram = std::make_unique();
54 | if (!inverse_spectrogram->Initialize(kFftSize, num_samples_per_hop)) {
55 | LOG(ERROR) << "Could not initialize inverse spectrogram.";
56 | return nullptr;
57 | }
58 |
59 | return absl::WrapUnique(new ComfortNoiseGenerator(
60 | sample_rate_hz, num_samples_per_hop, num_mel_bins,
61 | std::move(mel_filterbank), std::move(inverse_spectrogram)));
62 | }
63 |
64 | ComfortNoiseGenerator::ComfortNoiseGenerator(
65 | int sample_rate_hz, int num_samples_per_hop, int num_mel_bins,
66 | std::unique_ptr mel_filterbank,
67 | std::unique_ptr inverse_spectrogram)
68 | : GenerativeModel(num_samples_per_hop, num_mel_bins),
69 | mel_filterbank_(std::move(mel_filterbank)),
70 | inverse_spectrogram_(std::move(inverse_spectrogram)),
71 | squared_magnitude_fft_(num_samples_per_hop),
72 | reconstructed_samples_(num_samples_per_hop) {}
73 |
74 | bool ComfortNoiseGenerator::RunConditioning(
75 | const std::vector& features) {
76 | FftFromFeatures(features);
77 | return InvertFft();
78 | }
79 |
80 | std::optional> ComfortNoiseGenerator::RunModel(
81 | int num_samples) {
82 | return std::vector(
83 | reconstructed_samples_.begin() + next_sample_in_hop(),
84 | reconstructed_samples_.begin() + next_sample_in_hop() + num_samples);
85 | }
86 |
87 | void ComfortNoiseGenerator::FftFromFeatures(
88 | const std::vector& log_mel_features) {
89 | std::vector mel_features(log_mel_features.size());
90 | for (int i = 0; i < mel_features.size(); ++i) {
91 | mel_features.at(i) = static_cast(
92 | std::exp(log_mel_features.at(i) *
93 | LogMelSpectrogramExtractorImpl::GetNormalizationFactor()));
94 | }
95 | mel_filterbank_->EstimateInverse(mel_features, &squared_magnitude_fft_);
96 | }
97 |
98 | bool ComfortNoiseGenerator::InvertFft() {
99 | // Add random phase to squared-magnitude FFT to make it a complex FFT.
100 | // InverseSpectrogram class expects a 2D spectrogram, so one containing just
101 | // one slice is constructed.
102 | std::vector>> random_phase_spectrogram(1);
103 | absl::BitGen gen;
104 | for (int i = 0; i < squared_magnitude_fft_.size(); ++i) {
105 | double magnitude = sqrt(squared_magnitude_fft_.at(i));
106 | double random_angle = absl::Uniform(gen, 0, 2 * M_PI);
107 | random_phase_spectrogram[0].push_back(
108 | magnitude * std::exp(std::complex(0.0, 1.0) * random_angle));
109 | }
110 |
111 | std::vector temp_samples;
112 | if (!inverse_spectrogram_->Process(random_phase_spectrogram, &temp_samples)) {
113 | return false;
114 | }
115 |
116 | // Store samples in buffer to ensure continuity between samples.
117 | reconstructed_samples_ = ClipToInt16(absl::MakeConstSpan(temp_samples));
118 | return true;
119 | }
120 |
121 | } // namespace codec
122 | } // namespace chromemedia
123 |
--------------------------------------------------------------------------------
/lyra/comfort_noise_generator.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2021 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | #ifndef LYRA_COMFORT_NOISE_GENERATOR_H_
18 | #define LYRA_COMFORT_NOISE_GENERATOR_H_
19 |
20 | #include
21 | #include
22 | #include
23 | #include
24 |
25 | #include "audio/dsp/mfcc/mel_filterbank.h"
26 | #include "audio/dsp/spectrogram/inverse_spectrogram.h"
27 | #include "lyra/generative_model_interface.h"
28 |
29 | namespace chromemedia {
30 | namespace codec {
31 |
32 | // This class generates comfort noise by estimating audio samples that
33 | // correspond to the given features.
34 | class ComfortNoiseGenerator : public GenerativeModel {
35 | public:
36 | // Returns a nullptr on failure.
37 | static std::unique_ptr Create(
38 | int sample_rate_hz, int num_samples_per_hop, int window_length_samples,
39 | int num_mel_bins);
40 |
41 | ~ComfortNoiseGenerator() override {}
42 |
43 | private:
44 | ComfortNoiseGenerator(
45 | int sample_rate_hz, int num_samples_per_hop, int num_mel_bins,
46 | std::unique_ptr mel_filterbank,
47 | std::unique_ptr inverse_spectrogram);
48 |
49 | bool RunConditioning(const std::vector& features) override;
50 |
51 | std::optional> RunModel(int num_samples) override;
52 |
53 | // Estimates the Squared-Magnitude FFT that corresponds to the Log Mel
54 | // features. Returns true if the estimation completed successfully and false
55 | // otherwise.
56 | void FftFromFeatures(const std::vector& log_mel_features);
57 |
58 | // Produces time-domain inverse of a Squared-Magnitude FFT by adding a random
59 | // phase to each element. Returns true if the inversion completed successfully
60 | // and false otherwise.
61 | bool InvertFft();
62 |
63 | const std::unique_ptr mel_filterbank_;
64 | const std::unique_ptr inverse_spectrogram_;
65 |
66 | std::vector squared_magnitude_fft_;
67 | std::vector reconstructed_samples_;
68 | };
69 |
70 | } // namespace codec
71 | } // namespace chromemedia
72 |
73 | #endif // LYRA_COMFORT_NOISE_GENERATOR_H_
74 |
--------------------------------------------------------------------------------
/lyra/dsp_utils.cc:
--------------------------------------------------------------------------------
1 | // Copyright 2021 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | #include "lyra/dsp_utils.h"
16 |
17 | #include
18 | #include
19 |
20 | #include "absl/types/span.h"
21 | #include "audio/dsp/signal_vector_util.h"
22 | #include "glog/logging.h" // IWYU pragma: keep
23 |
24 | namespace chromemedia {
25 | namespace codec {
26 |
27 | std::optional LogSpectralDistance(
28 | const absl::Span first_log_spectrum,
29 | const absl::Span second_log_spectrum) {
30 | const int num_features = first_log_spectrum.size();
31 | if (num_features != second_log_spectrum.size()) {
32 | LOG(ERROR) << "Spectrum sizes are not equal.";
33 | return std::nullopt;
34 | }
35 | float log_spectral_distance = 0.f;
36 | for (int i = 0; i < num_features; ++i) {
37 | log_spectral_distance +=
38 | audio_dsp::Square(first_log_spectrum[i] - second_log_spectrum[i]);
39 | }
40 | return 10 * std::sqrt(log_spectral_distance / num_features);
41 | }
42 |
43 | } // namespace codec
44 | } // namespace chromemedia
45 |
--------------------------------------------------------------------------------
/lyra/dsp_utils.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2021 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | #ifndef LYRA_DSP_UTILS_H_
18 | #define LYRA_DSP_UTILS_H_
19 |
20 | #include
21 | #include
22 | #include
23 | #include
24 | #include
25 | #include
26 | #include
27 | #include
28 |
29 | #include "absl/types/span.h"
30 |
31 | namespace chromemedia {
32 | namespace codec {
33 |
34 | // The log-spectral distance (LSD) is a distance measure (expressed in dB)
35 | // between two spectra.
36 | std::optional LogSpectralDistance(
37 | const absl::Span first_log_spectrum,
38 | const absl::Span second_log_spectrum);
39 |
40 | // Given the source and target sample rate, this method converts the number of
41 | // samples from the former to the latter.
42 | inline int ConvertNumSamplesBetweenSampleRate(int source_num_samples,
43 | int source_sample_rate,
44 | int target_sample_rate) {
45 | return static_cast(std::ceil(static_cast(source_num_samples) *
46 | static_cast(target_sample_rate) /
47 | static_cast(source_sample_rate)));
48 | }
49 |
50 | // Clip values above max value or below min value for int16_t.
51 | // The quantization scheme uses native c rounding (non-centered, decimal
52 | // truncation)
53 | template
54 | typename std::enable_if::value, int16_t>::type
55 | ClipToInt16Scalar(T unit_value) {
56 | unit_value =
57 | std::max(unit_value, static_cast(std::numeric_limits::min()));
58 | return std::min(unit_value,
59 | static_cast(std::numeric_limits::max()));
60 | }
61 |
62 | // Clips a vector of unit-floats or unit-doubles to a vector of 16-bit
63 | // integers. Does not perform scaling.
64 | template
65 | typename std::enable_if::value,
66 | std::vector>::type
67 | ClipToInt16(const absl::Span input) {
68 | std::vector output;
69 | output.reserve(input.size());
70 | std::transform(input.begin(), input.end(), std::back_inserter(output),
71 | ClipToInt16Scalar);
72 | return output;
73 | }
74 |
75 | // Converts from a unit-float or unit-double to a 16-bit integer.
76 | // If |value| is in the [-1, 1) interval it will scale linearly to the
77 | // int16_t limits. Values outside the interval are clipped to the limits.
78 | // The clipping, rounding, and quantization follows ClipToInt16Scalar().
79 | template
80 | typename std::enable_if::value, int16_t>::type
81 | UnitToInt16Scalar(T value) {
82 | // First, scale |value| linearly to int16 ranges.
83 | // The unary negation is used here to scale by the negative min int16_t value,
84 | // which has a greater absolute value than the max.
85 | T int16_range_value = value * (-std::numeric_limits().min());
86 | // If value was outside the [-1, 1), clip to the min/max value.
87 | return ClipToInt16Scalar(int16_range_value);
88 | }
89 |
90 | // Converts from a vector of unit-floats or unit-doubles to a vector of 16-bit
91 | // integers.
92 | template
93 | typename std::enable_if::value,
94 | std::vector>::type
95 | UnitToInt16(const absl::Span input) {
96 | std::vector output;
97 | output.reserve(input.size());
98 | std::transform(input.begin(), input.end(), std::back_inserter(output),
99 | UnitToInt16Scalar);
100 | return output;
101 | }
102 |
103 | // Converts from a 16-bit integers to a unit-floats or unit-doubles.
104 | template
105 | typename std::enable_if::value, T>::type
106 | Int16ToUnitScalar(int16_t integer) {
107 | return -static_cast(integer) / std::numeric_limits().min();
108 | }
109 |
110 | // Converts from a vector of 16-bit integers to a vector of unit-floats or
111 | // unit-doubles.
112 | template
113 | typename std::enable_if::value, std::vector>::type
114 | Int16ToUnit(const absl::Span input) {
115 | std::vector output;
116 | output.reserve(input.size());
117 | std::transform(input.begin(), input.end(), std::back_inserter(output),
118 | Int16ToUnitScalar);
119 | return output;
120 | }
121 |
122 | } // namespace codec
123 | } // namespace chromemedia
124 |
125 | #endif // LYRA_DSP_UTILS_H_
126 |
--------------------------------------------------------------------------------
/lyra/dsp_utils_test.cc:
--------------------------------------------------------------------------------
1 | // Copyright 2021 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | #include "lyra/dsp_utils.h"
16 |
17 | #include
18 | #include
19 | #include
20 | #include
21 | #include
22 | #include
23 |
24 | #include "absl/types/span.h"
25 | #include "gtest/gtest.h"
26 |
27 | namespace chromemedia {
28 | namespace codec {
29 | namespace {
30 |
31 | TEST(DspUtilTest, LogSpectralDistanceTest) {
32 | std::vector first_log_spectrum(10);
33 | std::iota(first_log_spectrum.begin(), first_log_spectrum.end(), 0);
34 | std::vector second_log_spectrum(10);
35 | std::iota(second_log_spectrum.begin(), second_log_spectrum.end(), 1);
36 | const auto log_spectral_distance =
37 | LogSpectralDistance(absl::MakeConstSpan(first_log_spectrum),
38 | absl::MakeConstSpan(second_log_spectrum));
39 | ASSERT_TRUE(log_spectral_distance.has_value());
40 | EXPECT_NEAR(log_spectral_distance.value(), 10.0f, 0.0001);
41 | }
42 |
43 | using FloatingPointTypes = testing::Types;
44 | template
45 | class ConversionTest : public ::testing::Test {};
46 | TYPED_TEST_SUITE(ConversionTest, FloatingPointTypes);
47 |
48 | TYPED_TEST(ConversionTest, ClipToInt16ScalarTestClipsExtremeValues) {
49 | const int16_t kMaxExceeded =
50 | ClipToInt16Scalar(static_cast(10000000));
51 | EXPECT_EQ(kMaxExceeded, std::numeric_limits::max());
52 | const int16_t kMinExceeded =
53 | ClipToInt16Scalar(static_cast(-10000000));
54 | EXPECT_EQ(kMinExceeded, std::numeric_limits::min());
55 | }
56 |
57 | TYPED_TEST(ConversionTest, ClipToInt16ScalarTestTruncatesDecimal) {
58 | const int16_t kJustAboveZero =
59 | ClipToInt16Scalar(static_cast(.0001));
60 | EXPECT_EQ(kJustAboveZero, 0);
61 |
62 | const int16_t kJustBelowOne = ClipToInt16Scalar(static_cast(.999));
63 | EXPECT_EQ(kJustBelowOne, 0);
64 |
65 | const int16_t kShouldTruncateNegativeDecimalToZero =
66 | ClipToInt16Scalar(static_cast(-.0001));
67 | EXPECT_EQ(kShouldTruncateNegativeDecimalToZero, 0);
68 | }
69 |
70 | TYPED_TEST(ConversionTest, ClipToInt16ScalarTestBoundaryIdentity) {
71 | const int16_t kZero = ClipToInt16Scalar(static_cast(0));
72 | EXPECT_EQ(kZero, 0);
73 |
74 | const int16_t kMaxBoundary = ClipToInt16Scalar(
75 | static_cast(std::numeric_limits::max()));
76 | EXPECT_EQ(kMaxBoundary, std::numeric_limits::max());
77 |
78 | const int16_t kMinBoundary = ClipToInt16Scalar(
79 | static_cast(std::numeric_limits::min()));
80 | EXPECT_EQ(kMinBoundary, std::numeric_limits::min());
81 | }
82 |
83 | TYPED_TEST(ConversionTest, UnitToInt16ScalarTestExtremeValues) {
84 | const int16_t kMaxExceeded =
85 | UnitToInt16Scalar(static_cast(100000.0));
86 | EXPECT_EQ(kMaxExceeded, std::numeric_limits::max());
87 |
88 | const int16_t kMinExceeded =
89 | UnitToInt16Scalar(static_cast(-100000.0));
90 | EXPECT_EQ(kMinExceeded, std::numeric_limits::min());
91 | }
92 |
93 | TYPED_TEST(ConversionTest, UnitToInt16ScalarTestRoundsTowardsZero) {
94 | const int16_t kShouldRoundDownToZero =
95 | UnitToInt16Scalar(static_cast(1e-10));
96 | EXPECT_EQ(kShouldRoundDownToZero, 0);
97 |
98 | const int16_t kShouldRoundNegativeUpToZero =
99 | UnitToInt16Scalar(static_cast(-1e-10));
100 | EXPECT_EQ(kShouldRoundNegativeUpToZero, 0);
101 | }
102 |
103 | TYPED_TEST(ConversionTest, UnitToInt16ScalarTestBoundariesMapToLimits) {
104 | const int16_t kZero = UnitToInt16Scalar(static_cast(0.0));
105 | EXPECT_EQ(kZero, 0);
106 |
107 | const int16_t kMaxBoundary = UnitToInt16Scalar(static_cast(1.0));
108 | EXPECT_EQ(kMaxBoundary, std::numeric_limits::max());
109 |
110 | const int16_t kMinBoundary = UnitToInt16Scalar(static_cast(-1.0));
111 | EXPECT_EQ(kMinBoundary, std::numeric_limits::min());
112 | }
113 |
114 | TYPED_TEST(ConversionTest, Int16ToUnitScalarTestBoundariesMapToLimits) {
115 | const TypeParam kZero = Int16ToUnitScalar(0);
116 | EXPECT_EQ(kZero, 0);
117 |
118 | const TypeParam kMaxBoundary =
119 | Int16ToUnitScalar(std::numeric_limits::max());
120 | const TypeParam kStep = Int16ToUnitScalar(1);
121 | EXPECT_EQ(kMaxBoundary + kStep, 1.0);
122 |
123 | const TypeParam kMinBoundary =
124 | Int16ToUnitScalar(std::numeric_limits::min());
125 | EXPECT_EQ(kMinBoundary, -1.0);
126 | }
127 |
128 | } // namespace
129 | } // namespace codec
130 | } // namespace chromemedia
131 |
--------------------------------------------------------------------------------
/lyra/feature_estimator_interface.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2022 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | #ifndef LYRA_FEATURE_ESTIMATOR_INTERFACE_H_
18 | #define LYRA_FEATURE_ESTIMATOR_INTERFACE_H_
19 |
20 | #include
21 |
22 | #include "absl/types/span.h"
23 |
24 | namespace chromemedia {
25 | namespace codec {
26 |
27 | class FeatureEstimatorInterface {
28 | public:
29 | virtual ~FeatureEstimatorInterface() {}
30 |
31 | virtual void Update(absl::Span features) = 0;
32 |
33 | virtual std::vector Estimate() const = 0;
34 | };
35 |
36 | } // namespace codec
37 | } // namespace chromemedia
38 |
39 | #endif // LYRA_FEATURE_ESTIMATOR_INTERFACE_H_
40 |
--------------------------------------------------------------------------------
/lyra/feature_extractor_interface.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2021 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | #ifndef LYRA_FEATURE_EXTRACTOR_INTERFACE_H_
18 | #define LYRA_FEATURE_EXTRACTOR_INTERFACE_H_
19 |
20 | #include
21 | #include
22 | #include
23 |
24 | #include "absl/types/span.h"
25 |
26 | namespace chromemedia {
27 | namespace codec {
28 |
29 | // An interface to abstract the extraction of features from an incoming stream
30 | // of audio from the specific implementation of whichever features are
31 | // extracted.
32 | class FeatureExtractorInterface {
33 | public:
34 | virtual ~FeatureExtractorInterface() {}
35 |
36 | // Extracts features from the audio. On failure returns a nullopt.
37 | virtual std::optional> Extract(
38 | const absl::Span audio) = 0;
39 | };
40 |
41 | } // namespace codec
42 | } // namespace chromemedia
43 |
44 | #endif // LYRA_FEATURE_EXTRACTOR_INTERFACE_H_
45 |
--------------------------------------------------------------------------------
/lyra/fixed_packet_loss_model.cc:
--------------------------------------------------------------------------------
1 | // Copyright 2021 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | #include "lyra/fixed_packet_loss_model.h"
16 |
17 | #include
18 | #include
19 | #include
20 | #include
21 |
22 | namespace chromemedia {
23 | namespace codec {
24 |
25 | FixedPacketLossModel::FixedPacketLossModel(
26 | int sample_rate_hz, int num_samples_per_hop,
27 | const std::vector& burst_starts_seconds,
28 | const std::vector& burst_durations_seconds)
29 | : lost_packet_intervals_(burst_starts_seconds.size()), packet_index_(0) {
30 | // Combine entries from |burst_starts_seconds| and |burst_durations_seconds|
31 | // into pairs of packet loss intervals.
32 | std::transform(
33 | burst_starts_seconds.begin(), burst_starts_seconds.end(),
34 | burst_durations_seconds.begin(), lost_packet_intervals_.begin(),
35 | [sample_rate_hz, num_samples_per_hop](float start, float duration) {
36 | const int start_packet = static_cast(
37 | std::ceil(sample_rate_hz * start / num_samples_per_hop));
38 | const int end_packet = static_cast(std::ceil(
39 | sample_rate_hz * (start + duration) / num_samples_per_hop));
40 | return std::make_pair(start_packet, end_packet);
41 | });
42 | }
43 |
44 | bool FixedPacketLossModel::IsPacketReceived() {
45 | // The packet is received if none of the [start, end) intervals from
46 | // |lost_packet_intervals_| include the current |packet_index|.
47 | bool is_received =
48 | std::none_of(lost_packet_intervals_.begin(), lost_packet_intervals_.end(),
49 | [this](std::pair start_end) {
50 | return (this->packet_index_ >= start_end.first) &&
51 | (this->packet_index_ < start_end.second);
52 | });
53 | ++packet_index_;
54 | return is_received;
55 | }
56 |
57 | } // namespace codec
58 | } // namespace chromemedia
59 |
--------------------------------------------------------------------------------
/lyra/fixed_packet_loss_model.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2021 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | #ifndef LYRA_FIXED_PACKET_LOSS_MODEL_H_
18 | #define LYRA_FIXED_PACKET_LOSS_MODEL_H_
19 |
20 | #include
21 | #include
22 |
23 | #include "lyra/packet_loss_model_interface.h"
24 |
25 | namespace chromemedia {
26 | namespace codec {
27 |
28 | class FixedPacketLossModel : public PacketLossModelInterface {
29 | public:
30 | // Rounds burst durations up to align with hop boundaries.
31 | FixedPacketLossModel(int sample_rate_hz, int num_samples_per_hop,
32 | const std::vector& burst_starts_seconds,
33 | const std::vector& burst_durations_seconds);
34 |
35 | // Update the internal state according to the model parameters.
36 | // Returns true if the packet was received.
37 | // Returns false if the packet was lost.
38 | bool IsPacketReceived() override;
39 |
40 | private:
41 | std::vector> lost_packet_intervals_;
42 | int packet_index_;
43 | };
44 |
45 | } // namespace codec
46 | } // namespace chromemedia
47 |
48 | #endif // LYRA_FIXED_PACKET_LOSS_MODEL_H_
49 |
--------------------------------------------------------------------------------
/lyra/fixed_packet_loss_model_test.cc:
--------------------------------------------------------------------------------
1 | // Copyright 2021 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | #include "lyra/fixed_packet_loss_model.h"
16 |
17 | #include
18 | #include
19 |
20 | #include "gtest/gtest.h"
21 |
22 | namespace chromemedia {
23 | namespace codec {
24 | namespace {
25 |
26 | TEST(DeterministicPacketLossModelTest, TestPattern) {
27 | const int kSampleRateHz = 16000;
28 | const float kHopDurationSeconds = 0.02;
29 | const int kNumSamplesPerHop = kSampleRateHz * kHopDurationSeconds;
30 | // Starts are rounded to: 0.f, 0.04f, 0.12f.
31 | std::vector start_seconds({0.f, 0.05f, 0.12f});
32 | // Durations are rounded to 0.02f, 0.04f, 0.02f
33 | std::vector duration_seconds({0.02f, 0.04f, 0.01f});
34 | // Segements of packet loss are: [0.f, 0.02f), [0.06f, 0.1f), [0.12f, 0.14f).
35 | auto model = FixedPacketLossModel(kSampleRateHz, kNumSamplesPerHop,
36 | start_seconds, duration_seconds);
37 | EXPECT_FALSE(model.IsPacketReceived())
38 | << "0.00s to 0.02s packet should not be received.";
39 | EXPECT_TRUE(model.IsPacketReceived())
40 | << "0.02s to 0.04s packet should be received.";
41 | EXPECT_TRUE(model.IsPacketReceived())
42 | << "0.04s to 0.06s packet should be received.";
43 | EXPECT_FALSE(model.IsPacketReceived())
44 | << "0.06s to 0.08s packet should not be received.";
45 | EXPECT_FALSE(model.IsPacketReceived())
46 | << "0.08s to 0.10s packet should not be received.";
47 | EXPECT_TRUE(model.IsPacketReceived())
48 | << "0.10s to 0.12s packet should be received.";
49 | EXPECT_FALSE(model.IsPacketReceived())
50 | << "0.12s to 0.14s packet should not be received.";
51 | EXPECT_TRUE(model.IsPacketReceived())
52 | << "0.14s and onwards should be received.";
53 | }
54 |
55 | } // namespace
56 | } // namespace codec
57 | } // namespace chromemedia
58 |
--------------------------------------------------------------------------------
/lyra/generative_model_interface.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2021 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | #ifndef LYRA_GENERATIVE_MODEL_INTERFACE_H_
18 | #define LYRA_GENERATIVE_MODEL_INTERFACE_H_
19 |
20 | #include
21 | #include
22 | #include
23 | #include
24 |
25 | #include "glog/logging.h" // IWYU pragma: keep
26 |
27 | namespace chromemedia {
28 | namespace codec {
29 |
30 | // An interface to abstract the audio generation from the model
31 | // implementation.
32 | class GenerativeModelInterface {
33 | public:
34 | virtual ~GenerativeModelInterface() {}
35 |
36 | virtual bool AddFeatures(const std::vector& features) = 0;
37 |
38 | virtual std::optional> GenerateSamples(
39 | int num_samples) = 0;
40 |
41 | virtual int num_samples_available() const = 0;
42 | };
43 |
44 | // Enforces that features are added and then decoded via a FIFO queue.
45 | class GenerativeModel : public GenerativeModelInterface {
46 | public:
47 | virtual ~GenerativeModel() {}
48 |
49 | // Adds received features to the model.
50 | bool AddFeatures(const std::vector& features) override final {
51 | if (features.size() != num_features_) {
52 | LOG(ERROR) << "Expecting features to be of shape " << num_features_
53 | << " but were of shape " << features.size() << ".";
54 | return false;
55 | }
56 | features_queue_.push(features);
57 | return true;
58 | }
59 |
60 | // Runs the model and generates |num_samples| audio samples.
61 | // Returns a vector of audio samples on success. Returns a nullopt on failure.
62 | std::optional> GenerateSamples(
63 | int num_samples) override final {
64 | if (num_samples < 0) {
65 | LOG(ERROR) << "Number of samples must be positive.";
66 | return std::nullopt;
67 | }
68 | // Do not call costly models if no samples have been requested.
69 | if (num_samples == 0) {
70 | return std::vector(0);
71 | }
72 | if (num_samples_available() == 0) {
73 | LOG(ERROR) << "Tried generating " << num_samples << " samples but only "
74 | << num_samples_available() << " are available.";
75 | return std::nullopt;
76 | }
77 | if (next_sample_in_hop_ == 0) {
78 | if (!RunConditioning(features_queue_.front())) {
79 | return std::nullopt;
80 | }
81 | }
82 | const int num_samples_remaining =
83 | num_samples_per_hop_ - next_sample_in_hop_;
84 | if (num_samples > num_samples_remaining) {
85 | LOG(ERROR) << "Tried generating " << num_samples << " samples but only "
86 | << num_samples_remaining
87 | << " were available in current features.";
88 | return std::nullopt;
89 | }
90 | auto samples = RunModel(num_samples);
91 | if (samples.has_value()) {
92 | next_sample_in_hop_ += samples->size();
93 | // Cumulative samples generated are guaranteed to never straddle
94 | // multiples of |num_samples_per_hop_|.
95 | if (next_sample_in_hop_ == num_samples_per_hop_) {
96 | next_sample_in_hop_ = 0;
97 | features_queue_.pop();
98 | }
99 | }
100 | return samples;
101 | }
102 |
103 | int num_samples_available() const override final {
104 | return features_queue_.size() * num_samples_per_hop_ - next_sample_in_hop_;
105 | }
106 |
107 | protected:
108 | GenerativeModel(int num_samples_per_hop, int num_features)
109 | : num_samples_per_hop_(num_samples_per_hop),
110 | num_features_(num_features),
111 | next_sample_in_hop_(0) {
112 | VLOG(1) << "Number of features: " << num_features;
113 | VLOG(1) << "Number of samples per feature: " << num_samples_per_hop;
114 | }
115 |
116 | // Process the features on top of the queue.
117 | // Called from |GenerateSamples|.
118 | virtual bool RunConditioning(const std::vector& features) = 0;
119 |
120 | // Generate samples from the latest set of features added by |AddFeatures|,
121 | // which have already been processed by |RunConditioning|.
122 | virtual std::optional> RunModel(int num_samples) = 0;
123 |
124 | int next_sample_in_hop() const { return next_sample_in_hop_; }
125 |
126 | private:
127 | GenerativeModel() = delete;
128 |
129 | // Provide read-only access to these member variables in derived classes.
130 | const int num_samples_per_hop_;
131 | const int num_features_;
132 | int next_sample_in_hop_;
133 | std::queue> features_queue_;
134 | };
135 |
136 | } // namespace codec
137 | } // namespace chromemedia
138 |
139 | #endif // LYRA_GENERATIVE_MODEL_INTERFACE_H_
140 |
--------------------------------------------------------------------------------
/lyra/gilbert_model.cc:
--------------------------------------------------------------------------------
1 | // Copyright 2021 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | #include "lyra/gilbert_model.h"
16 |
17 | #include
18 | #include
19 |
20 | #include "absl/memory/memory.h"
21 | #include "glog/logging.h" // IWYU pragma: keep
22 |
23 | namespace chromemedia {
24 | namespace codec {
25 |
26 | std::unique_ptr GilbertModel::Create(float packet_loss_rate,
27 | float average_burst_length,
28 | bool random_seed) {
29 | if (average_burst_length < 1.f) {
30 | LOG(ERROR) << "Average Burst Length has to be at least 1, but was "
31 | << average_burst_length << ".";
32 | return nullptr;
33 | }
34 | if (packet_loss_rate < 0.f) {
35 | LOG(ERROR) << "Packet Loss Rate has to be positive, but was "
36 | << packet_loss_rate << ".";
37 | return nullptr;
38 | }
39 | if (packet_loss_rate > average_burst_length / (average_burst_length + 1.f)) {
40 | LOG(ERROR) << "Packet Loss Rate cannot be larger than "
41 | << "average_burst_length/(average_burst_length+1)="
42 | << average_burst_length / (average_burst_length + 1.f)
43 | << ", but was " << packet_loss_rate << ".";
44 | return nullptr;
45 | }
46 |
47 | unsigned int seed = 5489u;
48 | if (random_seed) {
49 | std::random_device rd;
50 | seed = rd();
51 | }
52 |
53 | return absl::WrapUnique(new GilbertModel(
54 | packet_loss_rate / (average_burst_length * (1.f - packet_loss_rate)),
55 | 1.f / average_burst_length, seed));
56 | }
57 |
58 | GilbertModel::GilbertModel(float received2lost_probability,
59 | float lost2received_probability, unsigned int seed)
60 | : received2lost_probability_(received2lost_probability),
61 | lost2received_probability_(lost2received_probability),
62 | is_packet_received_(true),
63 | gen_(seed) {}
64 |
65 | bool GilbertModel::IsPacketReceived() {
66 | bool current_packet_received = is_packet_received_;
67 | if (is_packet_received_) {
68 | if (prob_(gen_) < received2lost_probability_) {
69 | is_packet_received_ = false;
70 | }
71 | } else {
72 | if (prob_(gen_) < lost2received_probability_) {
73 | is_packet_received_ = true;
74 | }
75 | }
76 |
77 | return current_packet_received;
78 | }
79 |
80 | } // namespace codec
81 | } // namespace chromemedia
82 |
--------------------------------------------------------------------------------
/lyra/gilbert_model.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2021 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | #ifndef LYRA_GILBERT_MODEL_H_
18 | #define LYRA_GILBERT_MODEL_H_
19 |
20 | #include
21 | #include
22 |
23 | #include "lyra/packet_loss_model_interface.h"
24 |
25 | namespace chromemedia {
26 | namespace codec {
27 |
28 | // Gilbert model to simulate packet loss bursts.
29 | class GilbertModel : public PacketLossModelInterface {
30 | public:
31 | static std::unique_ptr Create(float packet_loss_rate,
32 | float average_burst_length,
33 | bool random_seed = true);
34 |
35 | // Update the internal state according to the corresponding probabilities.
36 | // Returns true if the packet was received.
37 | // Returns false if the packet was lost.
38 | bool IsPacketReceived() override;
39 |
40 | private:
41 | GilbertModel(float received2lost_probability, float lost2received_probability,
42 | unsigned int seed);
43 |
44 | const float received2lost_probability_;
45 | const float lost2received_probability_;
46 |
47 | bool is_packet_received_;
48 |
49 | // Not using absl::Uniform because it can't ensure the same output between
50 | // runs even with explicit seeding. GilbertModelTest.DistributionFollowsParams
51 | // would be more complex with a MockingBitGen where the results have to be set
52 | // manually.
53 | std::mt19937 gen_;
54 | std::uniform_real_distribution prob_;
55 | };
56 |
57 | } // namespace codec
58 | } // namespace chromemedia
59 |
60 | #endif // LYRA_GILBERT_MODEL_H_
61 |
--------------------------------------------------------------------------------
/lyra/gilbert_model_test.cc:
--------------------------------------------------------------------------------
1 | // Copyright 2021 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | #include "lyra/gilbert_model.h"
16 |
17 | #include
18 | #include
19 | #include
20 |
21 | #include "gtest/gtest.h"
22 |
23 | namespace chromemedia {
24 | namespace codec {
25 | namespace {
26 |
27 | TEST(GilbertModelTest, TooSmallAverageBurstLength) {
28 | ASSERT_EQ(nullptr, GilbertModel::Create(0.3, 0.5));
29 | }
30 |
31 | TEST(GilbertModelTest, NegativePacketLossRate) {
32 | ASSERT_EQ(nullptr, GilbertModel::Create(-0.5, 0.5));
33 | }
34 |
35 | TEST(GilbertModelTest, TooLargePacketLossRate) {
36 | ASSERT_EQ(nullptr, GilbertModel::Create(0.7, 2));
37 | }
38 |
39 | TEST(GilbertModelTest, DistributionFollowsParams) {
40 | // The tolerances have been determined empirically, since the model doesn't
41 | // make it easy to draw theoretical tolerances.
42 | const float kPacketLossRate = 0.5f;
43 | const float kPLRTolerance = 3e-5f;
44 | const float kAverageBurstLength = 2.f;
45 | const float kABLTolerance = 4e-4f;
46 | const int kNumPackets = 1000000;
47 | auto gilbert_model =
48 | GilbertModel::Create(kPacketLossRate, kAverageBurstLength, false);
49 | ASSERT_NE(nullptr, gilbert_model);
50 | int packets_received = 0;
51 | int burst_length = 0;
52 | std::vector burst_lengths;
53 |
54 | for (int i = 0; i < kNumPackets; ++i) {
55 | if (gilbert_model->IsPacketReceived()) {
56 | ++packets_received;
57 | if (burst_length > 0) {
58 | burst_lengths.push_back(burst_length);
59 | burst_length = 0;
60 | }
61 | } else {
62 | ++burst_length;
63 | }
64 | }
65 | float estimated_packet_loss_rate =
66 | static_cast(kNumPackets - packets_received) / kNumPackets;
67 | float estimated_average_burst_length =
68 | static_cast(
69 | std::accumulate(burst_lengths.begin(), burst_lengths.end(), 0ul)) /
70 | burst_lengths.size();
71 |
72 | EXPECT_NEAR(kPacketLossRate, estimated_packet_loss_rate, kPLRTolerance);
73 | EXPECT_NEAR(kAverageBurstLength, estimated_average_burst_length,
74 | kABLTolerance);
75 | }
76 |
77 | } // namespace
78 | } // namespace codec
79 | } // namespace chromemedia
80 |
--------------------------------------------------------------------------------
/lyra/log_mel_spectrogram_extractor_impl.h:
--------------------------------------------------------------------------------
1 | /*
2 | * Copyright 2021 Google LLC
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 |
17 | #ifndef LYRA_LOG_MEL_SPECTROGRAM_EXTRACTOR_IMPL_H_
18 | #define LYRA_LOG_MEL_SPECTROGRAM_EXTRACTOR_IMPL_H_
19 |
20 | #include
21 | #include
22 | #include
23 | #include
24 |
25 | #include "absl/types/span.h"
26 | #include "audio/dsp/mfcc/mel_filterbank.h"
27 | #include "audio/dsp/spectrogram/spectrogram.h"
28 | #include "lyra/feature_extractor_interface.h"
29 |
30 | namespace chromemedia {
31 | namespace codec {
32 |
33 | // This class extracts mel spectrogram features from audio samples.
34 | class LogMelSpectrogramExtractorImpl : public FeatureExtractorInterface {
35 | public:
36 | // Returns a nullptr if creation fails.
37 | static std::unique_ptr Create(
38 | int sample_rate_hz, int hop_length_samples, int window_length_samples,
39 | int num_mel_bins);
40 |
41 | ~LogMelSpectrogramExtractorImpl() override {}
42 |
43 | // Extracts the mel features from the audio. On failure returns a nullopt.
44 | // The size of |audio| must match the value of |hop_length_samples_|.
45 | // This assumes that audio samples are passed in order.
46 | std::optional> Extract(
47 | const absl::Span audio) override;
48 |
49 | // Returns the lower frequency limit used to initialize the MelFilterbank
50 | // class.
51 | static double GetLowerFreqLimit();
52 |
53 | // Returns the upper frequency limit used to initialize the MelFilterbank
54 | // class.
55 | static double GetUpperFreqLimit(int sample_rate_hz);
56 |
57 | // Returns the normalization factor used to normalize the log of the mel
58 | // features.
59 | static float GetNormalizationFactor();
60 |
61 | // Returns minimum value, that represents silence.
62 | static float GetSilenceValue();
63 |
64 | private:
65 | LogMelSpectrogramExtractorImpl() = delete;
66 | LogMelSpectrogramExtractorImpl(
67 | std::unique_ptr spectrogram,
68 | std::unique_ptr mel_filterbank,
69 | int hop_length_samples);
70 |
71 | const std::unique_ptr spectrogram_;
72 | const std::unique_ptr mel_filterbank_;
73 | const int hop_length_samples_;
74 | std::vector samples_;
75 | };
76 |
77 | } // namespace codec
78 | } // namespace chromemedia
79 |
80 | #endif // LYRA_LOG_MEL_SPECTROGRAM_EXTRACTOR_IMPL_H_
81 |
--------------------------------------------------------------------------------
/lyra/log_mel_spectrogram_extractor_impl_benchmark.cc:
--------------------------------------------------------------------------------
1 | // Copyright 2021 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | #include
16 | #include
17 | #include
18 |
19 | #include "absl/random/random.h"
20 | #include "absl/types/span.h"
21 | #include "benchmark/benchmark.h"
22 | #include "lyra/log_mel_spectrogram_extractor_impl.h"
23 |
24 | static constexpr int kTestSampleRateHz = 16000;
25 | static constexpr int kNumMelBins = 10;
26 |
27 | void BenchmarkExtractFeatures(benchmark::State& state, const int hop_length,
28 | const int window_length) {
29 | std::unique_ptr
30 | feature_extractor_ =
31 | chromemedia::codec::LogMelSpectrogramExtractorImpl::Create(
32 | kTestSampleRateHz, hop_length, window_length, kNumMelBins);
33 | // We create random audio vectors to avoid any caching in the benchmark.
34 | const int16_t num_rand_vectors = 10000;
35 | absl::BitGen gen;
36 | std::vector rand_vec(hop_length);
37 | std::vector> audio_vec(num_rand_vectors);
38 | for (auto& audio : audio_vec) {
39 | for (auto& sample : rand_vec) {
40 | sample = absl::Uniform(gen);
41 | }
42 | audio = absl::MakeConstSpan(rand_vec);
43 | }
44 |
45 | for (auto _ : state) {
46 | auto features = feature_extractor_->Extract(
47 | audio_vec[absl::Uniform(gen, 0, num_rand_vectors)]);
48 | }
49 | }
50 |
51 | void BM_ExtractSmallFeatures(benchmark::State& state) {
52 | BenchmarkExtractFeatures(state, 6, 12);
53 | }
54 |
55 | void BM_ExtractMediumFeatures(benchmark::State& state) {
56 | BenchmarkExtractFeatures(state, 480, 960);
57 | }
58 |
59 | void BM_ExtractLargeFeatures(benchmark::State& state) {
60 | BenchmarkExtractFeatures(state, 2400, 4800);
61 | }
62 |
63 | void BM_ExtractMediumFeaturesLongWindows(benchmark::State& state) {
64 | BenchmarkExtractFeatures(state, 480, 4800);
65 | }
66 |
67 | BENCHMARK(BM_ExtractSmallFeatures);
68 | BENCHMARK(BM_ExtractMediumFeatures);
69 | BENCHMARK(BM_ExtractLargeFeatures);
70 | BENCHMARK(BM_ExtractMediumFeaturesLongWindows);
71 | BENCHMARK_MAIN();
72 |
--------------------------------------------------------------------------------
/lyra/log_mel_spectrogram_extractor_impl_test.cc:
--------------------------------------------------------------------------------
1 | // Copyright 2021 Google LLC
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | #include "lyra/log_mel_spectrogram_extractor_impl.h"
16 |
17 | #include
18 | #include
19 | #include
20 | #include
21 | #include
22 |
23 | #include "absl/types/span.h"
24 | #include "gmock/gmock.h"
25 | #include "gtest/gtest.h"
26 |
27 | namespace chromemedia {
28 | namespace codec {
29 | namespace {
30 |
31 | static constexpr int kTestSampleRateHz = 16000;
32 | static constexpr int kNumMelBins = 10;
33 | static constexpr int kHopLengthSamples = 5;
34 | static constexpr int kWindowLengthSamples = 10;
35 | static constexpr int kNumOutputMelFeatures = 3;
36 |
37 | static constexpr int16_t kWavData[] = {7954, 10085, 8733, 10844, 29949,
38 | -549, 20833, 30345, 18086, 11375,
39 | -27309, 12323, -22891, -23360, 11958};
40 |
41 | // These results were obtained by running
42 | // audio/dsp/mfcc/mfcc_mel.LogMelSpectrogram on kWavData pre-pended with 5
43 | // zeros and then dividing the results by 10. The parameters used were:
44 | // audio_sample_rate=16000
45 | // log_additive_offset=0.0
46 | // log_floor=500
47 | // window_length_secs=0.000625 (window_length_samples=10)
48 | // hop_length_secs=0.0003125 (hop_length_samples=5)
49 | // window_type="hann"
50 | // fft_length=None
51 | // upper_edge_hz=0.99*(16000/2)
52 | // lower_edge_hz=0
53 | static constexpr float kMelBins[][10] = {
54 | {0.62146081, 0.62146081, 0.79771997, 1.00416802, 0.73013308, 0.96676503,
55 | 0.87643814, 0.89284485, 0.90586112, 0.8633126},
56 | {0.62146081, 0.62146081, 0.89000145, 1.09644949, 0.76740002, 1.00403196,
57 | 0.8919037, 0.99746922, 1.06052462, 1.08220812},
58 | {0.62146081, 0.62146081, 0.83526758, 1.04171563, 0.82093681, 1.05756876,
59 | 0.96348656, 1.01345318, 1.07686605, 1.12100911}};
60 |
61 | class LogMelSpectrogramExtractorImplTest : public testing::Test {
62 | protected:
63 | void SetUp() override {
64 | feature_extractor_ = LogMelSpectrogramExtractorImpl::Create(
65 | kTestSampleRateHz, kHopLengthSamples, kWindowLengthSamples,
66 | kNumMelBins);
67 | ASSERT_NE(feature_extractor_, nullptr);
68 | }
69 |
70 | std::unique_ptr feature_extractor_;
71 | };
72 |
73 | TEST_F(LogMelSpectrogramExtractorImplTest, ThreeFeaturesEqualExpected) {
74 | for (int i = 0; i < kNumOutputMelFeatures; ++i) {
75 | const absl::Span audio = absl::MakeConstSpan(
76 | &kWavData[i * kHopLengthSamples], kHopLengthSamples);
77 |
78 | auto features = feature_extractor_->Extract(audio);
79 |
80 | EXPECT_TRUE(features.has_value());
81 | EXPECT_THAT(features.value(),
82 | testing::Pointwise(testing::FloatEq(), kMelBins[i]));
83 | }
84 | }
85 |
86 | TEST_F(LogMelSpectrogramExtractorImplTest, SamplesLongerThanExpected) {
87 | std::vector