├── tests ├── resources │ ├── hello.wav │ └── hello.mel.spectrogram ├── esi-dictate-test.el ├── esi-dictate.erts ├── esi-utils-test.el ├── test-helper.el ├── test-prep.c └── esi-test.el ├── Makefile ├── .gitignore ├── src ├── esi-io.h ├── esi-prep.h ├── esi-io.c └── esi-core.c ├── dg.py ├── esi-utils.el ├── README.org ├── esi-dictate.el └── LICENSE /tests/resources/hello.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lepisma/emacs-speech-input/HEAD/tests/resources/hello.wav -------------------------------------------------------------------------------- /tests/esi-dictate-test.el: -------------------------------------------------------------------------------- 1 | ;;; -*- lexical-binding: t; -*- 2 | 3 | (require 'esi-dictate) 4 | 5 | (ert-deftest test-esi-dictate-edits () 6 | (ert-test-erts-file "esi-dictate.erts" 7 | (lambda () 8 | (let ((splits (dictate-break-input (buffer-string)))) 9 | (delete-region (point-min) (point-max)) 10 | (insert (esi-dictate-make-edits (car splits) (cdr splits))))))) 11 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test clean 2 | 3 | all: esi-core.so 4 | 5 | test: tests/test-prep 6 | cask exec buttercup -L . 7 | tests/test-prep 8 | touch tests/test-prep.c 9 | 10 | LIBS = -lsndfile -lfftw3 -lm -lcblas -lsoundio 11 | 12 | TORCH_DIR = ./resources/libtorch 13 | TORCH_FLAGS = -I $(TORCH_DIR)/include -L $(TORCH_DIR)/lib -L ./lib \ 14 | '-Wl,-rpath,$$ORIGIN/$(TORCH_DIR)/lib' '-Wl,-rpath,$$ORIGIN/lib' \ 15 | -ltorch -lc10 -lpthread 16 | 17 | clean: 18 | rm -f esi-core.so esi-embed-core.so src/embed/esi-embed.o 19 | 20 | tests/test-prep: tests/test-prep.c esi-core.so 21 | gcc -lcmocka $(LIBS) -I ./src/ tests/test-prep.c -o $@ 22 | 23 | esi-core.so: $(wildcard src/*.c) $(wildcard src/*.h) 24 | gcc $(LIBS) -fPIC -pthread -shared $(wildcard src/*.c) -o $@ 25 | -------------------------------------------------------------------------------- /tests/esi-dictate.erts: -------------------------------------------------------------------------------- 1 | Name: dots 2 | No-After-Newline: t 3 | =-= 4 | I want to write something that's difficult to transcribe and then try correcting that. Write my name as abcd. ||No separate the letters with dots please. 5 | =-= 6 | I want to write something that's difficult to transcribe and then try correcting that. Write my name as a.b.c.d. 7 | =-=-= 8 | 9 | Name: spell-1 10 | No-After-Newline: t 11 | =-= 12 | hi easy, what are you doing? ||it's e s i 13 | =-= 14 | hi esi, what are you doing? 15 | =-=-= 16 | 17 | Name: spell-2 18 | No-After-Newline: t 19 | =-= 20 | abhinav to sha. ||no it's t u s h a r after abhinav, not to sha 21 | =-= 22 | abhinav tushar. 23 | =-=-= 24 | 25 | Name: del 26 | No-After-Newline: t 27 | =-= 28 | Abhinav to Tushar ||remove the to 29 | =-= 30 | Abhinav Tushar 31 | =-=-= 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled 2 | *.elc 3 | 4 | # Packaging 5 | .cask 6 | 7 | # Backup files 8 | *~ 9 | 10 | # Undo-tree save-files 11 | *.~undo-tree 12 | /.ccls-cache 13 | *.so 14 | /tests/test-prep 15 | /scratch.el 16 | 17 | # Created by https://www.gitignore.io/api/c++ 18 | # Edit at https://www.gitignore.io/?templates=c++ 19 | 20 | ### C++ ### 21 | # Prerequisites 22 | *.d 23 | 24 | # Compiled Object files 25 | *.slo 26 | *.lo 27 | *.o 28 | *.obj 29 | 30 | # Precompiled Headers 31 | *.gch 32 | *.pch 33 | 34 | # Compiled Dynamic libraries 35 | *.so 36 | *.dylib 37 | *.dll 38 | 39 | # Fortran module files 40 | *.mod 41 | *.smod 42 | 43 | # Compiled Static libraries 44 | *.lai 45 | *.la 46 | *.a 47 | *.lib 48 | 49 | # Executables 50 | *.exe 51 | *.out 52 | *.app 53 | 54 | # End of https://www.gitignore.io/api/c++ 55 | /resources/libtorch 56 | /dist/ 57 | /.envrc 58 | -------------------------------------------------------------------------------- /src/esi-io.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | // Return an array of single channel samples from headered audio bytes. Number 7 | // of samples is written in n_samples 8 | double* samples_from_buffer(char* buffer, size_t buffer_size, size_t* n_samples); 9 | 10 | // Start a background recording process (not process process) and return 11 | // instream with context. `buffer_duration_seconds` defines the length of last n 12 | // seconds to keep in memory. 13 | struct SoundIoInStream* start_background_recording(size_t sample_rate, size_t buffer_duration_seconds); 14 | 15 | // Read PCM data from recording buffer and return. Length of output is written in `output_size` 16 | char* read_background_recording(struct SoundIoInStream* instream, size_t* output_size); 17 | 18 | // Stop recording specified by the instream. 19 | void stop_background_recording(struct SoundIoInStream *instream); 20 | -------------------------------------------------------------------------------- /tests/esi-utils-test.el: -------------------------------------------------------------------------------- 1 | (require 'esi-utils) 2 | 3 | (describe "Vector" 4 | (it "parses properly" 5 | (expect (vector-approx-equal (esi-utils--array-parse-line "1 2 3.4 0.2 \n") [1.0 2.0 3.4 0.2]) :to-be t)) 6 | (it "formats properly" 7 | (expect (string= "1.0 2.0 3.4 0.2" (esi-utils--array-format-line [1.0 2.0 3.4 0.2])) :to-be t))) 8 | 9 | (describe "Array IO" 10 | :var (filepath) 11 | (before-all 12 | (setq filepath (make-temp-file "esi-test"))) 13 | 14 | (after-all 15 | (delete-file filepath)) 16 | 17 | (it "works for vector" 18 | (let ((array [1 2 3 4 5.0])) 19 | (esi-utils-save-array array filepath) 20 | (expect (vector-approx-equal array (esi-utils-load-array filepath))))) 21 | 22 | (it "works for matrix" 23 | (let ((array [[1 2 3] [4 5 6]])) 24 | (esi-utils-save-array array filepath) 25 | (expect (matrix-approx-equal array (esi-utils-load-array filepath)))))) 26 | -------------------------------------------------------------------------------- /tests/test-helper.el: -------------------------------------------------------------------------------- 1 | ;;; -*- lexical-binding: t; -*- 2 | 3 | (require 'cl-lib) 4 | (require 'dash) 5 | (require 'json) 6 | (require 'seq) 7 | 8 | (defun approx-equal (x y &optional rtol atol) 9 | (<= (abs (- x y)) (+ (* (or rtol 1e-5) (abs y)) (or atol 1e-8)))) 10 | 11 | (defun vector-approx-equal (a b &optional rtol atol) 12 | (and (= (length a) 13 | (length b) 14 | (apply #'+ (cl-mapcar (lambda (a b) (if (approx-equal a b rtol atol) 1 0)) a b))))) 15 | 16 | (defun matrix-approx-equal (a b &optional rtol atol) 17 | (and (= (length a) 18 | (length b) 19 | (apply #'+ (cl-mapcar (lambda (a b) (if (vector-approx-equal a b rtol atol) 1 0)) a b))))) 20 | 21 | (defun ffprobe (filepath &rest args) 22 | "Run ffprobe result for first stream on FILEPATH and return 23 | sexp output. ARGS is a list of format option strings." 24 | (let ((file-arg (shell-quote-argument filepath)) 25 | (format-args (mapconcat #'shell-quote-argument args " ")) 26 | (json-object-type 'alist) 27 | (json-array-type 'list)) 28 | (--> (format "ffprobe -v quiet -print_format json -show_streams %s %s" format-args file-arg) 29 | (shell-command-to-string it) 30 | (json-read-from-string it) 31 | (alist-get 'streams it) 32 | (car it)))) 33 | 34 | (defun dictate-break-input (text) 35 | "Break input text in content and command pair, if the separator is 36 | present in `text'." 37 | (let ((splits (mapcar #'string-trim (string-split text "||")))) 38 | (if (= (length splits) 2) 39 | (cons (car splits) (cadr splits)) 40 | splits))) 41 | -------------------------------------------------------------------------------- /dg.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | 5 | from deepgram import (DeepgramClient, LiveOptions, LiveTranscriptionEvents, 6 | Microphone) 7 | 8 | 9 | def on_message(self, result, **kwargs): 10 | sentence = result.channel.alternatives[0].transcript 11 | 12 | if len(sentence) == 0: 13 | return 14 | 15 | print(f"Output: {result.to_json()}") 16 | 17 | def on_metadata(self, metadata, **kwargs): 18 | print(f"\n\n{metadata}\n\n") 19 | 20 | def on_speech_started(self, speech_started, **kwargs): 21 | print(f"\n\n{speech_started}\n\n") 22 | 23 | def on_utterance_end(self, utterance_end, **kwargs): 24 | print(f"\n\n{utterance_end}\n\n") 25 | 26 | def on_error(self, error, **kwargs): 27 | print(f"\n\n{error}\n\n") 28 | 29 | 30 | def main(): 31 | try: 32 | deepgram = DeepgramClient( 33 | os.getenv("DG_API_KEY") 34 | ) 35 | 36 | dg_connection = deepgram.listen.live.v("1") 37 | 38 | dg_connection.on(LiveTranscriptionEvents.Transcript, on_message) 39 | dg_connection.on(LiveTranscriptionEvents.Metadata, on_metadata) 40 | dg_connection.on(LiveTranscriptionEvents.SpeechStarted, on_speech_started) 41 | dg_connection.on(LiveTranscriptionEvents.UtteranceEnd, on_utterance_end) 42 | dg_connection.on(LiveTranscriptionEvents.Error, on_error) 43 | 44 | options = LiveOptions( 45 | model="nova-2", 46 | punctuate=True, 47 | language="en-IN", 48 | encoding="linear16", 49 | channels=1, 50 | sample_rate=16000, 51 | # To get UtteranceEnd, the following must be set: 52 | interim_results=True, 53 | utterance_end_ms="1000", 54 | vad_events=True, 55 | ) 56 | dg_connection.start(options) 57 | microphone = Microphone(dg_connection.send) 58 | 59 | microphone.start() 60 | input("Press Enter to stop recording...\n\n") 61 | microphone.finish() 62 | 63 | dg_connection.finish() 64 | 65 | except Exception as e: 66 | print(f"Could not open socket: {e}") 67 | return 68 | 69 | if __name__ == "__main__": 70 | main() 71 | -------------------------------------------------------------------------------- /esi-utils.el: -------------------------------------------------------------------------------- 1 | ;;; esi-utils.el --- General utilities for esi -*- lexical-binding: t; -*- 2 | 3 | ;; Copyright (c) 2019 Abhinav Tushar 4 | 5 | ;; Author: Abhinav Tushar 6 | 7 | ;;; Commentary: 8 | 9 | ;; General utilities for esi 10 | ;; This file is not a part of GNU Emacs. 11 | 12 | ;;; License: 13 | 14 | ;; This program is free software: you can redistribute it and/or modify 15 | ;; it under the terms of the GNU General Public License as published by 16 | ;; the Free Software Foundation, either version 3 of the License, or 17 | ;; (at your option) any later version. 18 | 19 | ;; This program is distributed in the hope that it will be useful, 20 | ;; but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 | ;; GNU General Public License for more details. 23 | 24 | ;; You should have received a copy of the GNU General Public License 25 | ;; along with this program. If not, see . 26 | 27 | ;;; Code: 28 | 29 | (require 's) 30 | (require 'f) 31 | (require 'cl-lib) 32 | 33 | (defun esi-utils--array-format-line (it) 34 | "Take a number or vector and return a string representing IT as 35 | a single line." 36 | (cl-typecase it 37 | (vector (s-join " " (mapcar #'number-to-string it))) 38 | (t (number-to-string it)))) 39 | 40 | (defun esi-utils--array-parse-line (line) 41 | "Parse given LINE in the stored array format and return either 42 | a vector or number." 43 | (let ((splits (s-split " " (s-trim line)))) 44 | (if (> (length splits) 1) 45 | (cl-map 'vector #'string-to-number splits) 46 | (string-to-number (car splits))))) 47 | 48 | (defun esi-utils-save-array (array filepath) 49 | "Save provided ARRAY (vector or 2d) in numpy style format." 50 | (let ((text (s-join "\n" (mapcar #'esi-utils--array-format-line array)))) 51 | (f-write-text text 'utf-8 filepath))) 52 | 53 | (defun esi-utils-load-array (filepath) 54 | "Load array from the given filepath." 55 | (let ((text (s-trim (f-read-text filepath 'utf-8)))) 56 | (cl-map 'vector #'esi-utils--array-parse-line (s-split "\n" text)))) 57 | 58 | (provide 'esi-utils) 59 | 60 | ;;; esi-utils.el ends here 61 | -------------------------------------------------------------------------------- /tests/test-prep.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "esi-prep.h" 8 | 9 | #define EPSILON 1e-07 10 | 11 | void match_arrays(double* a, double* b, size_t size) { 12 | for (size_t i = 0; i < size; i++) { 13 | assert_float_equal(a[i], b[i], EPSILON); 14 | } 15 | } 16 | 17 | void match_complex_arrays(fftw_complex* a, fftw_complex* b, size_t size) { 18 | for (size_t i = 0; i < size; i++) { 19 | assert_float_equal(creal(a[i]), creal(b[i]), EPSILON); 20 | assert_float_equal(cimag(a[i]), cimag(b[i]), EPSILON); 21 | } 22 | } 23 | 24 | void test_window(void **state) { 25 | double window_1[] = { 0.0, 0.75, 0.75 }; 26 | match_arrays(hanning_window(3), window_1, 3); 27 | 28 | double window_2[] = { 29 | 0., 30 | 0.0954915, 31 | 0.3454915, 32 | 0.6545085, 33 | 0.9045085, 34 | 1., 35 | 0.9045085, 36 | 0.6545085, 37 | 0.3454915, 38 | 0.0954915 39 | }; 40 | match_arrays(hanning_window(10), window_2, 10); 41 | } 42 | 43 | void test_stft(void **state) { 44 | double samples[] = { 0.0, 0.5, 1.0 }; 45 | 46 | size_t n_rows, n_cols; 47 | 48 | fftw_complex truth_1[] = { 49 | 0.0 + 0.0 * I, 50 | 0.5 + 0.0 * I, 51 | 1.0 + 0.0 * I, 52 | 0.5 + 0.0 * I, 53 | 0.0 + 0.0 * I, 54 | -0.5 + 0.0 * I, 55 | -1.0 + 0.0 * I, 56 | -0.5 + 0.0 * I 57 | }; 58 | fftw_complex* stft_matrix = stft(samples, 3, 2, 1, &n_rows, &n_cols); 59 | assert_int_equal(n_rows * n_cols, 8); 60 | match_complex_arrays(truth_1, stft_matrix, n_rows * n_cols); 61 | 62 | fftw_complex truth_2[] = { 63 | 0.375 + 0.0 * I, 64 | 1.125 + 0.0 * I, 65 | 1.125 + 0.0 * I, 66 | -0.1875 + 0.3247595 * I, 67 | -0.5625 + 0.3247595 * I, 68 | -0.5625 - 0.3247595 * I 69 | }; 70 | stft_matrix = stft(samples, 3, 3, 1, &n_rows, &n_cols); 71 | assert_int_equal(n_rows * n_cols, 6); 72 | match_complex_arrays(truth_2, stft_matrix, n_rows * n_cols); 73 | } 74 | 75 | void test_mel_frequencies(void **state) { 76 | double truth[] = { 77 | 0.0, 213.11369888, 426.22739775, 639.34109663, 78 | 852.45479551, 1069.957853, 1332.9552128, 1660.59774631, 79 | 2068.77534111, 2577.2836447, 3210.78410654, 4000.0 80 | }; 81 | double* mel_fs = mel_frequencies(12, 0, 4000); 82 | match_arrays(truth, mel_fs, 12); 83 | } 84 | 85 | void test_frequencies(void **state) { 86 | double truth[] = { 87 | 0.0, 1378.125, 2756.25, 88 | 4134.375, 5512.5, 6890.625, 89 | 8268.75, 9646.875, 11025.0 90 | }; 91 | double* fft_fs = fft_frequencies(22050, 16); 92 | match_arrays(truth, fft_fs, 9); 93 | } 94 | 95 | int main(void) { 96 | const struct CMUnitTest tests[] = { 97 | cmocka_unit_test(test_stft), 98 | cmocka_unit_test(test_window), 99 | cmocka_unit_test(test_mel_frequencies), 100 | cmocka_unit_test(test_frequencies) 101 | }; 102 | 103 | return cmocka_run_group_tests(tests, NULL, NULL); 104 | } 105 | -------------------------------------------------------------------------------- /README.org: -------------------------------------------------------------------------------- 1 | #+begin_quote 2 | Development for this project has moved to sourcehut [[https://git.sr.ht/~lepisma/emacs-speech-input][here]]. This repository is now in a read-only state. 3 | #+end_quote 4 | 5 | #+TITLE: Emacs Speech Input 6 | 7 | #+HTML: GitHub 8 | 9 | Set of packages for speech and voice inputs in Emacs. Use cases are explained 10 | next with the packages they are available in: 11 | 12 | ** Dictation with Real-Time Editing 13 | ~esi-dictate~ is aimed to help you input faster[fn::Needs more work and empirical 14 | validation.] than simply using voice with post-edits or typing in. It allows 15 | blended workflows of voice and keyboard inputs assisted via LLMs for real time 16 | edits. 17 | 18 | At the very basic, it's a dictation tool allowing real-time edits. Most of the 19 | content gets inserted in the buffer via voice at a /voice cursor/. Utterances are 20 | automatically inferred as having edit suggestions, in which case, the current 21 | /voice context/ (displayed differently) is edited via an LLM. 22 | 23 | While working, the keyboard cursor is kept free so you can keep making finer 24 | edits independent of the dictation work. 25 | 26 | Some more notes on the design are in my blog post [[https://lepisma.xyz/2024/09/12/emacs-dictation-mode/index.html][here]]. 27 | 28 | *** Usage 29 | Presently this systems uses Deepgram via a Python script ~dg.py~ that you need to 30 | put in your ~PATH~ somewhere. Python >=3.10 is required. Afterwards, you would need 31 | to set the following configuration: 32 | 33 | #+begin_src emacs-lisp 34 | (use-package esi-dictate 35 | :vc (:fetcher github :repo lepisma/emacs-speech-input) 36 | :custom 37 | (esi-dictate-dg-api-key "") 38 | (esi-dictate-llm-provider ) 39 | ; Here is an example configuration for using OpenAI's 40 | ; (esi-dictate-llm-provider (make-llm-openai :key "" :chat-model "gpt-4o-mini")) 41 | :bind (:map esi-dictate-mode-map 42 | ("C-g" . esi-dictate-stop)) 43 | :config 44 | (setq llm-warn-on-nonfree nil) 45 | :hook (esi-dictate-speech-final . esi-dictate-fix-context)) 46 | #+end_src 47 | 48 | Start dictation mode using ~esi-dictate-start~. Use ~esi-dictate-stop~ to exit 49 | dictation mode. ~esi-dictate-fix-context~ is hooked to utterance end and does 50 | general fixes without asking for explicit commands. If you mark a region and 51 | call ~esi-dictate-move-here~, the voice context and cursor will move to the region 52 | bounds. If you just need to move the voice cursor, call move here function after 53 | moving point to the appropriate position. 54 | 55 | As of now this uses non-free models for both transcriptions (Deepgram) and edits 56 | (GPT4o-mini). This will change in the future. 57 | 58 | *** Customization 59 | To improve LLM performance, you can customize the ~esi-dictate-llm-prompt~ and add 60 | few shot examples in ~esi-dictate-fix-examples~. 61 | 62 | For visual improvements, customize ~esi-dictate-cursor~, ~esi-dictate-cursor-face~, 63 | ~esi-dictate-intermittent-face~, and ~esi-dictate-context-face~. 64 | 65 | ** Flight-mode Recording 66 | This is available as a dynamic module (~esi-core~) which needs a bit of clean up 67 | before I document the usage. 68 | -------------------------------------------------------------------------------- /tests/esi-test.el: -------------------------------------------------------------------------------- 1 | (require 'esi-core) 2 | (require 'esi-utils) 3 | (require 'cl-extra) 4 | (require 'cl-lib) 5 | (require 'f) 6 | (require 's) 7 | 8 | (describe "Version" 9 | (it "is correct" 10 | (expect (esi-core-version) :to-equal "0.0.2"))) 11 | 12 | (defun fft-real (vector) 13 | (cl-map 'vector (lambda (elem) (aref elem 0)) vector)) 14 | 15 | (defun fft-imag (vector) 16 | (cl-map 'vector (lambda (elem) (aref elem 1)) vector)) 17 | 18 | (defun stft-real (matrix) 19 | (cl-map 'vector #'fft-real matrix)) 20 | 21 | (defun stft-imag (matrix) 22 | (cl-map 'vector #'fft-imag matrix)) 23 | 24 | (describe "Sample reading" 25 | (it "is correct" 26 | (let ((samples (esi-core-wav-to-samples (with-temp-buffer 27 | (insert-file-contents-literally "tests/resources/hello.wav") 28 | (buffer-string)))) 29 | (true-samples (esi-utils-load-array "tests/resources/hello.samples"))) 30 | (expect (vector-approx-equal true-samples samples) :to-be t)))) 31 | 32 | (describe "FFT" 33 | (it "has correct real values" 34 | (let* ((samples (esi-utils-load-array "tests/resources/hello.samples")) 35 | (fft (esi-core--rfft samples)) 36 | (true-real-values (esi-utils-load-array "tests/resources/hello.fft.real"))) 37 | (expect (vector-approx-equal true-real-values (fft-real fft))))) 38 | (it "has correct imaginary values" 39 | (let* ((samples (esi-utils-load-array "tests/resources/hello.samples")) 40 | (fft (esi-core--rfft samples)) 41 | (true-imag-values (esi-utils-load-array "tests/resources/hello.fft.imag"))) 42 | (expect (vector-approx-equal true-imag-values (fft-imag fft)))))) 43 | 44 | (describe "STFT" 45 | (it "has correct real values" 46 | (let* ((samples (esi-utils-load-array "tests/resources/hello.samples")) 47 | (stft (esi-core--stft samples 2048 512)) 48 | (true-real-values (esi-utils-load-array "tests/resources/hello.stft.real"))) 49 | (expect (matrix-approx-equal true-real-values (stft-real stft))))) 50 | (it "has correct imaginary values" 51 | (let* ((samples (esi-utils-load-array "tests/resources/hello.samples")) 52 | (stft (esi-core--stft samples 2048 512)) 53 | (true-imag-values (esi-utils-load-array "tests/resources/hello.stft.imag"))) 54 | (expect (matrix-approx-equal true-imag-values (stft-imag stft)))))) 55 | 56 | (describe "Spectrogram" 57 | (it "is correct" 58 | (let ((samples (esi-utils-load-array "tests/resources/hello.samples")) 59 | (true-spectrogram (esi-utils-load-array "tests/resources/hello.spectrogram"))) 60 | (expect (matrix-approx-equal true-spectrogram (esi-core--spectrogram samples 2048 512 2)))))) 61 | 62 | (describe "Mel filterbank" 63 | (it "is correct" 64 | (let ((true-melfb (esi-utils-load-array "tests/resources/mel.fb")) 65 | (melfb (esi-core--mel-filter 8000 2048 12))) 66 | (expect (matrix-approx-equal melfb true-melfb))))) 67 | 68 | (describe "Mel spectrogram" 69 | (it "is correct" 70 | (let ((samples (esi-utils-load-array "tests/resources/hello.samples")) 71 | (true-spectrogram (esi-utils-load-array "tests/resources/hello.mel.spectrogram"))) 72 | (expect (matrix-approx-equal true-spectrogram (esi-core--mel-spectrogram samples 8000 2048 512 40)))))) 73 | 74 | (describe "Background recording" 75 | :var (filepath instream) 76 | (before-all 77 | (setq filepath (make-temp-file "esi-audio-test") 78 | instream (esi-core--start-background-recording 44100 2))) 79 | 80 | (after-all 81 | (delete-file filepath) 82 | (esi-core--stop-background-recording instream)) 83 | 84 | (it "works without errors" 85 | (let ((data (esi-core--read-background-recording-buffer instream))) 86 | (f-write-text data 'utf-8 filepath) 87 | (expect (string-to-number (alist-get 'duration (ffprobe filepath "-f" "s16le" "-ar" "44.1k" "-ac" "2"))) 88 | :to-equal 2.0)))) 89 | -------------------------------------------------------------------------------- /src/esi-prep.h: -------------------------------------------------------------------------------- 1 | // Audio preprocessing module, replicating a little of librosa's (Python) API. 2 | #pragma once 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | double* linspace(double start, double end, size_t n) { 12 | double *items = malloc(sizeof(double) * n); 13 | double diff = (end - start) / (n - 1); 14 | 15 | for (size_t i = 0; i < n; i++) { 16 | items[i] = start + (i * diff); 17 | } 18 | 19 | return items; 20 | } 21 | 22 | double* pad_zero(double* array, size_t size, size_t pad_size) { 23 | double* padded_array = calloc(pad_size + size + pad_size, sizeof(double)); 24 | 25 | for (size_t i = 0; i < size; i++) { 26 | padded_array[pad_size + i] = array[i]; 27 | } 28 | 29 | return padded_array; 30 | } 31 | 32 | // Similar effects as np.pad(array, pad_size, "reflect") 33 | double* pad_reflect(double* array, size_t size, size_t pad_size) { 34 | double *padded_array = pad_zero(array, size, pad_size); 35 | 36 | // TODO: - avoid copying here 37 | // - support more mirrors 38 | // - use a single closed form index and loop 39 | if (pad_size > size) { 40 | printf("Pad size larger than array size, returning 0 padded array as we don't support multiple mirrors."); 41 | return padded_array; 42 | } 43 | 44 | // Right pad 45 | for (size_t i = 0; i < pad_size; i++) { 46 | padded_array[pad_size + size + i] = array[size - i - 2]; 47 | } 48 | 49 | // Left pad 50 | for (size_t i = 0; i < pad_size; i++) { 51 | padded_array[pad_size - i - 1] = array[i + 1]; 52 | } 53 | return padded_array; 54 | } 55 | 56 | double* boxcar_window(size_t window_size) { 57 | double* window = malloc(sizeof(double) * window_size); 58 | for (size_t i = 0; i < window_size; i++) { 59 | window[i] = 1.0; 60 | } 61 | 62 | return window; 63 | } 64 | 65 | double* hanning_window(size_t window_size) { 66 | double* window = calloc(window_size, sizeof(double)); 67 | double* angles = linspace(-M_PI, M_PI, window_size + 1); 68 | 69 | // NOTE: Changing this value should cover hamming too 70 | double alpha = 0.5; 71 | double coeffs[] = { alpha, 1.0 - alpha }; 72 | 73 | for (size_t k = 0; k < 2; k++) { 74 | for (size_t i = 0; i < window_size; i++) { 75 | window[i] += coeffs[k] * cos(k * angles[i]); 76 | } 77 | } 78 | 79 | free(angles); 80 | // We are keeping the last value allocated. Since the apparent size is 81 | // window_size, we don't worry about people peeking at the last cell. 82 | return window; 83 | } 84 | 85 | fftw_complex *rfft(double *samples, size_t n_samples, size_t *n_output) { 86 | *n_output = floor(n_samples / 2) + 1; 87 | 88 | fftw_complex *fft_output = fftw_malloc(sizeof(fftw_complex) * (*n_output)); 89 | fftw_plan plan = fftw_plan_dft_r2c_1d(n_samples, samples, fft_output, FFTW_ESTIMATE); 90 | fftw_execute(plan); 91 | 92 | fftw_destroy_plan(plan); 93 | return fft_output; 94 | } 95 | 96 | fftw_complex *stft(double *samples, size_t n_samples, size_t n_fft, 97 | size_t hop_length, size_t *n_rows, size_t *n_cols) { 98 | size_t pad_size = floor(n_fft / 2); 99 | size_t n_fft_out = 1 + pad_size; 100 | 101 | // For centering the stft frame number indices 102 | double* padded_samples = pad_reflect(samples, n_samples, pad_size); 103 | size_t n_padded_samples = pad_size + n_samples + pad_size; 104 | 105 | size_t n_frames = floor((n_padded_samples - n_fft) / hop_length + 1); 106 | fftw_complex *stft_matrix = fftw_malloc(n_fft_out * n_frames * sizeof(fftw_complex)); 107 | 108 | double* frame = fftw_malloc(n_fft * sizeof(double)); 109 | fftw_complex* output = fftw_malloc(n_fft_out * sizeof(fftw_complex)); 110 | fftw_plan plan = fftw_plan_dft_r2c_1d(n_fft, frame, output, FFTW_ESTIMATE); 111 | 112 | double* window = hanning_window(n_fft); 113 | 114 | for (size_t i = 0; i < n_frames; i++) { 115 | for (size_t j = 0; j < n_fft; j++) { 116 | frame[j] = padded_samples[j + i * hop_length] * window[j]; 117 | } 118 | 119 | fftw_execute(plan); 120 | 121 | for (size_t j = 0; j < n_fft_out; j++) { 122 | stft_matrix[i + j * n_frames] = output[j]; 123 | } 124 | } 125 | 126 | *n_rows = n_fft_out; 127 | *n_cols = n_frames; 128 | fftw_destroy_plan(plan); 129 | free(window); 130 | fftw_free(frame); 131 | fftw_free(output); 132 | free(padded_samples); 133 | return stft_matrix; 134 | } 135 | 136 | double *spectrogram(double *samples, size_t n_samples, size_t n_fft, 137 | size_t hop_length, size_t power, size_t *n_rows, 138 | size_t *n_cols) { 139 | fftw_complex* stft_matrix = stft(samples, n_samples, n_fft, hop_length, n_rows, n_cols); 140 | double* sg_matrix = malloc(*n_rows * *n_cols * sizeof(double)); 141 | 142 | for (size_t i = 0; i < ((*n_rows) * (*n_cols)); i++) { 143 | sg_matrix[i] = pow(cabs(stft_matrix[i]), power); 144 | } 145 | fftw_free(stft_matrix); 146 | 147 | return sg_matrix; 148 | } 149 | 150 | double hz_to_mel(double f) { 151 | double min_log_hz = 1000.0; 152 | 153 | if (f < min_log_hz) { 154 | return 3.0 * f / 200.0; 155 | } else { 156 | return 3.0 * (min_log_hz / 200.0) + log(f / min_log_hz) / (log(6.4) / 27.0); 157 | } 158 | } 159 | 160 | double mel_to_hz(double mel) { 161 | double min_log_hz = 1000.0; 162 | double min_log_mel = 3.0 * min_log_hz / 200.0; 163 | 164 | if (mel < min_log_mel) { 165 | return 200 * mel / 3.0 ; 166 | } else { 167 | return min_log_hz * exp((log(6.4) / 27.0) * (mel - min_log_mel)); 168 | } 169 | } 170 | 171 | // Return center frequencies for mel bands. Note that we use Slaney method for 172 | // all of the work with mels which is the default in librosa. 173 | double* mel_frequencies(size_t n_mels, double fmin, double fmax) { 174 | double min_mel = hz_to_mel(fmin); 175 | double max_mel = hz_to_mel(fmax); 176 | 177 | double *freqs = linspace(min_mel, max_mel, n_mels); 178 | 179 | for (size_t i = 0; i < n_mels; i++) { 180 | freqs[i] = mel_to_hz(freqs[i]); 181 | } 182 | 183 | return freqs; 184 | } 185 | 186 | double* fft_frequencies(size_t sr, size_t n_fft) { 187 | return linspace(0, (double)sr / 2, 1 + floor(n_fft / 2)); 188 | } 189 | 190 | // Return a mel filterbank matrix. Most of the explicitly unmentioned parameters 191 | // default to the values set in librosa. 192 | double* mel_filter(size_t sr, size_t n_fft, size_t n_mels) { 193 | size_t n_rows = n_mels; 194 | size_t n_cols = 1 + floor(n_fft / 2); 195 | 196 | double fmin = 0.0; 197 | double fmax = (double)sr / 2; 198 | double *weights = calloc(n_rows * n_cols, sizeof(double)); 199 | 200 | double *fft_fs = fft_frequencies(sr, n_fft); 201 | double *mel_fs = mel_frequencies(n_mels + 2, fmin, fmax); 202 | 203 | double norm_multiplier; 204 | for (size_t i = 0; i < n_rows; i++) { 205 | norm_multiplier = 2.0 / (mel_fs[i + 2] - mel_fs[i]); 206 | 207 | for (size_t j = 0; j < n_cols; j++) { 208 | if (fft_fs[j] >= mel_fs[i]) { 209 | if (fft_fs[j] <= mel_fs[i + 1]) { 210 | // Rise 211 | weights[j + (i * n_cols)] = norm_multiplier * 212 | (fft_fs[j] - mel_fs[i]) / 213 | (mel_fs[i + 1] - mel_fs[i]); 214 | } else if (fft_fs[j] <= mel_fs[i + 2]) { 215 | // Fall 216 | weights[j + (i * n_cols)] = norm_multiplier * 217 | (mel_fs[i + 2] - fft_fs[j]) / 218 | (mel_fs[i + 2] - mel_fs[i + 1]); 219 | } 220 | } 221 | } 222 | } 223 | 224 | free(fft_fs); 225 | free(mel_fs); 226 | return weights; 227 | } 228 | 229 | // NOTE: For most of the neural network based models, we might just stop here 230 | // and won't do the log + dct to get MFCC 231 | double* melspectrogram(double* samples, size_t n_samples, size_t sr, size_t n_fft, 232 | size_t hop_length, size_t n_mels, size_t *n_cols) { 233 | // NOTE: Default power is 2 234 | size_t n_rows; 235 | double* sg_matrix = spectrogram(samples, n_samples, n_fft, hop_length, 2, &n_rows, n_cols); 236 | double* filterbank = mel_filter(sr, n_fft, n_mels); 237 | 238 | double* msg_matrix = calloc(n_mels * (*n_cols), sizeof(double)); 239 | cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, 240 | n_mels, (*n_cols), n_rows, 241 | 1.0, filterbank, n_rows, sg_matrix, (*n_cols), 242 | 0.0, msg_matrix, (*n_cols)); 243 | free(filterbank); 244 | free(sg_matrix); 245 | return msg_matrix; 246 | } 247 | -------------------------------------------------------------------------------- /esi-dictate.el: -------------------------------------------------------------------------------- 1 | ;;; esi-dictate.el --- Dictation with Real-Time Editing -*- lexical-binding: t; -*- 2 | 3 | ;; Copyright (c) 2024 Abhinav Tushar 4 | 5 | ;; Author: Abhinav Tushar 6 | ;; Version: 0.3.0 7 | ;; Package-Requires: ((emacs "29") (llm "0.17.2")) 8 | ;; Keywords: speech 9 | ;; URL: https://github.com/lepisma/emacs-speech-input 10 | 11 | ;;; Commentary: 12 | 13 | ;; Dictation with Real-Time Editing 14 | ;; This file is not a part of GNU Emacs. 15 | 16 | ;;; License: 17 | 18 | ;; This program is free software: you can redistribute it and/or modify 19 | ;; it under the terms of the GNU General Public License as published by 20 | ;; the Free Software Foundation, either version 3 of the License, or 21 | ;; (at your option) any later version. 22 | 23 | ;; This program is distributed in the hope that it will be useful, 24 | ;; but WITHOUT ANY WARRANTY; without even the implied warranty of 25 | ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 26 | ;; GNU General Public License for more details. 27 | 28 | ;; You should have received a copy of the GNU General Public License 29 | ;; along with this program. If not, see . 30 | 31 | ;;; Code: 32 | 33 | (require 'json) 34 | (require 'llm) 35 | (require 'llm-openai) 36 | 37 | 38 | (defcustom esi-dictate-dg-api-key nil 39 | "API Key for Deepgram." 40 | :type 'string) 41 | 42 | (defcustom esi-dictate-llm-provider nil 43 | "LLM provider to use for corrections") 44 | 45 | (defcustom esi-dictate-speech-final-hook nil 46 | "Hook to keep functions that run once the speech utterance is 47 | finalized from the ASR." 48 | :type ':hook) 49 | 50 | (defcustom esi-dictate-cursor "⤳" 51 | "Symbol for voice cursor." 52 | :type 'string) 53 | 54 | (defvar esi-dictate--dg-process nil 55 | "Process holding the deepgram script") 56 | 57 | (defcustom esi-dictate-llm-prompt "You are a dictation assistant, you will be given transcript by the user with speech disfluencies, minor mistakes, and edits and you have to return a corrected transcript. The user might give you their stream of consciousness and you have to ensure that you correctly identify a request to edit and don't misfire. You don't have to generate any new information, just ensure fixes in spoken transcripts and edits as asked." 58 | "System prompt for the LLM editor." 59 | :type 'string) 60 | 61 | (defcustom esi-dictate-fix-examples (list (cons "I wan to write about umm something related to food. My name is name is Abhinav" 62 | "I want to write about umm something related to food. My name is Abhinav.") 63 | (cons "Okay we will start. Let's write something about chairs. No not chairs, make it tables." 64 | "Let's write something about tables.") 65 | (cons "I want to write something that's difficult to transcribe and then try correcting that. Write my name as abcd. No separate the letters with . please" 66 | "I want to write something that's difficult to transcribe and then try correcting that. Write my name as a.b.c.d.") 67 | (cons "hi easy, what are you doing? It's e s i." 68 | "hi esi, what are you doing?")) 69 | "Example inputs and outputs for few shot learning of auto 70 | edits. Change this to impact the behaviour of dictation 71 | intelligence." 72 | :type '(repeat (cons string string))) 73 | 74 | (defface esi-dictate-intermittent-face 75 | '((t (:inherit font-lock-comment-face))) 76 | "Face for transcription that's intermittent from ASR and could 77 | change later.") 78 | 79 | (defface esi-dictate-context-face 80 | '((t (:inherit link))) 81 | "Face for marking region of text that's under consideration as 82 | voice context (used in context-overlay). This is the region 83 | that's send to the LLM for edits.") 84 | 85 | (defface esi-dictate-cursor-face 86 | '((t (:inherit default))) 87 | "Face to use for the `esi-dictate-cursor'.") 88 | 89 | (defvar esi-dictate-mode-map 90 | (make-sparse-keymap) 91 | "Keymap for `esi-dictate-mode'.") 92 | 93 | (defvar-local esi-dictate-context-overlay nil 94 | "Overlay that specifies the region to be used as context 95 | for corrections. The end position of this is also the position 96 | where insertion happens from the ASR transcription. We also use 97 | this to track position of the context.") 98 | 99 | (define-minor-mode esi-dictate-mode 100 | "Toggle esi-dictate mode." 101 | :init-value nil 102 | :keymap esi-dictate-mode-map) 103 | 104 | (defun esi-dictate--write-fix (edited-content) 105 | "Make fixes to the voice context using the given `edited-content'." 106 | (let ((beg-pos (overlay-start esi-dictate-context-overlay)) 107 | (end-pos (overlay-end esi-dictate-context-overlay)) 108 | (past-point (point))) 109 | (delete-region beg-pos end-pos) 110 | (goto-char beg-pos) 111 | (insert edited-content) 112 | ;; We replicate save-excursion manually since we do full deletion and 113 | ;; replacement. 114 | (let ((current-point (point))) 115 | (if (<= past-point beg-pos) 116 | (goto-char past-point) 117 | (if (<= past-point end-pos) 118 | (goto-char (min past-point current-point)) 119 | (goto-char (+ current-point (- past-point end-pos))))) 120 | ;; Recover the overlay 121 | (move-overlay esi-dictate-context-overlay beg-pos current-point)))) 122 | 123 | (defun esi-dictate--call-llm (content) 124 | "Perform general fixes to given `content' assuming it's coming 125 | from dictation with speech disfluencies and other artifacts." 126 | (let ((prompt (make-llm-chat-prompt :context esi-dictate-llm-prompt 127 | :examples esi-dictate-fix-examples))) 128 | (llm-chat-prompt-append-response prompt content) 129 | (llm-chat-async esi-dictate-llm-provider prompt #'esi-dictate--write-fix 130 | (lambda (err err-message) (message "[esi] Error %s: %s" err err-message))))) 131 | 132 | (defun esi-dictate-fix-context () 133 | "Fix the context using the general transcription fixing 134 | instructions." 135 | (interactive) 136 | (let ((beg-pos (overlay-start esi-dictate-context-overlay)) 137 | (end-pos (overlay-end esi-dictate-context-overlay))) 138 | (esi-dictate--call-llm (buffer-substring-no-properties beg-pos end-pos)))) 139 | 140 | (defun esi-dictate--clear-process () 141 | (when esi-dictate--dg-process 142 | (delete-process esi-dictate--dg-process) 143 | (setq esi-dictate--dg-process nil))) 144 | 145 | (defun esi-dictate-make-context-overlay () 146 | "Make and return new context overlay." 147 | (let ((overlay (if (region-active-p) 148 | (make-overlay (region-beginning) (region-end) nil nil t) 149 | (make-overlay (point) (point) nil nil t)))) 150 | (overlay-put overlay 'face 'esi-dictate-context-face) 151 | (overlay-put overlay 'after-string (propertize esi-dictate-cursor 'face 'esi-dictate-cursor-face)) 152 | overlay)) 153 | 154 | (defun esi-dictate-clear-context-overlay () 155 | (when esi-dictate-context-overlay 156 | (delete-overlay esi-dictate-context-overlay)) 157 | (setq esi-dictate-context-overlay nil)) 158 | 159 | (defun esi-dictate-insert (transcription-item) 160 | "Insert transcription object in the current buffer preserving the 161 | semantics of intermittent results." 162 | (let* ((id (alist-get 'start transcription-item)) 163 | (text (alist-get 'transcript (aref (alist-get 'alternatives (alist-get 'channel transcription-item)) 0))) 164 | (prev-item (when (> (overlay-end esi-dictate-context-overlay) (point-min)) ;; Ensure the overlay isn't at (point-min) 165 | (get-text-property (- (overlay-end esi-dictate-context-overlay) 1) 'esi-dictate-transcription-item)))) 166 | ;; If previous item and current are the same utterance, delete the previous 167 | ;; item and then insert new one. This handles intermittent results from the 168 | ;; ASR. 169 | (when (and prev-item (= id (alist-get 'start prev-item))) 170 | (delete-region (get-text-property (- (overlay-end esi-dictate-context-overlay) 1) 'esi-dictate-start) (overlay-end esi-dictate-context-overlay))) 171 | 172 | (let ((insertion-pos (overlay-end esi-dictate-context-overlay))) 173 | (save-excursion 174 | (goto-char insertion-pos) 175 | (insert text " ")) 176 | (when (eq :false (alist-get 'is_final transcription-item)) 177 | (overlay-put (make-overlay insertion-pos (overlay-end esi-dictate-context-overlay)) 'face 'esi-dictate-intermittent-face)) 178 | 179 | ;; Saving properties which will be read later to handle intermittent 180 | ;; results. 181 | (put-text-property insertion-pos (overlay-end esi-dictate-context-overlay) 'esi-dictate-transcription-item transcription-item) 182 | (put-text-property insertion-pos (overlay-end esi-dictate-context-overlay) 'esi-dictate-start insertion-pos) 183 | 184 | ;; This is utterance end according to the ASR. In this case, we run a 185 | ;; few hooks. 186 | (when (not (eq :false (alist-get 'speech_final transcription-item))) 187 | (run-hooks 'esi-dictate-speech-final-hook))))) 188 | 189 | (defun esi-dictate-filter-fn (process string) 190 | "Filter function to read the output from python script that 191 | interacts with Deeepgram." 192 | (let ((existing (or (process-get process 'accumulated-output) ""))) 193 | (setq existing (concat existing string)) 194 | (while (string-match "\n" existing) 195 | (let ((line (substring existing 0 (match-beginning 0))) 196 | (rest (substring existing (match-end 0)))) 197 | (setq existing rest) 198 | (cond ((string-prefix-p "Output: " line) 199 | (let ((json-string (substring line (length "Output: ")))) 200 | (esi-dictate-insert (json-parse-string json-string :object-type 'alist)))) 201 | ((string-prefix-p "Press Enter to stop recording" line) 202 | (message "[esi] Dictation mode ready to use.") 203 | (esi-dictate-move-here))))) 204 | (process-put process 'accumulated-output existing))) 205 | 206 | (defun esi-dictate-move-here () 207 | "Move the voice cursor to the current point or, if active, the 208 | current region." 209 | (interactive) 210 | (esi-dictate-clear-context-overlay) 211 | (setq esi-dictate-context-overlay (esi-dictate-make-context-overlay))) 212 | 213 | ;;;###autoload 214 | (defun esi-dictate-start () 215 | "Start the real-time transcription process to start inserting text 216 | in current buffer." 217 | (interactive) 218 | (esi-dictate--clear-process) 219 | (setq esi-dictate--dg-process 220 | (let ((process-environment (cons (format "DG_API_KEY=%s" esi-dictate-dg-api-key) process-environment))) 221 | (make-process :name "esi-dictate-dg" 222 | :buffer "*esi-dictate-dg*" 223 | :command (list "dg.py") 224 | :filter #'esi-dictate-filter-fn))) 225 | (esi-dictate-mode) 226 | (message "[esi] Starting dictation mode ...")) 227 | 228 | (defun esi-dictate-stop () 229 | (interactive) 230 | (esi-dictate--clear-process) 231 | (esi-dictate-mode -1) 232 | (esi-dictate-clear-context-overlay) 233 | (message "[esi] Stopped dictation mode.")) 234 | 235 | (provide 'esi-dictate) 236 | 237 | ;;; esi-dictate.el ends here 238 | -------------------------------------------------------------------------------- /src/esi-io.c: -------------------------------------------------------------------------------- 1 | #include "sndfile.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "esi-io.h" 12 | 13 | typedef struct { 14 | sf_count_t offset, length; 15 | char *data; 16 | } VIO_DATA; 17 | 18 | sf_count_t vfget_filelen(void *user_data) { 19 | VIO_DATA *vf = (VIO_DATA *)user_data; 20 | return vf->length; 21 | } 22 | 23 | sf_count_t vfseek(sf_count_t offset, int whence, void *user_data) { 24 | VIO_DATA *vf = (VIO_DATA *)user_data; 25 | 26 | switch (whence) { 27 | case SEEK_SET: 28 | vf->offset = offset; 29 | break; 30 | 31 | case SEEK_CUR: 32 | vf->offset = vf->offset + offset; 33 | break; 34 | 35 | case SEEK_END: 36 | vf->offset = vf->length + offset; 37 | break; 38 | 39 | default: 40 | break; 41 | }; 42 | 43 | return vf->offset; 44 | } 45 | 46 | sf_count_t vfread(void *ptr, sf_count_t count, void *user_data) { 47 | VIO_DATA *vf = (VIO_DATA *)user_data; 48 | 49 | if (vf->offset + count > vf->length) 50 | count = vf->length - vf->offset; 51 | 52 | memcpy(ptr, vf->data + vf->offset, count); 53 | vf->offset += count; 54 | 55 | return count; 56 | } 57 | 58 | sf_count_t vftell(void *user_data) { 59 | VIO_DATA *vf = (VIO_DATA *)user_data; 60 | return vf->offset; 61 | } 62 | 63 | double* samples_from_buffer(char* buffer, size_t buffer_size, size_t* n_samples) { 64 | VIO_DATA vio_data; 65 | SF_VIRTUAL_IO vio; 66 | 67 | vio.get_filelen = vfget_filelen; 68 | vio.seek = vfseek; 69 | vio.read = vfread; 70 | vio.tell = vftell; 71 | 72 | vio_data.offset = 0; 73 | vio_data.length = buffer_size - 1; 74 | vio_data.data = buffer; 75 | 76 | SNDFILE *sfile; 77 | SF_INFO file_info; 78 | 79 | sfile = sf_open_virtual(&vio, SFM_READ, &file_info, &vio_data); 80 | 81 | sf_count_t frames_count = file_info.frames * file_info.channels; 82 | double *frames = malloc(sizeof(double) * frames_count); 83 | sf_readf_double(sfile, frames, frames_count); 84 | 85 | // KLUDGE: This is for picking up first channel data. There is too much 86 | // copying happening overall. 87 | double *samples = malloc(sizeof(double) * file_info.frames); 88 | for (size_t i = 0; i < file_info.frames; i++) { 89 | samples[i] = frames[i * file_info.channels]; 90 | } 91 | 92 | *n_samples = file_info.frames; 93 | 94 | sf_close(sfile); 95 | free(frames); 96 | return samples; 97 | } 98 | 99 | static int min_int(int a, int b) { 100 | return (a < b) ? a : b; 101 | } 102 | 103 | // Custom ring buffer. I might be wiser in using sio's implementation but there 104 | // I need to keep read pointer n-steps ahead and stuff to make it behave how I 105 | // want it to. 106 | struct buffer { 107 | char* array; 108 | size_t capacity; 109 | // Margin is the difference between write pointer and read pointer. If the 110 | // difference is sufficient (considering live audio io use case), they won't 111 | // step on each other. 112 | size_t margin; 113 | size_t write_offset; 114 | size_t read_offset; 115 | }; 116 | 117 | // Buffer chained in a linked list like manner 118 | struct chain_buffer { 119 | char* array; 120 | // Capacity tells total bytes which can be kept in this buffer 121 | size_t capacity; 122 | // Length is the filled size of buffer. 123 | size_t length; 124 | // Next points to either NULL (end of chain) or next buffer to jump for 125 | // putting more data. 126 | struct chain_buffer* next; 127 | }; 128 | 129 | // Create a new chain buffer of given size without a child. 130 | struct chain_buffer* chain_buffer_init(size_t capacity) { 131 | struct chain_buffer* buf = malloc(sizeof(struct chain_buffer)); 132 | buf->capacity = capacity; 133 | buf->length = 0; 134 | buf->array = calloc(capacity, sizeof(char)); 135 | buf->next = NULL; 136 | 137 | return buf; 138 | } 139 | 140 | // Extend buffer with another buffer of given capacity. Return the new child 141 | // buffer. 142 | struct chain_buffer* chain_buffer_extend(struct chain_buffer* buffer, size_t capacity) { 143 | struct chain_buffer* child_buffer = chain_buffer_init(capacity); 144 | buffer->next = child_buffer; 145 | return child_buffer; 146 | } 147 | 148 | // Read the size of all arrays (concatenated) kept in the buffer 149 | size_t chain_buffer_filled_length(struct chain_buffer* buffer) { 150 | size_t length = 0; 151 | 152 | struct chain_buffer* current = buffer; 153 | while (current) { 154 | length = length + current->length; 155 | current = buffer->next; 156 | } 157 | 158 | return length; 159 | } 160 | 161 | // Read the complete chain buffer and return a concatenated char array 162 | char* chain_buffer_read(struct chain_buffer* buffer) { 163 | char* array = malloc(sizeof(char) * chain_buffer_filled_length(buffer)); 164 | 165 | size_t position = 0; 166 | struct chain_buffer* current_buffer = buffer; 167 | size_t current_length; 168 | 169 | while (current_buffer) { 170 | current_length = current_buffer->length; 171 | for (size_t i = 0; i < current_length; i++) { 172 | array[position + i] = current_buffer->array[i]; 173 | } 174 | position = position + current_length; 175 | current_buffer = current_buffer->next; 176 | } 177 | return array; 178 | } 179 | 180 | // Destroy the buffer starting from the first item in chain. 181 | void chain_buffer_destroy(struct chain_buffer* buffer) { 182 | struct chain_buffer* current; 183 | struct chain_buffer* next = buffer; 184 | 185 | while (next) { 186 | current = next; 187 | next = current->next; 188 | 189 | free(current->array); 190 | free(current); 191 | } 192 | } 193 | 194 | struct buffer* buffer_init(size_t capacity, size_t margin) { 195 | struct buffer* buf = malloc(sizeof(struct buffer)); 196 | buf->capacity = capacity; 197 | buf->margin = margin; 198 | buf->array = calloc(capacity + margin, sizeof(char)); 199 | 200 | buf->write_offset = 0; 201 | buf->read_offset = buf->write_offset + buf->margin; 202 | 203 | return buf; 204 | } 205 | 206 | void buffer_destroy(struct buffer* buf) { 207 | free(buf->array); 208 | free(buf); 209 | } 210 | 211 | // Write data_len bytes from data in buffer. 212 | bool buffer_write(struct buffer* buf, char* data, size_t data_len) { 213 | size_t input_offset = 0; 214 | if (data_len > buf->capacity) { 215 | fprintf(stderr, "Data more than the capacity, taking the last chunks provided"); 216 | input_offset = (data_len - buf->capacity); 217 | data_len = buf->capacity; 218 | } 219 | 220 | size_t buffer_length = buf->capacity + buf->margin; 221 | for (size_t i = 0; i < data_len; i++) { 222 | buf->array[(buf->write_offset + i) % buffer_length] = data[input_offset + i]; 223 | } 224 | 225 | buf->write_offset = (buf->write_offset + data_len) % buffer_length; 226 | buf->read_offset = (buf->write_offset + buf->margin) % buffer_length; 227 | return true; 228 | } 229 | 230 | // Read upto a max of capacity chars from buffer. 231 | char* buffer_read(struct buffer* buf, size_t *output_size) { 232 | char* output = malloc(sizeof(char) * buf->capacity); 233 | 234 | size_t offset = buf->read_offset; 235 | size_t buffer_length = buf->capacity + buf->margin; 236 | 237 | for (size_t i = 0; i < buf->capacity; i++) { 238 | output[i] = buf->array[(offset + i) % buffer_length]; 239 | } 240 | 241 | *output_size = buf->capacity; 242 | return output; 243 | } 244 | 245 | struct RecordContext { 246 | struct buffer *buf; 247 | struct SoundIo *soundio; 248 | struct SoundIoDevice *selected_device; 249 | pthread_t *poll_thread; 250 | bool keep_recording; 251 | }; 252 | 253 | void recording_read_callback(struct SoundIoInStream *instream, int frame_count_min, int frame_count_max) { 254 | struct RecordContext *rc = instream->userdata; 255 | struct SoundIoChannelArea *areas; 256 | int err; 257 | 258 | size_t free_count = rc->buf->capacity / instream->bytes_per_frame; 259 | 260 | int write_frames = min_int(free_count, frame_count_max); 261 | int frames_left = write_frames; 262 | 263 | while (true) { 264 | int frame_count = frames_left; 265 | 266 | if ((err = soundio_instream_begin_read(instream, &areas, &frame_count))) { 267 | fprintf(stderr, "begin read error: %s", soundio_strerror(err)); 268 | exit(1); 269 | } 270 | 271 | if (!frame_count) 272 | break; 273 | 274 | if (!areas) { 275 | fprintf(stderr, "are error thing\n"); 276 | } else { 277 | for (int frame = 0; frame < frame_count; frame += 1) { 278 | for (int ch = 0; ch < instream->layout.channel_count; ch += 1) { 279 | buffer_write(rc->buf, areas[ch].ptr, instream->bytes_per_sample); 280 | areas[ch].ptr += areas[ch].step; 281 | } 282 | } 283 | } 284 | 285 | if ((err = soundio_instream_end_read(instream))) { 286 | fprintf(stderr, "end read error: %s", soundio_strerror(err)); 287 | exit(1); 288 | } 289 | 290 | frames_left -= frame_count; 291 | if (frames_left <= 0) 292 | break; 293 | } 294 | } 295 | 296 | void recording_overflow_callback(struct SoundIoInStream *instream) { 297 | static int count = 0; 298 | fprintf(stderr, "overflow %d\n", ++count); 299 | } 300 | 301 | void *recording_poll_fn(void *arg) { 302 | struct SoundIoInStream *instream = (struct SoundIoInStream*)arg; 303 | struct RecordContext *rc = instream->userdata; 304 | 305 | struct timespec tim, tim_e; 306 | tim.tv_sec = 0; 307 | tim.tv_nsec = 100000000L; // 100 ms 308 | 309 | while (rc->keep_recording) { 310 | soundio_flush_events(rc->soundio); 311 | nanosleep(&tim, &tim_e); 312 | } 313 | 314 | soundio_instream_destroy(instream); 315 | soundio_device_unref(rc->selected_device); 316 | soundio_destroy(rc->soundio); 317 | buffer_destroy(rc->buf); 318 | free(rc->poll_thread); 319 | free(rc); 320 | pthread_exit(NULL); 321 | } 322 | 323 | struct SoundIoInStream* start_background_recording(size_t sample_rate, size_t buffer_duration_seconds) { 324 | enum SoundIoBackend backend = SoundIoBackendPulseAudio; 325 | enum SoundIoFormat fmt = SoundIoFormatS16LE; 326 | struct RecordContext *rc = malloc(sizeof(struct RecordContext)); 327 | struct SoundIo *soundio = soundio_create(); 328 | int err; 329 | 330 | if (!soundio) { 331 | fprintf(stderr, "OOM while creating soundio\n"); 332 | return NULL; 333 | } 334 | 335 | if (soundio_connect_backend(soundio, backend)) { 336 | fprintf(stderr, "Error connecting: %s\n", soundio_strerror(err)); 337 | return NULL; 338 | } 339 | 340 | soundio_flush_events(soundio); 341 | 342 | struct SoundIoDevice *selected_device = NULL; 343 | 344 | int device_index = soundio_default_input_device_index(soundio); 345 | selected_device = soundio_get_input_device(soundio, device_index); 346 | 347 | if (!selected_device) { 348 | fprintf(stderr, "No input devices available\n"); 349 | return NULL; 350 | } 351 | 352 | if (selected_device->probe_error) { 353 | fprintf(stderr, "Unable to probe device: %s\n", soundio_strerror(selected_device->probe_error)); 354 | return NULL; 355 | } 356 | 357 | soundio_device_sort_channel_layouts(selected_device); 358 | 359 | struct SoundIoInStream *instream = soundio_instream_create(selected_device); 360 | 361 | if (!instream) { 362 | fprintf(stderr, "OOM while creating instream\n"); 363 | return NULL; 364 | } 365 | 366 | instream->format = fmt; 367 | instream->sample_rate = sample_rate; 368 | instream->read_callback = recording_read_callback; 369 | instream->overflow_callback = recording_overflow_callback; 370 | instream->userdata = rc; 371 | 372 | if ((err = soundio_instream_open(instream))) { 373 | fprintf(stderr, "Unable to open input stream: %s\n", soundio_strerror(err)); 374 | return NULL; 375 | } 376 | 377 | size_t buffer_capacity = buffer_duration_seconds * instream->sample_rate * instream->bytes_per_frame; 378 | 379 | // Assuming 32 bytes per sample and 5 second margin for a rough bound 380 | size_t margin = 5 * 32 * sample_rate; 381 | 382 | rc->buf = buffer_init(buffer_capacity, margin); 383 | rc->soundio = soundio; 384 | rc->selected_device = selected_device; 385 | rc->keep_recording = true; 386 | rc->poll_thread = malloc(sizeof(pthread_t)); 387 | 388 | if ((err = soundio_instream_start(instream))) { 389 | fprintf(stderr, "Unable to start input device: %s\n", soundio_strerror(err)); 390 | return NULL; 391 | } 392 | 393 | if (pthread_create(rc->poll_thread, NULL, recording_poll_fn, (void*)(instream))) { 394 | fprintf(stderr, "Error creating thread\n"); 395 | return NULL; 396 | } 397 | 398 | return instream; 399 | } 400 | 401 | char* read_background_recording(struct SoundIoInStream* instream, size_t* output_size) { 402 | struct RecordContext *rc = (struct RecordContext*)(instream->userdata); 403 | return buffer_read(rc->buf, output_size); 404 | } 405 | 406 | void stop_background_recording(struct SoundIoInStream *instream) { 407 | struct RecordContext *rc = instream->userdata; 408 | rc->keep_recording = false; 409 | pthread_join(*(rc->poll_thread), NULL); 410 | } 411 | -------------------------------------------------------------------------------- /src/esi-core.c: -------------------------------------------------------------------------------- 1 | #include "emacs-module.h" 2 | #include "sndfile.h" 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include "esi-io.h" 9 | #include "esi-prep.h" 10 | 11 | int plugin_is_GPL_compatible; 12 | const char *esi_core_version = "0.0.2"; 13 | 14 | static emacs_value Fesi_core_version(emacs_env *env, ptrdiff_t n, emacs_value args[], void *data) { 15 | return env->make_string(env, esi_core_version, strlen(esi_core_version)); 16 | } 17 | 18 | static emacs_value make_vector(emacs_env *env, int len, double init) { 19 | emacs_value Fmake_vector = env->intern(env, "make-vector"); 20 | emacs_value args[] = { env->make_integer(env, len), env->make_float(env, init) }; 21 | return env->funcall(env, Fmake_vector, 2, args); 22 | } 23 | 24 | // Return an elisp matrix from a given row major float matrix 25 | emacs_value make_matrix(emacs_env *env, double* matrix, size_t n_rows, size_t n_cols) { 26 | emacs_value output = make_vector(env, n_rows, 0); 27 | for (size_t i = 0; i < n_rows; i++) { 28 | emacs_value row = make_vector(env, n_cols, 0); 29 | for (size_t j = 0; j < n_cols; j++) { 30 | env->vec_set(env, row, j, env->make_float(env, matrix[(i * n_cols) + j])); 31 | } 32 | 33 | env->vec_set(env, output, i, row); 34 | } 35 | 36 | return output; 37 | } 38 | 39 | static emacs_value Fwav_to_samples(emacs_env *env, ptrdiff_t n, emacs_value args[], void *data) { 40 | ptrdiff_t buffer_size; 41 | env->copy_string_contents(env, args[0], NULL, &buffer_size); 42 | 43 | char *buffer = malloc(buffer_size); 44 | env->copy_string_contents(env, args[0], buffer, &buffer_size); 45 | 46 | sf_count_t n_samples; 47 | double *samples = samples_from_buffer(buffer, buffer_size, &n_samples); 48 | 49 | emacs_value vector = make_vector(env, n_samples, 0); 50 | for (size_t i = 0; i < n_samples; i++) { 51 | env->vec_set(env, vector, i, env->make_float(env, samples[i])); 52 | } 53 | 54 | free(samples); 55 | free(buffer); 56 | return vector; 57 | } 58 | 59 | // Does raw rfft using the following arguments: 60 | // - samples (a vector) 61 | static emacs_value Frfft(emacs_env *env, ptrdiff_t n, emacs_value args[], void *data) { 62 | size_t n_samples = env->vec_size(env, args[0]); 63 | 64 | double *samples = fftw_malloc(sizeof(double) * n_samples); 65 | for (size_t i = 0; i < n_samples; i++) { 66 | samples[i] = env->extract_float(env, env->vec_get(env, args[0], i)); 67 | } 68 | 69 | size_t n_output; 70 | fftw_complex *fft_output = rfft(samples, n_samples, &n_output); 71 | emacs_value vector = make_vector(env, n_output, 0); 72 | 73 | for (size_t i = 0; i < n_output; i++) { 74 | emacs_value complex_number = make_vector(env, 2, 0); 75 | env->vec_set(env, complex_number, 0, env->make_float(env, creal(fft_output[i]))); 76 | env->vec_set(env, complex_number, 1, env->make_float(env, cimag(fft_output[i]))); 77 | env->vec_set(env, vector, i, complex_number); 78 | } 79 | 80 | fftw_free(fft_output); 81 | fftw_free(samples); 82 | return vector; 83 | } 84 | 85 | // Create stft matrix (elisp vector based) using the following arguments: 86 | // - samples (a vector) 87 | // - n-fft 88 | // - hop-length 89 | static emacs_value Fstft(emacs_env *env, ptrdiff_t n, emacs_value args[], void *data) { 90 | size_t n_fft = env->extract_integer(env, args[1]); 91 | size_t hop_length = env->extract_integer(env, args[2]); 92 | 93 | size_t n_samples = env->vec_size(env, args[0]); 94 | double* samples = malloc(sizeof(double) * n_samples); 95 | for (size_t i = 0; i < n_samples; i++) { 96 | samples[i] = env->extract_float(env, env->vec_get(env, args[0], i)); 97 | } 98 | 99 | size_t n_rows, n_cols; 100 | fftw_complex* stft_matrix = stft(samples, n_samples, n_fft, hop_length, &n_rows, &n_cols); 101 | 102 | emacs_value matrix = make_vector(env, n_rows, 0); 103 | for (size_t i = 0; i < n_rows; i++) { 104 | emacs_value vector = make_vector(env, n_cols, 0); 105 | 106 | // stft_matrix is row major 107 | for (size_t j = 0; j < n_cols; j++) { 108 | emacs_value complex_number = make_vector(env, 2, 0); 109 | env->vec_set(env, complex_number, 0, env->make_float(env, creal(stft_matrix[(i * n_cols) + j]))); 110 | env->vec_set(env, complex_number, 1, env->make_float(env, cimag(stft_matrix[(i * n_cols) + j]))); 111 | env->vec_set(env, vector, j, complex_number); 112 | } 113 | 114 | env->vec_set(env, matrix, i, vector); 115 | } 116 | 117 | free(samples); 118 | free(stft_matrix); 119 | 120 | return matrix; 121 | } 122 | 123 | // Create spectrogram matrix (elisp vectors) using the following arguments: 124 | // - samples (a vector) 125 | // - n-fft 126 | // - hop-length 127 | // - power (defaults to 1) 128 | static emacs_value Fspectrogram(emacs_env *env, ptrdiff_t n, emacs_value args[], void *data) { 129 | size_t n_fft = env->extract_integer(env, args[1]); 130 | size_t hop_length = env->extract_integer(env, args[2]); 131 | size_t power = n == 3 ? 1 : env->extract_integer(env, args[3]); 132 | 133 | size_t n_samples = env->vec_size(env, args[0]); 134 | double* samples = malloc(sizeof(double) * n_samples); 135 | for (size_t i = 0; i < n_samples; i++) { 136 | samples[i] = env->extract_float(env, env->vec_get(env, args[0], i)); 137 | } 138 | 139 | size_t n_rows, n_cols; 140 | double* sg_matrix = spectrogram(samples, n_samples, n_fft, hop_length, power, &n_rows, &n_cols); 141 | emacs_value matrix = make_matrix(env, sg_matrix, n_rows, n_cols); 142 | free(samples); 143 | free(sg_matrix); 144 | return matrix; 145 | } 146 | 147 | // Create mel-spectrogram matrix using the following arguments: 148 | // - samples (a vector) 149 | // - sample rate 150 | // - n-fft 151 | // - hop-length 152 | // - n-mels 153 | static emacs_value Fmel_spectrogram(emacs_env *env, ptrdiff_t n, emacs_value args[], void *data) { 154 | size_t sr = env->extract_integer(env, args[1]); 155 | size_t n_fft = env->extract_integer(env, args[2]); 156 | size_t hop_length = env->extract_integer(env, args[3]); 157 | size_t n_mels = env->extract_integer(env, args[4]); 158 | 159 | size_t n_samples = env->vec_size(env, args[0]); 160 | double *samples = malloc(sizeof(double) * n_samples); 161 | for (size_t i = 0; i < n_samples; i++) { 162 | samples[i] = env->extract_float(env, env->vec_get(env, args[0], i)); 163 | } 164 | 165 | size_t n_cols; 166 | double *msg_matrix = melspectrogram(samples, n_samples, sr, n_fft, hop_length, n_mels, &n_cols); 167 | emacs_value matrix = make_matrix(env, msg_matrix, n_mels, n_cols); 168 | free(samples); 169 | free(msg_matrix); 170 | return matrix; 171 | } 172 | 173 | // Create mel filterbank matrix (elisp vectors) using the following arguments: 174 | // - sample rate 175 | // - n-fft 176 | // - n-mels 177 | static emacs_value Fmel_filter(emacs_env *env, ptrdiff_t n, emacs_value args[], void *data) { 178 | size_t sr = env->extract_integer(env, args[0]); 179 | size_t n_fft = env->extract_integer(env, args[1]); 180 | size_t n_mels = env->extract_integer(env, args[2]); 181 | 182 | double* filterbank = mel_filter(sr, n_fft, n_mels); 183 | emacs_value matrix = make_matrix(env, filterbank, n_mels, 1 + floor(n_fft / 2)); 184 | free(filterbank); 185 | return matrix; 186 | } 187 | 188 | static void fin_instream(void *instream_ut) { 189 | struct SoundIoInStream *instream = (struct SoundIoInStream*)instream_ut; 190 | stop_background_recording(instream); 191 | } 192 | 193 | // Start background audio recording with a max buffer limit. Return 1 in case of error. 194 | // Arguments: 195 | // - sample-rate 196 | // - buffer-duration (in seconds) 197 | static emacs_value Fstart_background_recording(emacs_env *env, ptrdiff_t n, emacs_value args[], void *data) { 198 | size_t sr = env->extract_integer(env, args[0]); 199 | size_t buffer_duration_seconds = env->extract_integer(env, args[1]); 200 | 201 | struct SoundIoInStream *instream = start_background_recording(sr, buffer_duration_seconds); 202 | 203 | if (instream) { 204 | return env->make_user_ptr(env, fin_instream, instream); 205 | } else { 206 | emacs_value nil = env->intern(env, "nil"); 207 | env->non_local_exit_signal(env, env->intern(env, "audio-input-stream-error"), nil); 208 | return nil; 209 | } 210 | } 211 | 212 | // Return current value of provided recording context 213 | static emacs_value Fread_background_recording_buffer(emacs_env *env, ptrdiff_t n, emacs_value args[], void *data) { 214 | struct SoundIoInStream *instream = (struct SoundIoInStream*)env->get_user_ptr(env, args[0]); 215 | 216 | size_t output_size; 217 | char* output = read_background_recording(instream, &output_size); 218 | 219 | return env->make_string(env, output, output_size); 220 | } 221 | 222 | // Stop the ongoing recording altogether and return buffer 223 | static emacs_value Fstop_background_recording(emacs_env *env, ptrdiff_t n, emacs_value args[], void *data) { 224 | struct SoundIoInStream *instream = (struct SoundIoInStream*)(env->get_user_ptr(env, args[0])); 225 | 226 | emacs_value nil = env->intern(env, "nil"); 227 | if (!instream) { 228 | // We put NULL in the value once a user pointer is finished 229 | env->non_local_exit_signal(env, env->intern(env, "audio-input-stream-finished"), nil); 230 | return nil; 231 | } 232 | 233 | // Check if we have valid user pointer value 234 | if (env->non_local_exit_check(env) == emacs_funcall_exit_return) { 235 | stop_background_recording(instream); 236 | env->set_user_ptr(env, args[0], NULL); 237 | env->set_user_finalizer(env, args[0], NULL); 238 | } 239 | 240 | return nil; 241 | } 242 | 243 | static void provide(emacs_env *env, const char *feature) { 244 | emacs_value Qfeat = env->intern(env, feature); 245 | emacs_value Qprovide = env->intern(env, "provide"); 246 | emacs_value args[] = {Qfeat}; 247 | 248 | env->funcall(env, Qprovide, 1, args); 249 | } 250 | 251 | static void bind_function(emacs_env *env, const char *name, emacs_value Sfun) { 252 | emacs_value Qfset = env->intern(env, "fset"); 253 | emacs_value Qsym = env->intern(env, name); 254 | emacs_value args[] = {Qsym, Sfun}; 255 | env->funcall(env, Qfset, 2, args); 256 | } 257 | 258 | int emacs_module_init(struct emacs_runtime *ert) { 259 | emacs_env *env = ert->get_environment(ert); 260 | 261 | emacs_value version_fn = env->make_function(env, 0, 0, Fesi_core_version, "Return version of esi-core.", NULL); 262 | bind_function(env, "esi-core-version", version_fn); 263 | 264 | emacs_value sample_fn = env->make_function(env, 1, 1, Fwav_to_samples, 265 | "Read and return vector of samples from wav bytes.\n\n" 266 | "\(fn wav-bytes)", 267 | NULL); 268 | bind_function(env, "esi-core-wav-to-samples", sample_fn); 269 | 270 | emacs_value rfft_fn = env->make_function(env, 1, 1, 271 | Frfft, 272 | "Calculate fft for given samples.\n\n" 273 | "\(fn samples-vector)", 274 | NULL); 275 | bind_function(env, "esi-core--rfft", rfft_fn); 276 | 277 | emacs_value stft_fn = env->make_function(env, 3, 3, 278 | Fstft, 279 | "Calculate stft for given samples.\n\n" 280 | "\(fn samples-vector n-fft hop-length)", 281 | NULL); 282 | bind_function(env, "esi-core--stft", stft_fn); 283 | 284 | emacs_value spectrogram_fn = env->make_function(env, 3, 4, 285 | Fspectrogram, 286 | "Calculate spectrogram for given samples.\n\n" 287 | "\(fn samples-vector n-fft hop-length &optional power)", 288 | NULL); 289 | bind_function(env, "esi-core--spectrogram", spectrogram_fn); 290 | 291 | emacs_value melfb_fn = env->make_function(env, 3, 3, 292 | Fmel_filter, 293 | "Calculate mel filterbank matrix.\n\n" 294 | "\(fn sample-rate n-fft n-mels)", 295 | NULL); 296 | bind_function(env, "esi-core--mel-filter", melfb_fn); 297 | 298 | emacs_value mel_spectrogram_fn = env->make_function(env, 5, 5, 299 | Fmel_spectrogram, 300 | "Calculate mel-spectrogram for given samples.\n\n" 301 | "\(fn samples-vector sample-rate n-fft hop-length n-mels)", 302 | NULL); 303 | bind_function(env, "esi-core--mel-spectrogram", mel_spectrogram_fn); 304 | 305 | emacs_value start_background_recording_fn = 306 | env->make_function(env, 2, 2, Fstart_background_recording, 307 | "Start background audio recording with given sample-rate and " 308 | "buffer duration in seconds. Return a user pointer.\n\n" 309 | "\(fn sample-rate buffer-duration)", 310 | NULL); 311 | 312 | bind_function(env, "esi-core--start-background-recording", 313 | start_background_recording_fn); 314 | 315 | emacs_value read_background_recording_buffer_fn = 316 | env->make_function(env, 1, 1, Fread_background_recording_buffer, 317 | "Return background audio recording buffer.\n\n" 318 | "\(fn recording-pointer)", NULL); 319 | 320 | bind_function(env, "esi-core--read-background-recording-buffer", 321 | read_background_recording_buffer_fn); 322 | 323 | emacs_value stop_background_recording_fn = 324 | env->make_function(env, 1, 1, Fstop_background_recording, 325 | "Stop background recording from pointer and return buffer.\n\n" 326 | "\(fn recording-pointer)", NULL); 327 | 328 | bind_function(env, "esi-core--stop-background-recording", 329 | stop_background_recording_fn); 330 | 331 | provide(env, "esi-core"); 332 | 333 | return 0; 334 | } 335 | -------------------------------------------------------------------------------- /tests/resources/hello.mel.spectrogram: -------------------------------------------------------------------------------- 1 | 3.439251708984375000e+01 1.804309654235839844e+01 5.645046234130859375e+00 1.561136126518249512e+00 4.827479720115661621e-01 4.068372845649719238e-01 5.594478845596313477e-01 8.240771889686584473e-01 1.023038864135742188e+00 7.924012541770935059e-01 6.440242528915405273e-01 5.937722325325012207e-01 4.389025866985321045e-01 4.483480751514434814e-01 3.671184778213500977e-01 3.435108661651611328e-01 4.018995761871337891e-01 3.467756509780883789e-01 3.723289072513580322e-01 5.214526057243347168e-01 6.614263057708740234e-01 3.557906150817871094e-01 1.756475120782852173e-01 2.494426071643829346e-01 2.930069565773010254e-01 2.409975826740264893e-01 1.868727356195449829e-01 1.334373205900192261e-01 1.116326004266738892e-01 1.652508527040481567e-01 1.419057846069335938e-01 1.643251031637191772e-01 2 | 6.309641361236572266e+00 1.762821912765502930e+00 6.978066563606262207e-01 3.827682137489318848e-01 1.060020029544830322e-01 1.052990332245826721e-01 1.623067557811737061e-01 6.184348464012145996e-01 7.821378111839294434e-01 3.210194110870361328e-01 1.088057309389114380e-01 1.070161908864974976e-01 2.239277660846710205e-01 4.338180124759674072e-01 2.095194309949874878e-01 1.204700917005538940e-01 1.411818712949752808e-01 1.675085574388504028e-01 1.817067414522171021e-01 1.708687394857406616e-01 1.388575732707977295e-01 7.205447554588317871e-02 4.275133088231086731e-02 1.349575370550155640e-01 1.807196736335754395e-01 1.413396596908569336e-01 1.587326377630233765e-01 1.549231112003326416e-01 1.613015979528427124e-01 1.500336825847625732e-01 1.142993941903114319e-01 9.031657129526138306e-02 3 | 4.893181324005126953e+00 1.292916655540466309e+00 2.946732342243194580e-01 1.185974106192588806e-01 2.405166439712047577e-02 3.225270286202430725e-02 5.385027825832366943e-02 1.878098547458648682e-01 1.423628628253936768e-01 4.769732803106307983e-02 5.407365038990974426e-02 4.310068115592002869e-02 1.163459941744804382e-01 1.995391547679901123e-01 7.720996439456939697e-02 5.117984116077423096e-02 5.167297646403312683e-02 8.700084686279296875e-02 1.136363297700881958e-01 3.845983743667602539e-02 3.116990253329277039e-02 4.616932570934295654e-02 4.699125885963439941e-02 2.850172854959964752e-02 3.948272019624710083e-02 6.747160106897354126e-02 4.538840800523757935e-02 5.001254379749298096e-02 5.915036052465438843e-02 6.402619928121566772e-02 6.097524240612983704e-02 5.153741687536239624e-02 4 | 3.782720804214477539e+00 1.077884078025817871e+00 6.171754598617553711e-01 2.142553180456161499e-01 2.072032727301120758e-02 2.232601121068000793e-02 2.882255241274833679e-02 8.770949393510818481e-02 7.601649314165115356e-02 2.158825099468231201e-02 2.901176922023296356e-02 3.130067139863967896e-02 7.202865928411483765e-02 9.183944761753082275e-02 3.463295847177505493e-02 3.458001092076301575e-02 1.791079901158809662e-02 6.178459897637367249e-02 1.296606212854385376e-01 4.117399454116821289e-02 1.163289602845907211e-02 2.247531153261661530e-02 3.104909881949424744e-02 2.225807309150695801e-02 3.153211995959281921e-02 3.683946281671524048e-02 2.966540120542049408e-02 3.400852158665657043e-02 3.092395514249801636e-02 2.850236743688583374e-02 1.496572978794574738e-02 2.510307729244232178e-02 5 | 2.685598373413085938e+00 7.725996375083923340e-01 4.213788509368896484e-01 1.497289389371871948e-01 1.676786504685878754e-02 2.175620757043361664e-02 2.015575766563415527e-02 9.237247705459594727e-02 7.827385514974594116e-02 4.646327719092369080e-02 4.648710414767265320e-02 3.051832504570484161e-02 5.231612548232078552e-02 6.578724086284637451e-02 5.028230696916580200e-02 3.589135780930519104e-02 1.948761194944381714e-02 7.379828393459320068e-02 1.508209258317947388e-01 5.146658793091773987e-02 2.612802200019359589e-02 2.030064538121223450e-02 2.492080628871917725e-02 3.098844736814498901e-02 1.711910590529441833e-02 1.688376255333423615e-02 2.143125236034393311e-02 3.150585293769836426e-02 3.668808564543724060e-02 3.257139027118682861e-02 1.031386665999889374e-02 1.244937255978584290e-02 6 | 1.186117291450500488e+00 3.683956563472747803e-01 2.604634463787078857e-01 9.619820863008499146e-02 1.508926693350076675e-02 1.580971665680408478e-02 1.593036949634552002e-02 9.585988521575927734e-02 8.418173342943191528e-02 2.705175988376140594e-02 2.409001067280769348e-02 1.705617085099220276e-02 7.432562857866287231e-02 1.112362965941429138e-01 3.721162676811218262e-02 2.149909362196922302e-02 1.531195454299449921e-02 7.083150744438171387e-02 1.375419795513153076e-01 4.041440039873123169e-02 2.441962249577045441e-02 1.220976281911134720e-02 1.757510937750339508e-02 1.879585161805152893e-02 8.801552467048168182e-03 1.542059239000082016e-02 1.820096001029014587e-02 2.032287418842315674e-02 2.337082661688327789e-02 2.394254133105278015e-02 2.171412296593189240e-02 3.262569382786750793e-02 7 | 5.499647259712219238e-01 1.860642433166503906e-01 2.182441502809524536e-01 8.638838678598403931e-02 1.279441360384225845e-02 3.511663153767585754e-02 4.795619472861289978e-02 1.327734440565109253e-01 1.125056743621826172e-01 2.639647573232650757e-02 1.611001789569854736e-02 1.012050732970237732e-02 5.576852709054946899e-02 8.050416409969329834e-02 3.033751435577869415e-02 2.426641434431076050e-02 2.153966389596462250e-02 3.060927055776119232e-02 4.557693004608154297e-02 4.279635101556777954e-02 2.782613039016723633e-02 2.132358029484748840e-02 1.476625166833400726e-02 1.450196094810962677e-02 1.147489715367555618e-02 1.805422641336917877e-02 2.492372319102287292e-02 1.960710622370243073e-02 1.966153271496295929e-02 1.506311446428298950e-02 1.917840726673603058e-02 1.752039603888988495e-02 8 | 1.132967025041580200e-01 6.180718913674354553e-02 1.543412208557128906e-01 5.950048193335533142e-02 1.101032178848981857e-02 2.120797149837017059e-02 2.882544696331024170e-02 1.742505282163619995e-01 1.372356116771697998e-01 2.084167487919330597e-02 1.611104421317577362e-02 8.378042839467525482e-03 1.250048261135816574e-02 2.203668653964996338e-02 1.257414184510707855e-02 9.121384471654891968e-03 1.244367286562919617e-02 4.621664807200431824e-02 7.315611094236373901e-02 3.307605907320976257e-02 1.677793078124523163e-02 1.372016314417123795e-02 1.377994660288095474e-02 1.347098313271999359e-02 1.385486312210559845e-02 1.587509177625179291e-02 1.494149398058652878e-02 1.749720796942710876e-02 2.344859763979911804e-02 2.904917858541011810e-02 2.492165938019752502e-02 3.530199825763702393e-02 9 | 2.530011720955371857e-02 1.923760958015918732e-02 6.138530373573303223e-02 3.123099915683269501e-02 5.923260934650897980e-03 5.349008832126855850e-03 1.348554063588380814e-02 1.058243438601493835e-01 8.726882189512252808e-02 9.189752861857414246e-03 1.005825586616992950e-02 8.973099291324615479e-03 1.383100263774394989e-02 2.202570624649524689e-02 1.217181794345378876e-02 7.189547177404165268e-03 7.876312360167503357e-03 5.821169912815093994e-02 1.229340434074401855e-01 2.975583635270595551e-02 1.132363080978393555e-02 9.610073640942573547e-03 1.000582147389650345e-02 8.450886234641075134e-03 8.401075378060340881e-03 1.048415154218673706e-02 1.252146717160940170e-02 1.079283747822046280e-02 9.814275428652763367e-03 1.285604294389486313e-02 1.111470907926559448e-02 1.631101593375205994e-02 10 | 1.062378138303756714e-01 6.641613692045211792e-02 2.095521390438079834e-01 8.675004541873931885e-02 6.939346436411142349e-03 3.565220162272453308e-03 7.281658239662647247e-03 5.087817087769508362e-02 4.368107020854949951e-02 6.876876577734947205e-03 3.941124305129051208e-03 6.509022787213325500e-03 4.074812307953834534e-02 7.299768179655075073e-02 2.035793475806713104e-02 9.463339112699031830e-03 7.156290579587221146e-03 7.156245410442352295e-02 1.617243289947509766e-01 3.257286921143531799e-02 7.131675258278846741e-03 4.155594855546951294e-03 4.012701101601123810e-03 5.222623236477375031e-03 4.246703814715147018e-03 3.134658560156822205e-03 2.716285642236471176e-03 3.293106565251946449e-03 4.342678003013134003e-03 4.070533439517021179e-03 4.276785068213939667e-03 3.702734829857945442e-03 11 | 2.321120649576187134e-01 1.219327822327613831e-01 3.498409092426300049e-01 1.319804936647415161e-01 5.563526879996061325e-03 5.994630977511405945e-03 7.283471059054136276e-03 3.079329617321491241e-02 2.717610262334346771e-02 9.735307656228542328e-03 7.446975912898778915e-03 4.098969977349042892e-03 4.718970134854316711e-02 9.371434897184371948e-02 1.793835684657096863e-02 6.537623237818479538e-03 6.023949477821588516e-03 6.779732555150985718e-02 1.438099294900894165e-01 2.881984785199165344e-02 6.470102816820144653e-03 4.720021970570087433e-03 7.901296950876712799e-03 9.239842183887958527e-03 6.551482249051332474e-03 5.505876615643501282e-03 5.496996454894542694e-03 6.527411285787820816e-03 4.518456291407346725e-03 5.396924447268247604e-03 6.533627398312091827e-03 3.939153626561164856e-03 12 | 3.124763667583465576e-01 1.222861036658287048e-01 2.270172387361526489e-01 9.945076704025268555e-02 1.127098407596349716e-02 2.334797382354736328e-02 1.201986428350210190e-02 2.502940781414508820e-02 3.597717359662055969e-02 1.551415957510471344e-02 9.924583137035369873e-03 1.117687113583087921e-02 9.543929249048233032e-02 1.795067787170410156e-01 3.118038736283779144e-02 1.371456589549779892e-02 1.310212537646293640e-02 6.164480000734329224e-02 1.307058185338973999e-01 3.399739041924476624e-02 6.732880603522062302e-03 9.657564572989940643e-03 1.036704983562231064e-02 6.069973111152648926e-03 5.622530821710824966e-03 1.245084404945373535e-02 1.918371021747589111e-02 1.391728222370147705e-02 9.231479838490486145e-03 1.085113175213336945e-02 7.347343489527702332e-03 7.540192455053329468e-03 13 | 3.257825672626495361e-01 1.022296473383903503e-01 5.724895000457763672e-02 4.686783999204635620e-02 2.615945413708686829e-02 2.713056840002536774e-02 2.155851200222969055e-02 8.151109516620635986e-02 8.913912624120712280e-02 2.991964481770992279e-02 1.684234105050563812e-02 1.685022935271263123e-02 1.152818053960800171e-01 2.311978489160537720e-01 5.077339336276054382e-02 1.529091224074363708e-02 2.053962647914886475e-02 3.518522530794143677e-02 8.754743635654449463e-02 5.582944676280021667e-02 2.482463046908378601e-02 3.256734833121299744e-02 1.977237313985824585e-02 1.443567778915166855e-02 8.878472261130809784e-03 2.136420831084251404e-02 2.113606780767440796e-02 1.168984360992908478e-02 1.854249648749828339e-02 2.477435395121574402e-02 1.710458286106586456e-02 2.081028744578361511e-02 14 | 3.206983506679534912e-01 1.043769866228103638e-01 1.307483464479446411e-01 8.689565956592559814e-02 2.360058017075061798e-02 1.374563854187726974e-02 2.098820358514785767e-02 1.614168733358383179e-01 1.532922238111495972e-01 2.867304719984531403e-02 1.468386687338352203e-02 1.670525781810283661e-02 7.551267743110656738e-02 1.637584716081619263e-01 4.675063490867614746e-02 2.845099754631519318e-02 2.818950079381465912e-02 6.529083847999572754e-02 1.739810109138488770e-01 6.777963787317276001e-02 2.938549965620040894e-02 3.780119493603706360e-02 3.183939680457115173e-02 2.454373426735401154e-02 1.684934273362159729e-02 2.179620601236820221e-02 2.807947061955928802e-02 2.130556479096412659e-02 1.669271290302276611e-02 1.634720340371131897e-02 1.930869743227958679e-02 2.651147358119487762e-02 15 | 1.891240030527114868e-01 7.837470620870590210e-02 1.745559871196746826e-01 1.260725706815719604e-01 3.588558360934257507e-02 1.345356833189725876e-02 1.696192473173141479e-02 2.004826217889785767e-01 1.813236773014068604e-01 3.059791773557662964e-02 1.011210400611162186e-02 7.593790534883737564e-03 1.877945847809314728e-02 4.655278474092483521e-02 4.646862298250198364e-02 3.055974282324314117e-02 1.806915737688541412e-02 7.205818593502044678e-02 1.610337644815444946e-01 6.422945857048034668e-02 2.426409535109996796e-02 1.431503705680370331e-02 1.134202070534229279e-02 1.337572187185287476e-02 1.187533047050237656e-02 1.019013766199350357e-02 1.139382459223270416e-02 1.010401081293821335e-02 1.307833194732666016e-02 1.878162659704685211e-02 2.279620990157127380e-02 1.698203943669795990e-02 16 | 1.039441600441932678e-01 5.914985761046409607e-02 1.914289444684982300e-01 1.002344191074371338e-01 1.817216910421848297e-02 1.249577384442090988e-02 2.358734980225563049e-02 2.809729278087615967e-01 2.408318370580673218e-01 2.773869968950748444e-02 9.003073908388614655e-03 4.785801749676465988e-03 2.295870706439018250e-02 6.733871996402740479e-02 3.162283077836036682e-02 9.680912829935550690e-03 6.287796888500452042e-03 6.760775297880172729e-02 1.641629934310913086e-01 5.122967436909675598e-02 1.762850023806095123e-02 1.486355718225240707e-02 1.219230610877275467e-02 1.042661815881729126e-02 9.623031131923198700e-03 5.261504091322422028e-03 4.620136693120002747e-03 5.593392066657543182e-03 6.645212881267070770e-03 8.468030020594596863e-03 9.822824969887733459e-03 8.874731138348579407e-03 17 | 2.358485385775566101e-02 2.969334088265895844e-02 1.450208723545074463e-01 1.061563715338706970e-01 2.117448113858699799e-02 6.459060590714216232e-03 1.445136964321136475e-02 1.611065566539764404e-01 1.561153680086135864e-01 2.389383874833583832e-02 1.062741596251726151e-02 6.454607937484979630e-03 3.050366044044494629e-02 9.773265570402145386e-02 4.557001218199729919e-02 1.338474452495574951e-02 6.760540418326854706e-03 6.659124046564102173e-02 1.765331774950027466e-01 6.491322070360183716e-02 1.559141278266906738e-02 8.659404702484607697e-03 5.982722621411085129e-03 3.875137306749820709e-03 4.672171548008918762e-03 4.411696456372737885e-03 5.080848000943660736e-03 7.495446596294641495e-03 7.755599450320005417e-03 9.141685441136360168e-03 5.958871450275182724e-03 8.539300411939620972e-03 18 | 7.170271128416061401e-03 4.344742372632026672e-02 2.533912956714630127e-01 1.237353160977363586e-01 1.377627626061439514e-02 5.938304588198661804e-03 7.877280004322528839e-03 1.536807417869567871e-01 2.133432179689407349e-01 3.561341390013694763e-02 8.152219466865062714e-03 5.714749451726675034e-03 8.440278470516204834e-02 2.351797819137573242e-01 5.885352194309234619e-02 6.596687249839305878e-03 3.820025362074375153e-03 1.632202863693237305e-01 4.223868250846862793e-01 9.144227951765060425e-02 8.786410093307495117e-03 5.155552178621292114e-03 4.553737584501504898e-03 2.020445419475436211e-03 2.229826292023062706e-03 3.477261168882250786e-03 4.252820741385221481e-03 4.598594270646572113e-03 4.245271440595388412e-03 4.004813265055418015e-03 4.996365401893854141e-03 5.716282874345779419e-03 19 | 6.015923246741294861e-02 7.694671303033828735e-02 3.858518302440643311e-01 1.764587759971618652e-01 8.043733425438404083e-03 3.828776068985462189e-03 5.333163775503635406e-03 1.057815253734588623e-01 1.640311926603317261e-01 2.834296785295009613e-02 2.765203360468149185e-03 1.726418267935514450e-03 5.870427936315536499e-02 1.824512034654617310e-01 5.240819603204727173e-02 5.529065150767564774e-03 3.859803080558776855e-03 2.353227585554122925e-01 5.719137787818908691e-01 1.009061262011528015e-01 5.321791861206293106e-03 3.574580885469913483e-03 4.151293076574802399e-03 2.349265851080417633e-03 2.513675717636942863e-03 3.040665527805685997e-03 2.376978751271963120e-03 1.967192627489566803e-03 2.453216351568698883e-03 3.026346908882260323e-03 4.779126029461622238e-03 3.802909515798091888e-03 20 | 1.134202480316162109e-01 1.089483425021171570e-01 5.054341554641723633e-01 2.371616661548614502e-01 1.776563376188278198e-02 4.820094443857669830e-03 5.347644910216331482e-03 9.540504217147827148e-02 1.582342237234115601e-01 2.856111899018287659e-02 2.752555767074227333e-03 1.775780110619962215e-03 2.382745593786239624e-02 1.014417260885238647e-01 5.082493647933006287e-02 7.115026470273733139e-03 3.720416920259594917e-03 2.994780540466308594e-01 7.141621112823486328e-01 1.311105936765670776e-01 8.213099092245101929e-03 2.677034819498658180e-03 3.455176018178462982e-03 4.964401014149188995e-03 2.914048265665769577e-03 1.803098595701158047e-03 1.285978825762867928e-03 1.296130125410854816e-03 1.778576057404279709e-03 2.413327572867274284e-03 3.077436704188585281e-03 2.705814084038138390e-03 21 | 1.102014705538749695e-01 6.858626008033752441e-02 2.539460659027099609e-01 1.349056363105773926e-01 1.572368666529655457e-02 4.959819838404655457e-03 5.051967222243547440e-03 6.531222909688949585e-02 1.128092855215072632e-01 2.795808576047420502e-02 6.817601155489683151e-03 5.183782428503036499e-03 1.787230186164379120e-02 7.473611831665039062e-02 5.277057737112045288e-02 1.053315680474042892e-02 4.125763196498155594e-03 2.542986273765563965e-01 5.902920365333557129e-01 1.060997620224952698e-01 8.032556623220443726e-03 4.334105178713798523e-03 5.397001747041940689e-03 7.063555065542459488e-03 6.388272624462842941e-03 3.683976130560040474e-03 2.061170060187578201e-03 4.158662166446447372e-03 5.247018299996852875e-03 3.388643031939864159e-03 3.932293038815259933e-03 8.567757904529571533e-03 22 | 1.112537160515785217e-01 3.914469480514526367e-02 1.027262061834335327e-01 9.751875698566436768e-02 1.703712344169616699e-02 2.915770281106233597e-03 5.366489756852388382e-03 7.519688457250595093e-02 1.468363404273986816e-01 4.664734378457069397e-02 8.603546768426895142e-03 5.612384062260389328e-03 2.142535522580146790e-02 8.213376998901367188e-02 5.564711242914199829e-02 1.061415392905473709e-02 5.087753757834434509e-03 2.164587378501892090e-01 5.535264611244201660e-01 1.287429183721542358e-01 1.230662409216165543e-02 6.516643334180116653e-03 5.987393204122781754e-03 4.713487811386585236e-03 5.251117050647735596e-03 3.761180909350514412e-03 3.375495551154017448e-03 4.098068922758102417e-03 4.412115551531314850e-03 3.537715645506978035e-03 4.093138035386800766e-03 7.233345881104469299e-03 23 | 6.032273545861244202e-02 4.453867301344871521e-02 1.867594718933105469e-01 1.007924005389213562e-01 1.206580549478530884e-02 4.838072229176759720e-03 4.400090314447879791e-03 8.071778714656829834e-02 1.319244801998138428e-01 4.062459617853164673e-02 1.101597398519515991e-02 4.409829154610633850e-03 1.756020076572895050e-02 5.311058834195137024e-02 3.002061881124973297e-02 1.100757624953985214e-02 4.045520909130573273e-03 1.642599403858184814e-01 3.942108750343322754e-01 7.873631268739700317e-02 1.281396672129631042e-02 6.630329415202140808e-03 5.757253617048263550e-03 5.553888157010078430e-03 4.786053672432899475e-03 4.115046467632055283e-03 3.088311990723013878e-03 1.945603871718049049e-03 2.956432988867163658e-03 4.685006104409694672e-03 4.168445710092782974e-03 3.671654732897877693e-03 24 | 1.227184571325778961e-02 4.662164300680160522e-02 2.676701545715332031e-01 1.256621032953262329e-01 9.015048854053020477e-03 4.462531302124261856e-03 6.011282093822956085e-03 1.064134389162063599e-01 1.573275774717330933e-01 3.263954818248748779e-02 9.729519486427307129e-03 3.513620235025882721e-03 7.172721624374389648e-02 1.852230131626129150e-01 6.124209612607955933e-02 8.504976518452167511e-03 3.020625561475753784e-03 4.996621608734130859e-02 1.450221836566925049e-01 4.999429360032081604e-02 7.977233268320560455e-03 3.225299064069986343e-03 2.535140374675393105e-03 2.393023809418082237e-03 1.891206251457333565e-03 1.930931583046913147e-03 1.969441073015332222e-03 2.136785537004470825e-03 2.551153767853975296e-03 4.130095709115266800e-03 3.184115979820489883e-03 2.383013023063540459e-03 25 | 8.581989444792270660e-03 3.097399696707725525e-02 2.273187488317489624e-01 1.356775313615798950e-01 1.250839326530694962e-02 3.385554766282439232e-03 9.160053916275501251e-03 1.866079121828079224e-01 2.480418533086776733e-01 5.823375284671783447e-02 1.236117817461490631e-02 3.512488678097724915e-03 2.209313213825225830e-01 5.022495388984680176e-01 1.050384715199470520e-01 1.152023486793041229e-02 2.516306936740875244e-03 3.245726972818374634e-02 1.155696958303451538e-01 5.313542857766151428e-02 1.030843425542116165e-02 2.843460766598582268e-03 1.537741743959486485e-03 1.476573990657925606e-03 1.727420254610478878e-03 1.622410374693572521e-03 1.998928841203451157e-03 2.159459516406059265e-03 2.223072107881307602e-03 2.341578947380185127e-03 1.980311702936887741e-03 1.535043586045503616e-03 26 | 3.979542478919029236e-02 5.786051973700523376e-02 3.288322687149047852e-01 1.797005087137222290e-01 1.518054213374853134e-02 3.587914630770683289e-03 1.141082495450973511e-02 2.669641971588134766e-01 3.599734306335449219e-01 7.526125013828277588e-02 1.000506430864334106e-02 2.832139143720269203e-03 3.622314631938934326e-01 7.778945565223693848e-01 1.413040757179260254e-01 1.462055649608373642e-02 2.528309589251875877e-03 1.264973580837249756e-01 3.502070605754852295e-01 9.896560013294219971e-02 1.140904240310192108e-02 2.348455833271145821e-03 1.308095059357583523e-03 1.346367411315441132e-03 1.537057105451822281e-03 1.459808903746306896e-03 1.190947834402322769e-03 1.690196921117603779e-03 1.840994809754192829e-03 1.178744249045848846e-03 1.138344989158213139e-03 1.453566364943981171e-03 27 | 6.579335778951644897e-02 5.642750859260559082e-02 2.641425132751464844e-01 1.353995949029922485e-01 1.419154647737741470e-02 2.299071755260229111e-03 7.521538063883781433e-03 3.186939954757690430e-01 5.384193062782287598e-01 1.057048812508583069e-01 8.492249064147472382e-03 3.317713737487792969e-03 2.194891571998596191e-01 5.016857981681823730e-01 1.208953484892845154e-01 1.311730779707431793e-02 2.113473601639270782e-03 1.435203850269317627e-01 4.023140966892242432e-01 1.121722385287284851e-01 1.025778148323297501e-02 2.054468961432576180e-03 1.011936226859688759e-03 1.337333698756992817e-03 1.580696087330579758e-03 1.056457054801285267e-03 8.292244165204465389e-04 1.102430745959281921e-03 9.659942006692290306e-04 9.456286788918077946e-04 9.061315213330090046e-04 9.359852410852909088e-04 28 | 4.179463908076286316e-02 2.533252909779548645e-02 1.076266765594482422e-01 7.271375507116317749e-02 1.367528550326824188e-02 2.962955273687839508e-03 2.384287118911743164e-02 9.485401511192321777e-01 1.278023481369018555e+00 1.519368439912796021e-01 1.284688990563154221e-02 3.375208703801035881e-03 1.653360873460769653e-01 4.462676048278808594e-01 1.177011057734489441e-01 1.308318227529525757e-02 2.250947989523410797e-03 8.987198770046234131e-02 3.072420358657836914e-01 1.027628779411315918e-01 1.172661315649747849e-02 2.111224923282861710e-03 1.198701676912605762e-03 1.188719295896589756e-03 1.931087928824126720e-03 1.382814021781086922e-03 1.057813176885247231e-03 1.013184664770960808e-03 8.818289497867226601e-04 1.494786934927105904e-03 1.949981087818741798e-03 1.165172550827264786e-03 29 | 9.631960652768611908e-03 7.446409668773412704e-03 4.862125962972640991e-02 5.043849721550941467e-02 1.178016699850559235e-02 2.841887762770056725e-03 5.068433284759521484e-02 1.287481546401977539e+00 1.623007059097290039e+00 3.199129998683929443e-01 3.881716728210449219e-02 7.569964975118637085e-03 2.929584383964538574e-01 7.330922484397888184e-01 2.089142352342605591e-01 3.280121088027954102e-02 6.055636797100305557e-03 1.274890899658203125e-01 4.146008193492889404e-01 1.686022430658340454e-01 2.665791288018226624e-02 5.281721707433462143e-03 1.734055695123970509e-03 1.369360834360122681e-03 1.478682272136211395e-03 1.248796586878597736e-03 1.305551617406308651e-03 1.318323775194585323e-03 1.156821264885365963e-03 1.026988378725945950e-03 1.098782755434513092e-03 8.965790621004998684e-04 30 | 1.330214366316795349e-02 7.976381480693817139e-02 5.212401151657104492e-01 2.631945312023162842e-01 2.232385240495204926e-02 4.416058771312236786e-03 1.327693909406661987e-01 4.672755718231201172e+00 7.448472976684570312e+00 1.377251148223876953e+00 1.236068159341812134e-01 2.768471650779247284e-02 5.251240730285644531e-01 1.632122039794921875e+00 5.727202296257019043e-01 5.248939990997314453e-02 9.153003804385662079e-03 3.995727002620697021e-01 1.493453741073608398e+00 5.807961225509643555e-01 4.856199398636817932e-02 9.486572816967964172e-03 2.816464751958847046e-03 1.612198306247591972e-03 1.249321969226002693e-03 1.092946156859397888e-03 1.048926496878266335e-03 1.112693920731544495e-03 1.098988577723503113e-03 9.210853022523224354e-04 1.085410127416253090e-03 1.338574453257024288e-03 31 | 3.740615025162696838e-02 3.565778434276580811e-01 2.418581962585449219e+00 1.138780474662780762e+00 7.130043208599090576e-02 1.382011733949184418e-02 4.247445166110992432e-01 1.111819076538085938e+01 1.256879806518554688e+01 1.494189262390136719e+00 1.873011142015457153e-01 3.543023020029067993e-02 8.261398673057556152e-01 1.919498085975646973e+00 3.863633871078491211e-01 2.799097262322902679e-02 5.379952024668455124e-03 1.031174778938293457e+00 3.223943948745727539e+00 8.571786284446716309e-01 5.793482065200805664e-02 1.093479245901107788e-02 2.681854646652936935e-03 1.344685559161007404e-03 1.105883740819990635e-03 1.171631389297544956e-03 1.011224696412682533e-03 7.271804497577250004e-04 6.784138386137783527e-04 5.595592083409428596e-04 8.365273242816329002e-04 1.104215043596923351e-03 32 | 2.833391726016998291e-02 3.969122171401977539e-01 2.777248144149780273e+00 1.460666179656982422e+00 1.772676110267639160e-01 3.047147952020168304e-02 2.472758591175079346e-01 6.308411121368408203e+00 6.923930644989013672e+00 7.741277813911437988e-01 1.083122566342353821e-01 1.922213844954967499e-02 3.478311300277709961e-01 7.703925371170043945e-01 1.629657000303268433e-01 1.811541616916656494e-02 3.472362179309129715e-03 3.687669932842254639e-01 1.086215615272521973e+00 3.266819715499877930e-01 5.047408118844032288e-02 9.304773062467575073e-03 2.133125439286231995e-03 8.427166612818837166e-04 6.849303608760237694e-04 7.499171770177781582e-04 5.877350340597331524e-04 4.885339876636862755e-04 4.835184372495859861e-04 5.025746067985892296e-04 4.810639948118478060e-04 5.189540679566562176e-04 33 | 5.035302601754665375e-03 3.292015194892883301e-01 2.544537305831909180e+00 1.543016433715820312e+00 1.721246838569641113e-01 1.601516827940940857e-02 7.726091891527175903e-02 1.887284159660339355e+00 2.226051807403564453e+00 4.156014323234558105e-01 3.657425194978713989e-02 5.430553574115037918e-03 1.484685540199279785e-01 4.101911485195159912e-01 1.530206203460693359e-01 1.914369873702526093e-02 1.869490137323737144e-03 1.414426714181900024e-01 4.991188645362854004e-01 2.240915894508361816e-01 2.882006205618381500e-02 3.575783688575029373e-03 1.289720064960420132e-03 5.824753898195922375e-04 5.649048252962529659e-04 6.236364715732634068e-04 3.969004901591688395e-04 4.386719665490090847e-04 4.267793556209653616e-04 5.164758185856044292e-04 5.666881916113197803e-04 4.646680026780813932e-04 34 | 1.309482939541339874e-02 1.031707376241683960e-01 9.371187686920166016e-01 8.033061623573303223e-01 1.438807696104049683e-01 1.487454771995544434e-02 6.954865064471960068e-03 2.158440798521041870e-01 4.328329861164093018e-01 1.285810619592666626e-01 1.098471134901046753e-02 2.114361850544810295e-03 5.195359140634536743e-02 2.061031758785247803e-01 1.103487908840179443e-01 1.421446632593870163e-02 1.516971038654446602e-03 5.222219228744506836e-02 2.482278198003768921e-01 1.416627019643783569e-01 1.686160638928413391e-02 1.405821763910353184e-03 5.081937415525317192e-04 2.260944456793367863e-04 3.440703731030225754e-04 3.619031922426074743e-04 4.041900974698364735e-04 4.640105471480637789e-04 3.558222088031470776e-04 3.283335536252707243e-04 4.173470370005816221e-04 4.436449089553207159e-04 35 | 2.047791145741939545e-02 5.227687954902648926e-02 5.745836496353149414e-01 5.737116336822509766e-01 1.011773496866226196e-01 1.113189291208982468e-02 4.252830985933542252e-03 7.086678594350814819e-02 1.201863437891006470e-01 4.130349308252334595e-02 6.466937251389026642e-03 9.271475719287991524e-04 2.640423364937305450e-02 1.168472841382026672e-01 6.875644624233245850e-02 1.119040511548519135e-02 1.289202366024255753e-03 2.330355904996395111e-02 1.077638566493988037e-01 7.181815803050994873e-02 1.308210100978612900e-02 1.495412318035960197e-03 4.526303673628717661e-04 2.845055423676967621e-04 2.959007397294044495e-04 2.818013599608093500e-04 3.513935371302068233e-04 3.060914750676602125e-04 2.505956217646598816e-04 2.774547610897570848e-04 3.180184867233037949e-04 3.975601284764707088e-04 36 | 4.679189529269933701e-03 3.584796562790870667e-02 5.138161182403564453e-01 5.249187350273132324e-01 6.278142333030700684e-02 6.901551503688097000e-03 5.250128451734781265e-03 6.600922346115112305e-02 7.269643992185592651e-02 1.109600160270929337e-02 1.980521250516176224e-03 6.260768859647214413e-04 3.186254575848579407e-02 1.074562221765518188e-01 4.418713599443435669e-02 5.334989633411169052e-03 1.634342595934867859e-03 2.660334855318069458e-02 9.800989180803298950e-02 4.224918782711029053e-02 5.578906275331974030e-03 9.909766959026455879e-04 2.946561435237526894e-04 2.404941915301606059e-04 3.029468061868101358e-04 3.408596967346966267e-04 2.361571532674133778e-04 1.936790358740836382e-04 2.073013311019167304e-04 1.785295753506943583e-04 1.820522884372621775e-04 2.074668882414698601e-04 37 | 5.503922700881958008e-03 2.956134453415870667e-02 4.411146342754364014e-01 4.219540059566497803e-01 3.827901929616928101e-02 3.760384162887930870e-03 5.824612453579902649e-03 8.874557167291641235e-02 7.618311792612075806e-02 5.158706568181514740e-03 7.729286444373428822e-04 4.015188023913651705e-04 3.504673391580581665e-02 1.048175245523452759e-01 3.604566305875778198e-02 4.216405563056468964e-03 9.725802228786051273e-04 2.338466420769691467e-02 8.196212351322174072e-02 3.017255477607250214e-02 2.619329141452908516e-03 4.584148409776389599e-04 1.851062843343243003e-04 1.953867758857086301e-04 1.800422760425135493e-04 1.755462290020659566e-04 1.342329196631908417e-04 1.192110576084814966e-04 1.837640738813206553e-04 2.120478166034445167e-04 1.855010195868089795e-04 1.760452869348227978e-04 38 | 4.538163542747497559e-03 1.709543354809284210e-02 1.932642459869384766e-01 1.740665137767791748e-01 2.171459048986434937e-02 3.199445316568017006e-03 3.480065381154417992e-03 4.792797937989234924e-02 3.769804909825325012e-02 1.942810718901455402e-03 4.696254618465900421e-04 2.271280682180076838e-04 2.010120823979377747e-02 5.950972810387611389e-02 1.805625855922698975e-02 1.998838502913713455e-03 4.098312638234347105e-04 9.952458553016185760e-03 3.951823338866233826e-02 1.580925285816192627e-02 1.504576997831463814e-03 4.005572700407356024e-04 2.162300661439076066e-04 1.662673894315958023e-04 1.390492107020691037e-04 1.209367692354135215e-04 1.102092719520442188e-04 1.247166073881089687e-04 1.035986060742288828e-04 9.540257451590150595e-05 9.616326133254915476e-05 6.819821282988414168e-05 39 | 5.687961238436400890e-04 1.141126081347465515e-02 1.340616792440414429e-01 1.124817058444023132e-01 1.272108405828475952e-02 1.673279562965035439e-03 7.554494077339768410e-04 9.039324708282947540e-03 8.755370043218135834e-03 9.895105613395571709e-04 1.325038756476715207e-04 7.303383608814328909e-05 8.587210439145565033e-03 2.685839496552944183e-02 8.278382942080497742e-03 9.332316112704575062e-04 1.774753473000600934e-04 4.115307703614234924e-03 1.936418190598487854e-02 8.550228551030158997e-03 9.596426971256732941e-04 2.085205196635797620e-04 6.755036156391724944e-05 5.347281330614350736e-05 4.892583820037543774e-05 3.927126090275123715e-05 3.767585440073162317e-05 3.741700493264943361e-05 3.651619044831022620e-05 4.122254904359579086e-05 5.084453732706606388e-05 3.789979018620215356e-05 40 | 3.570872941054403782e-04 3.920293878763914108e-03 5.596086010336875916e-02 4.873585328459739685e-02 4.798600915819406509e-03 5.696851876564323902e-04 3.788952308241277933e-04 5.834728013724088669e-03 5.527226254343986511e-03 5.396364140324294567e-04 5.879447053303010762e-05 3.068143632845021784e-05 3.513399278745055199e-03 1.080729532986879349e-02 3.756566438823938370e-03 4.334254190325737000e-04 7.159399683587253094e-05 2.256477251648902893e-03 9.214599616825580597e-03 3.887343220412731171e-03 3.597314353100955486e-04 6.103887426434084773e-05 2.090954694722313434e-05 1.674018676567357033e-05 1.237253854924347252e-05 1.605286706762854010e-05 2.133403722837101668e-05 1.714036443445365876e-05 1.466842513764277101e-05 2.158893767045810819e-05 2.010548450925853103e-05 1.438437266187975183e-05 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | --------------------------------------------------------------------------------