├── requirements.txt ├── README.md ├── LICENSE ├── .gitignore └── openai-whisper-realtime.py /requirements.txt: -------------------------------------------------------------------------------- 1 | git+https://github.com/openai/whisper.git 2 | sounddevice 3 | asyncio 4 | numpy 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OpenAI Whisper Realtime 2 | 3 | This is a quick experiment to achieve almost realtime transcription using Whisper. 4 | 5 | ## How to use 6 | 7 | Install the requirements: 8 | ``` 9 | pip install -r requirements.txt 10 | ``` 11 | 12 | Run the script: 13 | ``` 14 | python openai-whisper-realtime.py 15 | ``` 16 | 17 | Dependencies: 18 | - Python > 3.7 19 | - whisper 20 | - sounddevice 21 | - numpy 22 | - asyncio 23 | 24 | A very fast CPU or GPU is recommended. 25 | 26 | ## How it works 27 | 28 | The systems default audio input is captured with python, split into small chunks and is then fed to OpenAI's original transcription function. It tries (currently rather poorly) to detect word breaks and doesn't split the audio buffer in those cases. 29 | With how the model is designed, it doesn't make the most sense to do this, but i found it would be worth trying. It works acceptably well. 30 | 31 | 32 | ## ToDo: 33 | - Improve transcription performance 34 | - Improve detection of word breaks or pauses, split the buffer dynamically 35 | - Refactoring 36 | - Clean stdout 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Tobias Huttinger 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /openai-whisper-realtime.py: -------------------------------------------------------------------------------- 1 | import sounddevice as sd 2 | import numpy as np 3 | 4 | import whisper 5 | 6 | import asyncio 7 | import queue 8 | import sys 9 | 10 | 11 | # SETTINGS 12 | MODEL_TYPE="base.en" 13 | # the model used for transcription. https://github.com/openai/whisper#available-models-and-languages 14 | LANGUAGE="English" 15 | # pre-set the language to avoid autodetection 16 | BLOCKSIZE=24678 17 | # this is the base chunk size the audio is split into in samples. blocksize / 16000 = chunk length in seconds. 18 | SILENCE_THRESHOLD=400 19 | # should be set to the lowest sample amplitude that the speech in the audio material has 20 | SILENCE_RATIO=100 21 | # number of samples in one buffer that are allowed to be higher than threshold 22 | 23 | 24 | global_ndarray = None 25 | model = whisper.load_model(MODEL_TYPE) 26 | 27 | async def inputstream_generator(): 28 | """Generator that yields blocks of input data as NumPy arrays.""" 29 | q_in = asyncio.Queue() 30 | loop = asyncio.get_event_loop() 31 | 32 | def callback(indata, frame_count, time_info, status): 33 | loop.call_soon_threadsafe(q_in.put_nowait, (indata.copy(), status)) 34 | 35 | stream = sd.InputStream(samplerate=16000, channels=1, dtype='int16', blocksize=BLOCKSIZE, callback=callback) 36 | with stream: 37 | while True: 38 | indata, status = await q_in.get() 39 | yield indata, status 40 | 41 | 42 | async def process_audio_buffer(): 43 | global global_ndarray 44 | async for indata, status in inputstream_generator(): 45 | 46 | indata_flattened = abs(indata.flatten()) 47 | 48 | # discard buffers that contain mostly silence 49 | if(np.asarray(np.where(indata_flattened > SILENCE_THRESHOLD)).size < SILENCE_RATIO): 50 | continue 51 | 52 | if (global_ndarray is not None): 53 | global_ndarray = np.concatenate((global_ndarray, indata), dtype='int16') 54 | else: 55 | global_ndarray = indata 56 | 57 | # concatenate buffers if the end of the current buffer is not silent 58 | if (np.average((indata_flattened[-100:-1])) > SILENCE_THRESHOLD/15): 59 | continue 60 | else: 61 | local_ndarray = global_ndarray.copy() 62 | global_ndarray = None 63 | indata_transformed = local_ndarray.flatten().astype(np.float32) / 32768.0 64 | result = model.transcribe(indata_transformed, language=LANGUAGE) 65 | print(result["text"]) 66 | 67 | del local_ndarray 68 | del indata_flattened 69 | 70 | 71 | async def main(): 72 | print('\nActivating wire ...\n') 73 | audio_task = asyncio.create_task(process_audio_buffer()) 74 | while True: 75 | await asyncio.sleep(1) 76 | audio_task.cancel() 77 | try: 78 | await audio_task 79 | except asyncio.CancelledError: 80 | print('\nwire was cancelled') 81 | 82 | 83 | if __name__ == "__main__": 84 | try: 85 | asyncio.run(main()) 86 | except KeyboardInterrupt: 87 | sys.exit('\nInterrupted by user') 88 | --------------------------------------------------------------------------------