├── .gitattributes ├── menuselect-deps.in.diff ├── .dockerignore ├── config ├── rtp.conf ├── res_speech_gdfe.conf ├── musiconhold.conf ├── confbridge.conf ├── cdr.conf ├── logger.conf ├── pjsip.conf ├── asterisk.conf ├── voicemail.conf ├── cdr_custom.conf ├── queues.conf ├── indications.conf ├── extensions.conf └── modules.conf ├── makeopts.in.diff ├── .gitignore ├── Dockerfile ├── configure.ac.diff ├── Readme.md ├── res ├── res_metering.c └── res_speech_gdfe.c └── LICENSE /.gitattributes: -------------------------------------------------------------------------------- 1 | *.sh text eol=lf 2 | -------------------------------------------------------------------------------- /menuselect-deps.in.diff: -------------------------------------------------------------------------------- 1 | DFEGRPC=@PBX_DFEGRPC@ -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | google 2 | googleapis 3 | .vagrant 4 | -------------------------------------------------------------------------------- /config/rtp.conf: -------------------------------------------------------------------------------- 1 | [general] 2 | rtpstart=8000 3 | rtpend=8100 -------------------------------------------------------------------------------- /makeopts.in.diff: -------------------------------------------------------------------------------- 1 | DFEGRPC_INCLUDE=@DFEGRPC_INCLUDE@ 2 | DFEGRPC_LIB=@DFEGRPC_LIB@ -------------------------------------------------------------------------------- /config/res_speech_gdfe.conf: -------------------------------------------------------------------------------- 1 | [general] 2 | service_key=/etc/asterisk/svc_key.json 3 | -------------------------------------------------------------------------------- /config/musiconhold.conf: -------------------------------------------------------------------------------- 1 | [general] 2 | 3 | [default] 4 | mode = files 5 | directory = moh 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant 2 | google 3 | .vscode 4 | *.json 5 | *.wav 6 | 7 | #because it's easy to keep this here 8 | dfe_testsuite 9 | -------------------------------------------------------------------------------- /config/confbridge.conf: -------------------------------------------------------------------------------- 1 | ; All conferences use default settings. This config must be present to load the confbridge application 2 | -------------------------------------------------------------------------------- /config/cdr.conf: -------------------------------------------------------------------------------- 1 | [general] 2 | enable=yes 3 | 4 | [custom] 5 | ; We log the unique ID as it can be useful for troubleshooting any issues 6 | ; that arise. 7 | loguniqueid=yes 8 | -------------------------------------------------------------------------------- /config/logger.conf: -------------------------------------------------------------------------------- 1 | [general] 2 | 3 | [logfiles] 4 | 5 | console = verbose,notice,warning,error 6 | 7 | ;messages = notice,warning,error 8 | full = verbose,notice,warning,error,debug 9 | ;security = security 10 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:7 2 | WORKDIR /tmp 3 | 4 | ADD http://downloads.asterisk.org/pub/telephony/asterisk/asterisk-13.21.0.tar.gz . 5 | 6 | RUN tar xvfz asterisk-13.21.0.tar.gz 7 | 8 | WORKDIR /tmp/asterisk-13.21.0 9 | 10 | RUN contrib/scripts/install_prereq install 11 | 12 | COPY res_speech_gdfe.cc res/res_speech_gdfe.cc 13 | 14 | RUN ./configure --with-pjproject-bundled && make 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /config/pjsip.conf: -------------------------------------------------------------------------------- 1 | ;================================ TRANSPORTS == 2 | ; Our primary transport definition for UDP communication behind NAT. 3 | [transport-udp-nat] 4 | type = transport 5 | protocol = udp 6 | bind = 0.0.0.0 7 | 8 | [transport-tcp-nat] 9 | type = transport 10 | protocol = tcp 11 | bind = 0.0.0.0 12 | 13 | [anonymous] ;name is required to be anonymous 14 | type=endpoint 15 | context=dialogflow 16 | disallow=all 17 | allow=ulaw 18 | -------------------------------------------------------------------------------- /config/asterisk.conf: -------------------------------------------------------------------------------- 1 | [options] 2 | ; If we want to start Asterisk with a default verbosity for the verbose 3 | ; or debug logger channel types, then we use these settings (by default 4 | ; they are disabled). 5 | verbose = 5 6 | debug = 2 7 | 8 | ; User and group to run asterisk as. NOTE: This will require changes to 9 | ; directory and device permissions. 10 | ;runuser = asterisk ; The user to run as. The default is root. 11 | ;rungroup = asterisk ; The group to run as. The default is root 12 | 13 | ;defaultlanguage = es 14 | -------------------------------------------------------------------------------- /config/voicemail.conf: -------------------------------------------------------------------------------- 1 | [general] 2 | format = wav49|gsm|wav 3 | 4 | [default] 5 | 6 | 7 | [example] 8 | ; Voicemail context for all internal users in the example.com domain. 9 | 1101 = 0717,Maria Berny 10 | 1102 = 7085,Tommie Briar 11 | 1103 = 1809,Penelope Bronte 12 | 1104 = 0039,Richard Casey 13 | 1105 = 6618,Garnet Claude 14 | 1106 = 9805,Aaron Courtney 15 | 1107 = 7484,Lindsey Freddie 16 | 1108 = 7788,Colby Hildred 17 | 1109 = 5750,Terry Jules 18 | 1110 = 3702,Hollis Justy 19 | 1111 = 1878,Temple Morgan 20 | 1112 = 5497,Franny Ocean 21 | 1113 = 1637,Laverne Roberts 22 | 1114 = 3717,Sal Smith 23 | 1115 = 3088,Dusty Williams 24 | -------------------------------------------------------------------------------- /config/cdr_custom.conf: -------------------------------------------------------------------------------- 1 | [mappings] 2 | ; Our CDR log will be written to /var/log/asterisk/cdr-custom/Master.csv 3 | ; with the following schema. 4 | Master.csv => ${CSV_QUOTE(${CDR(clid)})},${CSV_QUOTE(${CDR(src)})},${CSV_QUOTE(${CDR(dst)})},${CSV_QUOTE(${CDR(dcontext)})},${CSV_QUOTE(${CDR(channel)})},${CSV_QUOTE(${CDR(dstchannel)})},${CSV_QUOTE(${CDR(lastapp)})},${CSV_QUOTE(${CDR(lastdata)})},${CSV_QUOTE(${CDR(start)})},${CSV_QUOTE(${CDR(answer)})},${CSV_QUOTE(${CDR(end)})},${CSV_QUOTE(${CDR(duration)})},${CSV_QUOTE(${CDR(billsec)})},${CSV_QUOTE(${CDR(disposition)})},${CSV_QUOTE(${CDR(amaflags)})},${CSV_QUOTE(${CDR(accountcode)})},${CSV_QUOTE(${CDR(uniqueid)})},${CSV_QUOTE(${CDR(userfield)})},${CDR(sequence)} 5 | -------------------------------------------------------------------------------- /config/queues.conf: -------------------------------------------------------------------------------- 1 | [general] 2 | monitor-type = MixMonitor 3 | 4 | ;========================Sales Queue == 5 | ; Calls all sales persons in a ring-all fashion 6 | [sales] 7 | strategy=ringall 8 | member => PJSIP/1109 ; Terry Jules - Director of Sales 9 | member => PJSIP/1105 ; Garnet Claude - Sales Associate 10 | member => PJSIP/1112 ; Franny Ocean - Sales Associate 11 | 12 | ;===================== Customer Advocate Queue == 13 | ; Calls all customer advocates in a ring-all fashion 14 | [customer_advocate] 15 | strategy=ringall 16 | member => PJSIP/1101 ; Maria Berny - Director of Customer Experience 17 | member => PJSIP/1115 ; Dusty Williams - Customer Advocate 18 | member => PJSIP/1102 ; Tommy Briar - Customer Advocate 19 | -------------------------------------------------------------------------------- /config/indications.conf: -------------------------------------------------------------------------------- 1 | [general] 2 | country = us ; We are in Waldo, Al, USA so the US is our default. 3 | 4 | [us] 5 | description = United States / North America 6 | ringcadence = 2000,4000 7 | dial = 350+440 8 | busy = 480+620/500,0/500 9 | ring = 440+480/2000,0/4000 10 | congestion = 480+620/250,0/250 11 | callwaiting = 440/300,0/10000 12 | dialrecall = !350+440/100,!0/100,!350+440/100,!0/100,!350+440/100,!0/100,350+440 13 | record = 1400/500,0/15000 14 | info = !950/330,!1400/330,!1800/330,0 15 | stutter = !350+440/100,!0/100,!350+440/100,!0/100,!350+440/100,!0/100,!350+440/100,!0/100,!350+440/100,!0/100,!350+440/100,!0/100,350+440 16 | 17 | ; Additional country configurations can be found in the Asterisk source 18 | ; at /configs/samples/indications.conf.sample 19 | -------------------------------------------------------------------------------- /configure.ac.diff: -------------------------------------------------------------------------------- 1 | --- configure.ac 2 | +++ configure.ac 3 | @@ -464,6 +464,9 @@ AST_EXT_LIB_SETUP([VORBIS], [Vorbis], [vorbis]) 4 | AST_EXT_LIB_SETUP([VPB], [Voicetronix API], [vpb]) 5 | AST_EXT_LIB_SETUP([X11], [X11], [x11]) 6 | AST_EXT_LIB_SETUP([ZLIB], [zlib compression], [z]) 7 | +# BEGIN RES_SPEECH_DFE 8 | +AST_EXT_LIB_SETUP([DFEGRPC], [Dialogflow gRPC], [dfegrpc]) 9 | +# END RES_SPEECH_DFE 10 | 11 | # check for basic system features and functionality before 12 | # checking for package libraries 13 | @@ -2118,6 +2121,9 @@ else 14 | fi 15 | AST_C_DECLARE_CHECK([VORBIS_OPEN_CALLBACKS], [OV_CALLBACKS_NOCLOSE], [vorbis/vorbisfile.h]) 16 | 17 | +# BEGIN RES_SPEECH_DFE 18 | +AST_EXT_LIB_CHECK([DFEGRPC], [dfegrpc], [df_init], [libdfegrpc.h], [-lprotobuf -lgrpc++ -lstdc++]) 19 | +# END RES_SPEECH_DFE 20 | AC_LANG_PUSH(C++) 21 | 22 | if test "${USE_VPB}" != "no"; then 23 | -------------------------------------------------------------------------------- /config/extensions.conf: -------------------------------------------------------------------------------- 1 | [globals] 2 | 3 | [dialogflow] 4 | exten => test,1,NoOp(Simple test) 5 | same => n,Answer 6 | same => n,Playback(pls-hold-while-try) 7 | same => n,Hangup 8 | 9 | exten => _[a-z0-9].,1,NoOp(Dialogflow) 10 | same => n,Answer 11 | same => n,SpeechCreate() 12 | same => n,Set(LANG_OVERRIDE=${PJSIP_HEADER(read,Language)}) 13 | same => n,Set(SPEECH_ENGINE(project_id)=${EXTEN}) 14 | same => n,Set(SPEECH_ENGINE(session_id)=${UNIQUEID}) 15 | same => n,Set(SPEECH_ENGINE(language)=${CHANNEL(language)}) 16 | same => n,GotoIf($["${LANG_OVERRIDE}"==""]?clear_prompt) 17 | same => n,Set(SPEECH_ENGINE(language)=${LANG_OVERRIDE}) 18 | same => n(clear_prompt),Set(PROMPT=) 19 | same => n,SpeechActivateGrammar(event:welcome) 20 | same => n,Goto(dialogflow-loop,${EXTEN},start) 21 | 22 | [dialogflow-loop] 23 | exten => _[a-z0-9].,1(start),NoOp(Dialogflow Loop Start) 24 | same => n,SpeechBackground(${PROMPT}) 25 | ;same => n,System("rm -f ${PROMPTFULLNAME}") 26 | same => n,GoSub(dialogflow-process-results,${EXTEN},start) 27 | same => n,GotoIf($[${LEN("${TRANSFER}")}>0]?dialogflow-transfer,${EXTEN},start) 28 | same => n,GotoIf($[${LEN("${PROMPT}")}>0]?start) 29 | same => n,Hangup 30 | 31 | [dialogflow-process-results] 32 | exten => _[a-z0-9].,1(start),NoOp(Got ${SPEECH(results)} results) 33 | same => n,Set(PROMPT=) 34 | same => n,Set(i=0) 35 | same => n(process-results),Set(GRAMMAR=${SPEECH_GRAMMAR(${i})}) 36 | same => n,Set(VALUE=${SPEECH_TEXT(${i})}) 37 | same => n(checkfortransfer),GotoIf($["${GRAMMAR}"=~"transfer_call"=0]?checkforaudio) 38 | same => n,Set(TRANSFER=${VALUE}) 39 | same => n(checkforaudio),GotoIf($["${GRAMMAR}"!="fulfillment_audio"]?increment) 40 | ; prompt is everything but .wav 41 | same => n,Set(PROMPT="${VALUE:0:-4}") 42 | same => n,Set(PROMPTFULLNAME="${VALUE}") 43 | same => n(increment),Set(i=$[${i}+1]) 44 | same => n,GotoIf($[${SPEECH(results)}>${i}]?process-results) 45 | same => n,Return() 46 | 47 | [dialogflow-transfer] 48 | exten => _[a-z0-9].,1(start),NoOp(Transferring to ${EXTEN}) 49 | same => n,GotoIf($[${LEN("${PROMPT}")}=0]?dotransfer) 50 | same => n,Playback(${PROMPT}) 51 | same => n(dotransfer),Transfer(${TRANSFER}) 52 | same => n,GotoIf($["${TRANSFERSTATUS}"="${FAILURE}"]?transferfailed) 53 | same => n,GotoIf($["${TRANSFERSTATUS}"="${UNSUPPORTED}"]?transferunsupported) 54 | same => n,Hangup ; success! 55 | same => n(transferfailed),Playback(all-circuits-busy-now) 56 | same => n,Hangup 57 | same => n(transferunsupported),Playback(ss-noservice) 58 | same => n,Hangup 59 | -------------------------------------------------------------------------------- /config/modules.conf: -------------------------------------------------------------------------------- 1 | [modules] 2 | autoload = no 3 | 4 | ; This is a minimal module load. We are loading only the modules required for 5 | ; the Asterisk features used in the Super Awesome Company configuration. 6 | 7 | ; Applications 8 | 9 | load = app_bridgewait.so 10 | load = app_dial.so 11 | load = app_playback.so 12 | load = app_stack.so 13 | load = app_verbose.so 14 | load = app_voicemail.so 15 | load = app_directory.so 16 | load = app_confbridge.so 17 | load = app_queue.so 18 | load = app_transfer.so 19 | 20 | ; Bridging 21 | 22 | load = bridge_builtin_features.so 23 | load = bridge_builtin_interval_features.so 24 | load = bridge_holding.so 25 | load = bridge_native_rtp.so 26 | load = bridge_simple.so 27 | load = bridge_softmix.so 28 | 29 | ; Call Detail Records 30 | 31 | load = cdr_custom.so 32 | 33 | ; Channel Drivers 34 | 35 | load = chan_bridge_media.so 36 | load = chan_pjsip.so 37 | 38 | ; Codecs 39 | 40 | load = codec_gsm.so 41 | load = codec_resample.so 42 | load = codec_ulaw.so 43 | load = codec_g722.so 44 | 45 | ; Formats 46 | 47 | load = format_gsm.so 48 | load = format_pcm.so 49 | load = format_wav_gsm.so 50 | load = format_wav.so 51 | 52 | ; Functions 53 | 54 | load = func_callerid.so 55 | load = func_cdr.so 56 | load = func_pjsip_endpoint.so 57 | load = func_sorcery.so 58 | load = func_devstate.so 59 | load = func_strings.so 60 | load = func_channel.so 61 | 62 | ; Core/PBX 63 | 64 | load = pbx_config.so 65 | 66 | ; Resources 67 | 68 | load = res_musiconhold.so 69 | load = res_pjproject.so 70 | load = res_pjsip_acl.so 71 | load = res_pjsip_authenticator_digest.so 72 | load = res_pjsip_caller_id.so 73 | load = res_pjsip_dialog_info_body_generator.so 74 | load = res_pjsip_diversion.so 75 | load = res_pjsip_dtmf_info.so 76 | load = res_pjsip_endpoint_identifier_anonymous.so 77 | load = res_pjsip_endpoint_identifier_ip.so 78 | load = res_pjsip_endpoint_identifier_user.so 79 | load = res_pjsip_exten_state.so 80 | load = res_pjsip_header_funcs.so 81 | load = res_pjsip_logger.so 82 | load = res_pjsip_messaging.so 83 | load = res_pjsip_mwi_body_generator.so 84 | load = res_pjsip_mwi.so 85 | load = res_pjsip_nat.so 86 | load = res_pjsip_notify.so 87 | load = res_pjsip_one_touch_record_info.so 88 | load = res_pjsip_outbound_authenticator_digest.so 89 | load = res_pjsip_outbound_publish.so 90 | load = res_pjsip_outbound_registration.so 91 | load = res_pjsip_path.so 92 | load = res_pjsip_pidf_body_generator.so 93 | load = res_pjsip_pidf_digium_body_supplement.so 94 | load = res_pjsip_pidf_eyebeam_body_supplement.so 95 | load = res_pjsip_publish_asterisk.so 96 | load = res_pjsip_pubsub.so 97 | load = res_pjsip_refer.so 98 | load = res_pjsip_registrar_expire.so 99 | load = res_pjsip_registrar.so 100 | load = res_pjsip_rfc3326.so 101 | load = res_pjsip_sdp_rtp.so 102 | load = res_pjsip_send_to_voicemail.so 103 | load = res_pjsip_session.so 104 | load = res_pjsip.so 105 | load = res_pjsip_t38.so 106 | load = res_pjsip_transport_websocket.so 107 | load = res_pjsip_xpidf_body_generator.so 108 | load = res_rtp_asterisk.so 109 | load = res_sorcery_astdb.so 110 | load = res_sorcery_config.so 111 | load = res_sorcery_memory.so 112 | load = res_sorcery_realtime.so 113 | load = res_timing_timerfd.so 114 | 115 | load = res_speech.so 116 | load = res_speech_gdfe.so 117 | load = app_speech_utils.so 118 | load = res_metering.so -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | ## Default Configuration 4 | 5 | ### res_speech_gdfe.conf 6 | 7 | The configuration for the Google DFE speech module is in the `res_speech_gdfe.conf` file in the Asterisk configuration directory. It is a standard format Asterisk configuration file. 8 | 9 | #### [general] section 10 | - `service_key` - (required) the path to a JSON-format Google service key or the actual key itself. 11 | - `endpoint` - (optional) the URL for the DialogFlow API endpoint. Leave blank to use the default `dialogflow.googleapis.com`. 12 | - `vad_voice_threshold` - (optional) the average absolute amplitude of a packet to consider that packet to be 'voice'. The default is 512. Valid range 0-32767. 13 | - `vad_voice_minimum_duration` - (optional, milliseconds) the cumulative duration of consecutive 'voice' packets to consider the caller to be speaking. The default is 40 (milliseconds). Valid range 0-2147483647. 14 | - `vad_silence_minimum_duration` - (optional, milliseconds, not implemented) the cumulative duration of consecutive non-'voice' packets to consider the caller to be not speaking. The default is 500 (milliseconds). Valid range 0-2147483647. This setting currently has no effect as the end of speech is determined by DialogFlow. 15 | 16 | ### Environment Variables 17 | 18 | #### http_proxy 19 | 20 | To access the DialogFlow endpoint via a proxy you must set the environment variable `http_proxy` to the URL of your proxy. This must be done for the Asterisk process as a whole. 21 | 22 | ## Per-Call Configuration 23 | 24 | Speech module behavior may be modified by using the `SPEECH_ENGINE` dialplan function. Available settings are: 25 | 26 | - `session_id` - set a session identifier to use when making DialogFlow API calls. This will be reflected in the history of the agent on the DialogFlow console. A default random value will be used if not provided. 27 | - `project_id` - set the project identifier to use when making DialogFlow API calls. This setting is required in order to determine which agent to use. 28 | - `language` - set the language for the recognition engine for when doing intent detection and prompt generation. The default is `en`. The engine has no visibility into the channel language -- if it has changed it is still necessary to set the engine language. 29 | - `voice_threshold` - set the average absolute amplitude of a packet to consider that packet to be 'voice' (see `vad_voice_threshold`, above). 30 | - `voice_duration` - set the cumulative duration of consecutive 'voice' packets to consider the caller to be speaking (see `vad_voice_minimum_duration`, above). 31 | - `silence_duration` - the cumulative duration of consecutive non-'voice' packets to consider the caller to be not speaking (see `vad_silence_minimum_duration`, above). 32 | 33 | # Usage 34 | 35 | ## Setup 36 | 37 | Before detecting intent for your calls, you must first: 38 | 1. create the speech resource (only do this once), 39 | 1. set the `project_id`, and 40 | 41 | ``` 42 | same => n,SpeechCreate() 43 | same => n,Set(SPEECH_ENGINE(project_id)=my-project-12345) 44 | ``` 45 | 46 | ## Detecting Intent from Voice 47 | 48 | To detect intent from voice, you should call `SpeechBackground` to send audio to the module. You may specify a prompt to play while performing the detection. 49 | 50 | ``` 51 | same => n,SpeechBackground(hello-world) 52 | ``` 53 | 54 | ## Detecting Intent by Event 55 | 56 | To detect intent by event, you should activate a grammar with the name `event:{your event name}` prior to calling `SpeechBackground`. The `SpeechBackground` application will return immediately -- you should not include a prompt. 57 | 58 | ``` 59 | same => n,SpeechActivateGrammar(event:welcome) 60 | same => n,SpeechBackground(hello-world) 61 | ``` 62 | 63 | ## Processing results 64 | 65 | The DialogFlow module returns the following results (when available): 66 | - `response_id` - the unique identifier for this response 67 | - `query_text` - the text of the speech recognized by DialogFlow 68 | - `language_code` - the detected language of the recognized speech 69 | - `action` - the action for the detected intent 70 | - `fulfillment_text` - the text of the next prompt for the caller 71 | - `intent_name` - the API name of the intent detected 72 | - `intent_display_name` - the displayed name of the intent detected 73 | - `raw_score` - the raw recognition score 74 | - `fulfillment_message_N_text_M` - the fulfillment text messages from the response 75 | - `fulfillment_message_N_simple_response_M` - the simple response fulfillment messages from the response 76 | - `fulfillment_message_N_telephony_play_audio` - a URI from the telephony audio response fulfillment message 77 | - `fulfillment_message_N_telephony_synthesize_speech` - text or SSML from the telephony synthesized speech fulfillment message 78 | - `fulfillment_message_N_telephony_transfer_call` - a phone number for transferring from the telephony transfer call fulfillment message 79 | - `fulfillment_message_N_telephony_terminate_call` - a flag indicating that the fulfillment message requested call termination 80 | - `fulfillment_audio` - a path to audio corresponding to the fulfillment text 81 | 82 | (those with _N_ or _M_ in the name may occur multiple times with different indexes in those positions) 83 | 84 | -------------------------------------------------------------------------------- /res/res_metering.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Asterisk -- An open source telephony toolkit. 3 | * 4 | * Copyright (C) 2018, USAN, Inc. 5 | * 6 | * Daniel Collins 7 | * based on sample code by David M. Lee, II 8 | * 9 | * See http://www.asterisk.org for more information about 10 | * the Asterisk project. Please do not directly contact 11 | * any of the maintainers of this project for assistance; 12 | * the project provides a web site, mailing lists and IRC 13 | * channels for your use. 14 | * 15 | * This program is free software, distributed under the terms of 16 | * the GNU General Public License Version 2. See the LICENSE file 17 | * at the top of the source tree. 18 | */ 19 | 20 | /*! 21 | * \brief Google metering engine 22 | * 23 | * This module subscribes to the channel caching topic and issues metering 24 | * events to the Google billing container. 25 | * 26 | * \author Daniel Collins 27 | */ 28 | 29 | /*** MODULEINFO 30 | curl 31 | extended 32 | ***/ 33 | 34 | #include "asterisk.h" 35 | 36 | #include 37 | 38 | #include "asterisk/file.h" 39 | #include "asterisk/module.h" 40 | #include "asterisk/stasis_channels.h" 41 | #include "asterisk/stasis_message_router.h" 42 | #include "asterisk/statsd.h" 43 | #include "asterisk/sched.h" 44 | #include "asterisk/time.h" 45 | #include "asterisk/cli.h" 46 | 47 | /*! Stasis message router */ 48 | static struct stasis_message_router *router; 49 | /*! metering message scheduler */ 50 | static struct ast_sched_context *metric_scheduler; 51 | /*! current schedule id */ 52 | static int metric_sched_id = 0; 53 | /*! consecutive failure count */ 54 | static int consecutive_failures = 0; 55 | 56 | static ast_mutex_t count_lock; 57 | static int current_channel_count = 0; 58 | static int interval_max_channel_count = 0; 59 | static struct timeval interval_start; 60 | static struct timeval interval_end; 61 | 62 | static int max_consecutive_failures = 2; 63 | static int interval = 5; 64 | static const char *report_url = "http://localhost:4242/report"; 65 | 66 | static struct timeval next_interval_end(int interval_minutes, struct timeval basis) 67 | { 68 | struct timeval next = basis; 69 | next.tv_sec = ((next.tv_sec / (interval * 60)) + 1) * interval * 60; 70 | next.tv_usec = 0; 71 | return next; 72 | } 73 | 74 | static int ms_until_next_interval(int interval_minutes, struct timeval basis) 75 | { 76 | struct timeval next_interval = next_interval_end(interval_minutes, basis); 77 | ast_assert(ast_tvcmp(next_interval, ast_tvnow()) >= 0); 78 | return ast_tvdiff_ms(next_interval, ast_tvnow()); 79 | } 80 | 81 | static void crash_asterisk(void) 82 | { 83 | char *buff = NULL; 84 | ast_log(LOG_ERROR, "Forcing a process crash\n"); 85 | usleep(1000); /* to let the last few log messages be written */ 86 | memmove(buff, "crashing now", 12); /* that should do it */ 87 | } 88 | 89 | static void stop_asterisk(void) 90 | { 91 | char template[] = "/tmp/res-metering-XXXXXX"; 92 | int ret; 93 | RAII_VAR(int, fd, -1, close); 94 | 95 | ast_log(LOG_ERROR, "Shutting down asterisk due to metering failures.\n"); 96 | 97 | if ((fd = mkstemp(template)) < 0) { 98 | ast_log(LOG_WARNING, "Failed to create temporary file to shut down asterisk: %s\n", strerror(errno)); 99 | crash_asterisk(); 100 | return; 101 | } 102 | 103 | ret = ast_cli_command(fd, "core stop now"); 104 | if (ret != RESULT_SUCCESS) { 105 | ast_log(LOG_WARNING, "Failed to 'core stop now'\n"); 106 | crash_asterisk(); 107 | } 108 | } 109 | 110 | static size_t curl_report_data_callback(char *ptr, size_t size, size_t nmemb, void *data) 111 | { 112 | ast_log(LOG_DEBUG, "Got unexpected data on usage report -- '%.*s'.\n", (size * nmemb), ptr); 113 | return size * nmemb; 114 | } 115 | 116 | static size_t curl_report_read_callback(char *buffer, size_t size, size_t nitems, char *data) 117 | { 118 | size_t towrite = 0; 119 | if (!ast_strlen_zero(data)) { 120 | size_t datalen = strlen(data); 121 | towrite = size * nitems; 122 | if (datalen > towrite) { 123 | memcpy(buffer, data, towrite); 124 | /* shift over the remaining data */ 125 | memmove(data, data + towrite, datalen - towrite + 1); 126 | } else { 127 | towrite = datalen; 128 | memcpy(buffer, data, towrite); 129 | *data = '\0'; 130 | } 131 | } 132 | return towrite; 133 | } 134 | 135 | static int send_metric_data(const void *_) 136 | { 137 | int count; 138 | struct timeval tvstart; 139 | struct timeval tvend; 140 | struct ast_json *obj; 141 | int resched_ms; 142 | RAII_VAR(struct ast_json *, data, ast_json_object_create(), ast_json_unref); 143 | RAII_VAR(CURL *, curl, curl_easy_init(), curl_easy_cleanup); 144 | 145 | ast_mutex_lock(&count_lock); 146 | count = interval_max_channel_count; 147 | interval_max_channel_count = current_channel_count; 148 | tvstart = interval_start; 149 | tvend = interval_end; 150 | interval_start = interval_end; 151 | interval_end = next_interval_end(interval, interval_start); 152 | ast_mutex_unlock(&count_lock); 153 | 154 | if (!data) { 155 | ast_log(LOG_WARNING, "Error allocating json structure for metric body\n"); 156 | } else if (!curl) { 157 | ast_log(LOG_WARNING, "Unable to init curl to report metrics\n"); 158 | } else { 159 | RAII_VAR(char *, body, NULL, ast_json_free); 160 | RAII_VAR(struct curl_slist *, headers, NULL, curl_slist_free_all); 161 | RAII_VAR(struct ast_str *, header, ast_str_create(128), ast_free); 162 | CURLcode res; 163 | char start[AST_ISO8601_LEN]; 164 | char end[AST_ISO8601_LEN]; 165 | struct ast_tm tmstart = {}; 166 | struct ast_tm tmend = {}; 167 | long http_code = 0; 168 | 169 | ast_localtime(&tvstart, &tmstart, NULL); 170 | ast_localtime(&tvend, &tmend, NULL); 171 | 172 | ast_strftime(start, sizeof(start), "%FT%TZ", &tmstart); 173 | ast_strftime(end, sizeof(end), "%FT%TZ", &tmend); 174 | 175 | ast_json_object_set(data, "name", ast_json_string_create("concurrentCalls")); 176 | ast_json_object_set(data, "startTime", ast_json_string_create(start)); 177 | ast_json_object_set(data, "endTime", ast_json_string_create(end)); 178 | obj = ast_json_object_create(); 179 | ast_json_object_set(data, "value", obj); 180 | ast_json_object_set(obj, "int64Value", ast_json_integer_create(count)); 181 | 182 | body = ast_json_dump_string(data); 183 | 184 | curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); 185 | curl_easy_setopt(curl, CURLOPT_TIMEOUT, 180); 186 | curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_report_data_callback); 187 | curl_easy_setopt(curl, CURLOPT_USERAGENT, "res-metering/1.0"); 188 | curl_easy_setopt(curl, CURLOPT_URL, report_url); 189 | curl_easy_setopt(curl, CURLOPT_READFUNCTION, curl_report_read_callback); 190 | curl_easy_setopt(curl, CURLOPT_READDATA, body); 191 | curl_easy_setopt(curl, CURLOPT_POST, 1); 192 | 193 | ast_str_set(&header, 0, "Content-Length: %d", (int)strlen(body)); 194 | headers = curl_slist_append(headers, ast_str_buffer(header)); 195 | headers = curl_slist_append(headers, "Content-Type: application/json"); 196 | 197 | curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); 198 | 199 | if (option_debug) { 200 | ast_log(LOG_DEBUG, "Posting usage metric '%s' to %s\n", body, report_url); 201 | } 202 | 203 | res = curl_easy_perform(curl); 204 | curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); 205 | if (res != CURLE_OK || http_code != 200) { 206 | consecutive_failures++; 207 | ast_log(LOG_WARNING, "Got error %d posting metric to %s -- %s\n", http_code, report_url, curl_easy_strerror(res)); 208 | 209 | /* try to reset the metrics so if we do eventually get through it's correct */ 210 | ast_mutex_lock(&count_lock); 211 | interval_max_channel_count = MAX(current_channel_count, count); 212 | interval_start = tvstart; 213 | interval_end = next_interval_end(interval, tvend); 214 | ast_mutex_unlock(&count_lock); 215 | 216 | if (consecutive_failures > max_consecutive_failures) { 217 | stop_asterisk(); 218 | } 219 | } else { 220 | consecutive_failures = 0; 221 | } 222 | } 223 | 224 | resched_ms = ms_until_next_interval(interval, ast_tvnow()); 225 | ast_assert(resched_ms <= (interval * 60 * 1000)); 226 | metric_sched_id = ast_sched_add(metric_scheduler, resched_ms, send_metric_data, NULL); 227 | if (metric_sched_id < 0) { 228 | ast_log(LOG_WARNING, "Failed to add schedule for metering\n"); 229 | } else if (option_debug) { 230 | ast_log(LOG_DEBUG, "Scheduled next metering event %dms from now\n", resched_ms); 231 | } 232 | return 0; 233 | } 234 | 235 | static int update_channel_count(int addend) 236 | { 237 | int current; 238 | ast_mutex_lock(&count_lock); 239 | current_channel_count += addend; 240 | current = current_channel_count; 241 | if (current > interval_max_channel_count) { 242 | interval_max_channel_count = current; 243 | } 244 | ast_mutex_unlock(&count_lock); 245 | return current; 246 | } 247 | 248 | /*! 249 | * \brief Router callback for \ref stasis_cache_update messages. 250 | * \param data Data pointer given when added to router. 251 | * \param sub This subscription. 252 | * \param topic The topic the message was posted to. This is not necessarily the 253 | * topic you subscribed to, since messages may be forwarded between 254 | * topics. 255 | * \param message The message itself. 256 | */ 257 | static void channel_updates(void *data, struct stasis_subscription *sub, 258 | struct stasis_message *message) 259 | { 260 | /* Since this came from a message router, we know the type of the 261 | * message. We can cast the data without checking its type. 262 | */ 263 | struct stasis_cache_update *update = stasis_message_data(message); 264 | 265 | /* We're only interested in channel snapshots, so check the type 266 | * of the underlying message. 267 | */ 268 | if (ast_channel_snapshot_type() != update->type) { 269 | return; 270 | } 271 | 272 | /* There are three types of cache updates. 273 | * !old && new -> Initial cache entry 274 | * old && new -> Updated cache entry 275 | * old && !new -> Cache entry removed. 276 | */ 277 | 278 | if (!update->old_snapshot && update->new_snapshot) { 279 | /* Initial cache entry; count a channel creation */ 280 | struct ast_channel_snapshot *snapshot = stasis_message_data(update->new_snapshot); 281 | if (snapshot && !ast_test_flag(&snapshot->flags, AST_FLAG_OUTGOING)) { 282 | /* do not count outbound channels */ 283 | int count = update_channel_count(+1); 284 | ast_log(LOG_DEBUG, "Add channel. Current count - %d\n", count); 285 | } 286 | } else if (update->old_snapshot && !update->new_snapshot) { 287 | /* Cache entry removed. Compute the age of the channel and post 288 | * that, as well as decrementing the channel count. 289 | */ 290 | struct ast_channel_snapshot *snapshot = stasis_message_data(update->old_snapshot); 291 | if (snapshot && !ast_test_flag(&snapshot->flags, AST_FLAG_OUTGOING)) { 292 | /* do not count outbound channels */ 293 | int count = update_channel_count(-1); 294 | ast_log(LOG_DEBUG, "Remove channel. Current count - %d\n", count); 295 | } 296 | } 297 | } 298 | 299 | /*! 300 | * \brief Router callback for any message that doesn't otherwise have a route. 301 | * \param data Data pointer given when added to router. 302 | * \param sub This subscription. 303 | * \param topic The topic the message was posted to. This is not necessarily the 304 | * topic you subscribed to, since messages may be forwarded between 305 | * topics. 306 | * \param message The message itself. 307 | */ 308 | static void default_route(void *data, struct stasis_subscription *sub, 309 | struct stasis_message *message) 310 | { 311 | if (stasis_subscription_final_message(sub, message)) { 312 | /* Much like with the regular subscription, you may need to 313 | * perform some cleanup when done with a message router. You 314 | * can look for the final message in the default route. 315 | */ 316 | return; 317 | } 318 | } 319 | 320 | static int unload_module(void) 321 | { 322 | if (metric_scheduler) { 323 | ast_sched_context_destroy(metric_scheduler); 324 | } 325 | if (router) { 326 | stasis_message_router_unsubscribe_and_join(router); 327 | router = NULL; 328 | } 329 | ast_mutex_destroy(&count_lock); 330 | return 0; 331 | } 332 | 333 | static void load_config(void) 334 | { 335 | RAII_VAR(struct ast_config *, cfg, NULL, ast_config_destroy); 336 | struct ast_flags config_flags = { 0 }; 337 | const char *val; 338 | 339 | cfg = ast_config_load("res_metering.conf", config_flags); 340 | 341 | if (cfg == CONFIG_STATUS_FILEINVALID) { 342 | ast_log(LOG_WARNING, "Configuration file invalid\n"); 343 | cfg = ast_config_new(); 344 | } else if (cfg == CONFIG_STATUS_FILEMISSING) { 345 | ast_log(LOG_WARNING, "Configuration not found, using defaults\n"); 346 | cfg = ast_config_new(); 347 | } 348 | 349 | val = ast_variable_retrieve(cfg, "general", "interval"); 350 | if (!ast_strlen_zero(val)) { 351 | int i; 352 | if (sscanf(val, "%d", &i) == 1) { 353 | if (i >= 1 && i <= 60) { 354 | interval = i; 355 | } else { 356 | ast_log(LOG_WARNING, "'interval' exceeds allowable limits.\n"); 357 | } 358 | } else { 359 | ast_log(LOG_WARNING, "'interval' must be numeric\n"); 360 | } 361 | } 362 | 363 | val = ast_variable_retrieve(cfg, "general", "max_consecutive_failures"); 364 | if (!ast_strlen_zero(val)) { 365 | int i; 366 | if (sscanf(val, "%d", &i) == 1) { 367 | max_consecutive_failures = i; 368 | } else { 369 | ast_log(LOG_WARNING, "'max_consecutive_failures' must be numeric\n"); 370 | } 371 | } 372 | 373 | val = ast_variable_retrieve(cfg, "general", "report_url"); 374 | if (!ast_strlen_zero(val)) { 375 | report_url = ast_strdup(val); 376 | } 377 | } 378 | 379 | static int load_module(void) 380 | { 381 | int sched_ms; 382 | 383 | ast_mutex_init(&count_lock); 384 | 385 | load_config(); 386 | 387 | metric_scheduler = ast_sched_context_create(); 388 | if (!metric_scheduler) { 389 | ast_log(LOG_WARNING, "Failed to initialize scheduling context\n"); 390 | return AST_MODULE_LOAD_DECLINE; 391 | } 392 | 393 | if (ast_sched_start_thread(metric_scheduler)) { 394 | ast_log(LOG_WARNING, "Failed to create scheduler thread\n"); 395 | return AST_MODULE_LOAD_DECLINE; 396 | } 397 | 398 | /* You can create a message router to route messages by type */ 399 | router = stasis_message_router_create( 400 | ast_channel_topic_all_cached()); 401 | if (!router) { 402 | ast_log(LOG_WARNING, "Failed to initialize statis router\n"); 403 | unload_module(); 404 | return AST_MODULE_LOAD_DECLINE; 405 | } 406 | stasis_message_router_add(router, stasis_cache_update_type(), 407 | channel_updates, NULL); 408 | stasis_message_router_set_default(router, default_route, NULL); 409 | 410 | interval_start = ast_tvnow(); 411 | interval_end = next_interval_end(interval, interval_start); 412 | sched_ms = ms_until_next_interval(interval, ast_tvnow()); 413 | ast_assert(sched_ms <= (interval * 60 * 1000)); 414 | 415 | metric_sched_id = ast_sched_add(metric_scheduler, sched_ms, send_metric_data, NULL); 416 | if (metric_sched_id < 0) { 417 | ast_log(LOG_WARNING, "Failed to schedule metric event\n"); 418 | unload_module(); 419 | return AST_MODULE_LOAD_DECLINE; 420 | } else { 421 | ast_log(LOG_DEBUG, "Scheduled first metric event %dms from now\n", sched_ms); 422 | } 423 | 424 | return AST_MODULE_LOAD_SUCCESS; 425 | } 426 | 427 | AST_MODULE_INFO(ASTERISK_GPL_KEY, AST_MODFLAG_DEFAULT, "Usage metric tracking", 428 | .support_level = AST_MODULE_SUPPORT_EXTENDED, 429 | .load = load_module, 430 | .unload = unload_module 431 | ); -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /res/res_speech_gdfe.c: -------------------------------------------------------------------------------- 1 | /* 2 | * res_speech_gdfe -- an Asterisk speech driver for Google DialogFlow for Enterprise 3 | * 4 | * Copyright (C) 2018, USAN, Inc. 5 | * 6 | * Daniel Collins 7 | * 8 | * See http://www.asterisk.org for more information about 9 | * the Asterisk project. Please do not directly contact 10 | * any of the maintainers of this project for assistance; 11 | * the project provides a web site, mailing lists and IRC 12 | * channels for your use. 13 | * 14 | * This program is free software, distributed under the terms of 15 | * the GNU General Public License Version 2. See the LICENSE file 16 | * at the top of the source asterisk tree. 17 | * 18 | */ 19 | 20 | /*** MODULEINFO 21 | res_speech 22 | dfegrpc 23 | ***/ 24 | 25 | //#define REF_DEBUG 1 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | #ifdef RAII_VAR 36 | #define ASTERISK_13_OR_LATER 37 | #endif 38 | 39 | #ifdef ASTERISK_13_OR_LATER 40 | #ifndef AST_SPEECH_HAVE_GET_SETTING 41 | #define AST_SPEECH_HAVE_GET_SETTING 42 | #endif 43 | #endif 44 | 45 | #ifdef ASTERISK_13_OR_LATER 46 | #include 47 | #include 48 | #include 49 | #include 50 | #else 51 | #include 52 | #include 53 | #endif 54 | 55 | #include 56 | #include 57 | #include 58 | #include 59 | 60 | #include 61 | 62 | #include 63 | #include 64 | #include 65 | 66 | #ifndef ASTERISK_13_OR_LATER 67 | #include 68 | #endif 69 | 70 | #define GDF_PROP_SESSION_ID_NAME "session_id" 71 | #define GDF_PROP_ALTERNATE_SESSION_NAME "name" 72 | #define GDF_PROP_PROJECT_ID_NAME "project_id" 73 | #define GDF_PROP_LANGUAGE_NAME "language" 74 | #define GDF_PROP_LOG_CONTEXT "log_context" 75 | #define GDF_PROP_ALTERNATE_LOG_CONTEXT "logContext" 76 | #define GDF_PROP_APPLICATION_CONTEXT "application" 77 | #define GDF_PROP_REQUEST_SENTIMENT_ANALYSIS "request_sentiment_analysis" 78 | #define VAD_PROP_VOICE_THRESHOLD "voice_threshold" 79 | #define VAD_PROP_VOICE_DURATION "voice_duration" 80 | #define VAD_PROP_SILENCE_DURATION "silence_duration" 81 | #define VAD_PROP_BARGE_DURATION "barge_duration" 82 | #define VAD_PROP_END_OF_SPEECH_DURATION "end_of_speech_duration" 83 | 84 | #define GDF_PROP_UTTERANCE_DURATION_MS "utterance_duration_ms" 85 | 86 | typedef int milliseconds_t; 87 | 88 | enum VAD_STATE { 89 | VAD_STATE_START, 90 | VAD_STATE_SPEAK, 91 | VAD_STATE_SILENT, 92 | VAD_STATE_END, 93 | }; 94 | 95 | enum SENTIMENT_ANALYSIS_STATE { 96 | SENTIMENT_ANALYSIS_DEFAULT, 97 | SENTIMENT_ANALYSIS_NEVER, 98 | SENTIMENT_ANALYSIS_ALWAYS 99 | }; 100 | 101 | enum GDFE_STATE { 102 | GDFE_STATE_START, 103 | GDFE_STATE_PROCESSING, 104 | GDFE_STATE_HAVE_RESULTS, 105 | GDFE_STATE_DONE 106 | }; 107 | 108 | struct gdf_request; 109 | 110 | struct gdf_pvt { 111 | struct ast_speech *speech; 112 | 113 | milliseconds_t vad_state_duration; 114 | milliseconds_t vad_change_duration; /* cumulative time of "not current state" audio */ 115 | 116 | int voice_threshold; /* 0 - (2^16 - 1) */ 117 | milliseconds_t voice_minimum_duration; 118 | milliseconds_t silence_minimum_duration; 119 | milliseconds_t barge_in_minimum_duration; 120 | milliseconds_t end_of_speech_minimum_silence; 121 | 122 | milliseconds_t incomplete_timeout; 123 | milliseconds_t no_speech_timeout; 124 | milliseconds_t maximum_speech_timeout; 125 | 126 | int call_log_open_already_attempted; 127 | FILE *call_log_file_handle; 128 | 129 | int utterance_counter; 130 | struct gdf_request *current_request; 131 | 132 | enum SENTIMENT_ANALYSIS_STATE effective_sentiment_analysis_state; 133 | int request_sentiment_analysis; 134 | int use_internal_endpointer_for_end_of_speech; 135 | 136 | int record_next_utterance; 137 | 138 | struct timeval session_start; /* log only, no store duration */ 139 | 140 | char **hints; 141 | size_t hint_count; 142 | 143 | AST_DECLARE_STRING_FIELDS( 144 | AST_STRING_FIELD(logical_agent_name); 145 | AST_STRING_FIELD(project_id); 146 | AST_STRING_FIELD(session_id); 147 | AST_STRING_FIELD(service_key); 148 | AST_STRING_FIELD(endpoint); 149 | AST_STRING_FIELD(event); 150 | AST_STRING_FIELD(language); 151 | AST_STRING_FIELD(lastAudioResponse); 152 | AST_STRING_FIELD(model); 153 | 154 | AST_STRING_FIELD(call_log_path); 155 | AST_STRING_FIELD(call_log_file_basename); 156 | AST_STRING_FIELD(call_logging_application_name); 157 | AST_STRING_FIELD(call_logging_context); 158 | ); 159 | }; 160 | 161 | struct gdf_request { 162 | struct gdf_pvt *pvt; 163 | struct dialogflow_session *session; 164 | 165 | int current_utterance_number; 166 | int current_start_retry; 167 | 168 | enum VAD_STATE vad_state; 169 | milliseconds_t vad_state_duration; 170 | milliseconds_t vad_change_duration; /* cumulative time of "not current state" audio */ 171 | 172 | int voice_threshold; /* 0 - (2^16 - 1) */ 173 | int heard_speech; 174 | milliseconds_t voice_minimum_duration; 175 | milliseconds_t silence_minimum_duration; 176 | milliseconds_t barge_in_minimum_duration; 177 | milliseconds_t end_of_speech_minimum_silence; 178 | 179 | milliseconds_t incomplete_timeout; 180 | milliseconds_t no_speech_timeout; 181 | milliseconds_t maximum_speech_timeout; 182 | 183 | int record_utterance; 184 | 185 | int utterance_preendpointer_recording_open_already_attempted; 186 | FILE *utterance_preendpointer_recording_file_handle; 187 | int utterance_postendpointer_recording_open_already_attempted; 188 | FILE *utterance_postendpointer_recording_file_handle; 189 | 190 | char *mulaw_endpointer_audio_cache; 191 | size_t mulaw_endpointer_audio_cache_size; 192 | size_t mulaw_endpointer_audio_cache_start; 193 | size_t mulaw_endpointer_audio_cache_len; 194 | 195 | struct timeval session_start; /* log only, no store duration */ 196 | struct timeval endpointer_end_of_speech_time; 197 | struct timeval request_start; 198 | struct timeval recognition_initial_attempt; 199 | struct timeval speech_start; 200 | struct timeval endpointer_barge_in_time; 201 | struct timeval dialogflow_barge_in_time; 202 | long long last_request_duration_ms; 203 | long long last_audio_duration_ms; /* calculated by packet, not clock */ 204 | 205 | pthread_t thread; 206 | enum GDFE_STATE state; 207 | AST_LIST_HEAD_NOLOCK(, ast_frame) frame_queue; 208 | int frame_queue_len; 209 | 210 | AST_DECLARE_STRING_FIELDS( 211 | AST_STRING_FIELD(project_id); 212 | AST_STRING_FIELD(service_key); 213 | AST_STRING_FIELD(endpoint); 214 | AST_STRING_FIELD(event); 215 | AST_STRING_FIELD(language); 216 | AST_STRING_FIELD(pre_recording_filename); 217 | AST_STRING_FIELD(post_recording_filename); 218 | AST_STRING_FIELD(model); 219 | ); 220 | }; 221 | 222 | struct ao2_container *config; 223 | 224 | struct gdf_logical_agent { 225 | AST_DECLARE_STRING_FIELDS( 226 | AST_STRING_FIELD(name); 227 | AST_STRING_FIELD(project_id); 228 | AST_STRING_FIELD(service_key); 229 | AST_STRING_FIELD(endpoint); 230 | AST_STRING_FIELD(model); 231 | ); 232 | enum SENTIMENT_ANALYSIS_STATE enable_sentiment_analysis; 233 | int use_internal_endpointer_for_end_of_speech; 234 | struct ao2_container *hints; 235 | }; 236 | 237 | struct gdf_config { 238 | int vad_voice_threshold; 239 | milliseconds_t vad_voice_minimum_duration; 240 | milliseconds_t vad_silence_minimum_duration; 241 | milliseconds_t vad_barge_minimum_duration; 242 | milliseconds_t vad_end_of_speech_silence_duration; 243 | 244 | milliseconds_t endpointer_cache_audio_pretrigger_ms; 245 | 246 | milliseconds_t default_incomplete_timeout; 247 | milliseconds_t default_no_speech_timeout; 248 | milliseconds_t default_maximum_speech_timeout; 249 | 250 | int enable_call_logs; 251 | int enable_preendpointer_recordings; 252 | int enable_postendpointer_recordings; 253 | int record_preendpointer_on_demand; 254 | int use_internal_endpointer_for_end_of_speech; 255 | enum SENTIMENT_ANALYSIS_STATE enable_sentiment_analysis; 256 | 257 | int stop_writes_on_final_transcription; 258 | int start_recognition_on_start; /* vs. on speech */ 259 | int recognition_start_failure_retries; 260 | int recognition_start_failure_retry_max_time_ms; 261 | 262 | struct ao2_container *logical_agents; 263 | struct ao2_container *hints; 264 | 265 | int synthesize_fulfillment_text; 266 | 267 | AST_DECLARE_STRING_FIELDS( 268 | AST_STRING_FIELD(service_key); 269 | AST_STRING_FIELD(endpoint); 270 | AST_STRING_FIELD(call_log_location); 271 | AST_STRING_FIELD(start_failure_retry_codes); 272 | AST_STRING_FIELD(model); 273 | ); 274 | }; 275 | 276 | enum gdf_call_log_type { 277 | CALL_LOG_TYPE_SESSION, 278 | CALL_LOG_TYPE_RECOGNITION, 279 | CALL_LOG_TYPE_ENDPOINTER, 280 | CALL_LOG_TYPE_DIALOGFLOW 281 | }; 282 | 283 | static struct gdf_config *gdf_get_config(void); 284 | static struct gdf_logical_agent *get_logical_agent_by_name(struct gdf_config *config, const char *name); 285 | static void gdf_log_call_event(struct gdf_pvt *pvt, struct gdf_request *req, enum gdf_call_log_type type, const char *event, size_t log_data_size, const struct dialogflow_log_data *log_data); 286 | #define gdf_log_call_event_only(pvt, req, type, event) gdf_log_call_event(pvt, req, type, event, 0, NULL) 287 | 288 | static struct ast_str *build_log_related_filename_to_thread_local_str(struct gdf_pvt *pvt, struct gdf_request *req, const char *type, const char *extension); 289 | 290 | static void *gdf_exec(void *arg); 291 | 292 | #ifdef ASTERISK_13_OR_LATER 293 | typedef struct ast_format *local_ast_format_t; 294 | #else 295 | typedef int local_ast_format_t; 296 | #endif 297 | 298 | static void gdf_request_destructor(void *obj) 299 | { 300 | struct gdf_request *req = obj; 301 | struct ast_frame *f; 302 | 303 | ast_log(LOG_DEBUG, "Destroying gdf request %d@%s\n", req->current_utterance_number, req->pvt->session_id); 304 | 305 | df_close_session(req->session); 306 | 307 | if (req->mulaw_endpointer_audio_cache) { 308 | ast_free(req->mulaw_endpointer_audio_cache); 309 | req->mulaw_endpointer_audio_cache = NULL; 310 | } 311 | 312 | while ((f = AST_LIST_REMOVE_HEAD(&req->frame_queue, frame_list))) { 313 | ast_frfree(f); 314 | } 315 | 316 | ast_string_field_free_memory(req); 317 | 318 | if (req->pvt) { 319 | ao2_t_ref(req->pvt, -1, "Destroying request"); 320 | req->pvt = NULL; 321 | } 322 | } 323 | 324 | static void gdf_pvt_destructor(void *obj) 325 | { 326 | struct gdf_pvt *pvt = obj; 327 | 328 | ast_log(LOG_DEBUG, "Destroying gdf pvt %s\n", pvt->session_id); 329 | 330 | if (pvt->call_log_file_handle != NULL) { 331 | fclose(pvt->call_log_file_handle); 332 | } 333 | 334 | if (pvt->hints) { 335 | size_t i; 336 | for (i = 0; i < pvt->hint_count; i++) { 337 | ast_free(pvt->hints[i]); 338 | } 339 | ast_free(pvt->hints); 340 | } 341 | 342 | if (!ast_strlen_zero(pvt->lastAudioResponse)) { 343 | unlink(pvt->lastAudioResponse); 344 | ast_string_field_set(pvt, lastAudioResponse, ""); 345 | } 346 | 347 | ast_string_field_free_memory(pvt); 348 | } 349 | 350 | static int gdf_create(struct ast_speech *speech, local_ast_format_t format) 351 | { 352 | struct gdf_pvt *pvt; 353 | struct gdf_config *cfg; 354 | char session_id[32]; 355 | size_t sidlen = sizeof(session_id); 356 | char *sid = session_id; 357 | 358 | pvt = ao2_alloc(sizeof(struct gdf_pvt), gdf_pvt_destructor); 359 | if (!pvt) { 360 | ast_log(LOG_WARNING, "Error allocating memory for GDF private structure\n"); 361 | return -1; 362 | } 363 | 364 | if (ast_string_field_init(pvt, 252)) { 365 | ast_log(LOG_WARNING, "Error allocating GDF private string fields\n"); 366 | ao2_t_ref(pvt, -1, "Error allocating string fields"); 367 | return -1; 368 | } 369 | 370 | ast_build_string(&sid, &sidlen, "%p", pvt); 371 | 372 | cfg = gdf_get_config(); 373 | 374 | ast_string_field_set(pvt, session_id, session_id); 375 | pvt->voice_threshold = cfg->vad_voice_threshold; 376 | pvt->voice_minimum_duration = cfg->vad_voice_minimum_duration; 377 | pvt->silence_minimum_duration = cfg->vad_silence_minimum_duration; 378 | pvt->barge_in_minimum_duration = cfg->vad_barge_minimum_duration; 379 | pvt->end_of_speech_minimum_silence = cfg->vad_end_of_speech_silence_duration; 380 | pvt->session_start = ast_tvnow(); 381 | pvt->incomplete_timeout = cfg->default_incomplete_timeout; 382 | pvt->no_speech_timeout = cfg->default_no_speech_timeout; 383 | pvt->maximum_speech_timeout = cfg->default_maximum_speech_timeout; 384 | ast_string_field_set(pvt, call_logging_application_name, "unknown"); 385 | pvt->speech = speech; 386 | 387 | ast_log(LOG_DEBUG, "Creating GDF session %s\n", pvt->session_id); 388 | 389 | ast_mutex_lock(&speech->lock); 390 | speech->state = AST_SPEECH_STATE_NOT_READY; 391 | speech->data = pvt; 392 | /* speech will borrow this reference */ 393 | ast_mutex_unlock(&speech->lock); 394 | 395 | ao2_t_ref(cfg, -1, "done with creating session"); 396 | 397 | return 0; 398 | } 399 | 400 | static void reset_pvt_timeouts_to_defaults_on_new_request(struct gdf_pvt *pvt_locked, struct gdf_config *cfg) 401 | { 402 | pvt_locked->incomplete_timeout = cfg->default_incomplete_timeout; 403 | pvt_locked->no_speech_timeout = cfg->default_no_speech_timeout; 404 | pvt_locked->maximum_speech_timeout = cfg->default_maximum_speech_timeout; 405 | } 406 | 407 | static struct gdf_request *create_new_request(struct gdf_pvt *pvt_locked, int utterance_number) 408 | { 409 | struct gdf_request *req; 410 | struct gdf_config *cfg; 411 | int res; 412 | 413 | req = ao2_alloc(sizeof(struct gdf_request), gdf_request_destructor); 414 | if (!req) { 415 | ast_log(LOG_WARNING, "Error allocating memory for GDF request structure\n"); 416 | return NULL; 417 | } 418 | 419 | if (ast_string_field_init(req, 252)) { 420 | ast_log(LOG_WARNING, "Error allocating GDF request string fields\n"); 421 | ao2_t_ref(req, -1, "Error allocating string fields"); 422 | return NULL; 423 | } 424 | 425 | ao2_t_ref(pvt_locked, 1, "Adding backpointer from request to private"); 426 | req->pvt = pvt_locked; 427 | req->current_utterance_number = utterance_number; 428 | 429 | cfg = gdf_get_config(); 430 | 431 | req->session = df_create_session(req); 432 | if (!req->session) { 433 | ast_log(LOG_WARNING, "Error creating session for GDF\n"); 434 | ao2_t_ref(cfg, -1, "done with creating request"); 435 | ao2_t_ref(req, -1, "Error creating dialogflow request"); 436 | return NULL; 437 | } 438 | df_set_auth_key(req->session, pvt_locked->service_key); 439 | df_set_endpoint(req->session, pvt_locked->endpoint); 440 | df_set_session_id(req->session, pvt_locked->session_id); 441 | df_set_stop_writes_on_final_transcription(req->session, cfg->stop_writes_on_final_transcription); 442 | 443 | ast_string_field_set(req, project_id, pvt_locked->project_id); 444 | ast_string_field_set(req, event, pvt_locked->event); 445 | ast_string_field_set(req, language, pvt_locked->language); 446 | ast_string_field_set(req, service_key, pvt_locked->service_key); 447 | ast_string_field_set(req, endpoint, pvt_locked->endpoint); 448 | ast_string_field_set(req, model, pvt_locked->model); 449 | 450 | req->voice_threshold = pvt_locked->voice_threshold; 451 | req->voice_minimum_duration = pvt_locked->voice_minimum_duration; 452 | req->silence_minimum_duration = pvt_locked->silence_minimum_duration; 453 | req->barge_in_minimum_duration = pvt_locked->barge_in_minimum_duration; 454 | req->end_of_speech_minimum_silence = pvt_locked->end_of_speech_minimum_silence; 455 | req->session_start = ast_tvnow(); 456 | req->last_audio_duration_ms = 0; 457 | req->incomplete_timeout = pvt_locked->incomplete_timeout; 458 | req->no_speech_timeout = pvt_locked->no_speech_timeout; 459 | req->maximum_speech_timeout = pvt_locked->maximum_speech_timeout; 460 | reset_pvt_timeouts_to_defaults_on_new_request(pvt_locked, cfg); 461 | 462 | if (req->voice_minimum_duration || cfg->endpointer_cache_audio_pretrigger_ms) { 463 | size_t cache_needed_size = (req->voice_minimum_duration + cfg->endpointer_cache_audio_pretrigger_ms) * 8; /* bytes per millisecond */ 464 | req->mulaw_endpointer_audio_cache = ast_calloc(1, cache_needed_size); 465 | if (req->mulaw_endpointer_audio_cache) { 466 | req->mulaw_endpointer_audio_cache_size = cache_needed_size; 467 | req->mulaw_endpointer_audio_cache_start = 0; 468 | req->mulaw_endpointer_audio_cache_len = 0; 469 | } 470 | } 471 | 472 | ast_log(LOG_DEBUG, "Creating GDF request %d@%s\n", req->current_utterance_number, pvt_locked->session_id); 473 | 474 | ast_speech_change_state(pvt_locked->speech, AST_SPEECH_STATE_READY); 475 | 476 | ao2_t_ref(req, 1, "Bump ref for background thread"); 477 | res = ast_pthread_create_detached(&req->thread, NULL, gdf_exec, req); 478 | if (res) { 479 | ast_log(LOG_WARNING, "Unable to create background thread for GDF"); 480 | ao2_t_ref(cfg, -1, "done with creating request"); 481 | ao2_t_ref(req, -1, "Error creating background thread"); 482 | return NULL; 483 | } 484 | 485 | ao2_t_ref(cfg, -1, "done with creating request"); 486 | 487 | return req; 488 | } 489 | 490 | static void log_session_end(struct gdf_pvt *pvt, long long duration_ms) 491 | { 492 | char duration_buffer[20] = ""; 493 | char *buff = duration_buffer; 494 | size_t buffLen = sizeof(duration_buffer); 495 | struct dialogflow_log_data log_data[] = { 496 | { "duration_ms", duration_buffer } 497 | }; 498 | 499 | ast_build_string(&buff, &buffLen, "%lld", duration_ms); 500 | 501 | gdf_log_call_event(pvt, NULL, CALL_LOG_TYPE_SESSION, "end", ARRAY_LEN(log_data), log_data); 502 | } 503 | 504 | static int gdf_destroy(struct ast_speech *speech) 505 | { 506 | struct gdf_pvt *pvt = speech->data; 507 | 508 | ao2_lock(pvt); 509 | pvt->speech = NULL; 510 | if (pvt->current_request) { 511 | ao2_t_ref(pvt->current_request, -1, "Destroying session, releasing request"); 512 | } 513 | pvt->current_request = NULL; 514 | ao2_unlock(pvt); 515 | 516 | log_session_end(pvt, ast_tvdiff_ms(ast_tvnow(), pvt->session_start)); 517 | 518 | ast_log(LOG_DEBUG, "Destroying GDF %s\n", pvt->session_id); 519 | ao2_t_ref(pvt, -1, "Destroying speech session"); 520 | return 0; 521 | } 522 | 523 | static int gdf_load(struct ast_speech *speech, const char *grammar_name, const char *grammar) 524 | { 525 | return 0; 526 | } 527 | 528 | static int gdf_unload(struct ast_speech *speech, const char *grammar_name) 529 | { 530 | return 0; 531 | } 532 | 533 | #define EVENT_COLON_LEN 6 534 | #define EVENT_COLON "event:" 535 | static int is_grammar_old_style_event(const char *grammar_name) 536 | { 537 | return !strncasecmp(grammar_name, EVENT_COLON, EVENT_COLON_LEN); 538 | } 539 | 540 | static void activate_old_style_event(struct gdf_pvt *pvt, const char *grammar_name) 541 | { 542 | const char *name = grammar_name + EVENT_COLON_LEN; 543 | ast_log(LOG_DEBUG, "Activating event %s on %s\n", name, pvt->session_id); 544 | ao2_lock(pvt); 545 | ast_string_field_set(pvt, event, name); 546 | ao2_unlock(pvt); 547 | } 548 | 549 | #define BUILTIN_COLON_GRAMMAR_SLASH_LEN 16 550 | #define BUILTIN_COLON_GRAMMAR_SLASH "builtin:grammar/" 551 | static int is_grammar_new_style_format(const char *grammar_name) 552 | { 553 | return !strncasecmp(grammar_name, BUILTIN_COLON_GRAMMAR_SLASH, BUILTIN_COLON_GRAMMAR_SLASH_LEN); 554 | } 555 | 556 | static void calculate_effective_sentiment_analysis_state(struct gdf_pvt *pvt_locked, struct gdf_config *config, struct gdf_logical_agent *logical_agent) 557 | { 558 | pvt_locked->effective_sentiment_analysis_state = SENTIMENT_ANALYSIS_DEFAULT; 559 | if (config && config->enable_sentiment_analysis != SENTIMENT_ANALYSIS_DEFAULT) { 560 | pvt_locked->effective_sentiment_analysis_state = config->enable_sentiment_analysis; 561 | } else if (logical_agent) { 562 | pvt_locked->effective_sentiment_analysis_state = logical_agent->enable_sentiment_analysis; 563 | } 564 | } 565 | 566 | static void calculate_effective_hints(struct gdf_pvt *pvt_locked, struct gdf_config *config, struct gdf_logical_agent *logical_agent) 567 | { 568 | struct ao2_container *hint_container = NULL; 569 | 570 | if (pvt_locked->hints) { 571 | size_t i; 572 | for (i = 0; i < pvt_locked->hint_count; i++) { 573 | ast_free(pvt_locked->hints[i]); 574 | } 575 | ast_free(pvt_locked->hints); 576 | pvt_locked->hints = NULL; 577 | } 578 | 579 | pvt_locked->hint_count = 0; 580 | 581 | if (logical_agent) { 582 | hint_container = logical_agent->hints; 583 | } 584 | if (hint_container == NULL || !ao2_container_count(hint_container)) { 585 | hint_container = config->hints; 586 | } 587 | 588 | if (hint_container) { 589 | size_t hint_count = ao2_container_count(hint_container); 590 | if (hint_count > 0) { 591 | pvt_locked->hints = ast_calloc(hint_count, sizeof(char *)); 592 | if (pvt_locked->hints) { 593 | size_t i; 594 | struct ao2_iterator hint_iterator; 595 | char *entry; 596 | 597 | hint_iterator = ao2_iterator_init(hint_container, 0); 598 | for (i = 0, entry = ao2_iterator_next(&hint_iterator); entry && i < hint_count; i++, entry = ao2_iterator_next(&hint_iterator)) { 599 | pvt_locked->hints[i] = ast_strdup(entry); 600 | pvt_locked->hint_count = i + 1; 601 | ao2_ref(entry, -1); 602 | } 603 | ao2_iterator_destroy(&hint_iterator); 604 | } 605 | } 606 | } 607 | } 608 | 609 | static void activate_agent_for_name(struct gdf_pvt *pvt, const char *name, size_t name_len, const char *event) 610 | { 611 | struct gdf_config *config; 612 | 613 | ao2_lock(pvt); 614 | ast_string_field_build(pvt, logical_agent_name, "%.*s", (int) name_len, name); 615 | ast_string_field_set(pvt, event, event); 616 | ao2_unlock(pvt); 617 | 618 | config = gdf_get_config(); 619 | if (config) { 620 | struct gdf_logical_agent *logical_agent_map = get_logical_agent_by_name(config, pvt->logical_agent_name); 621 | ao2_lock(pvt); 622 | ast_string_field_set(pvt, project_id, S_OR(logical_agent_map ? logical_agent_map->project_id : NULL, pvt->logical_agent_name)); 623 | ast_string_field_set(pvt, service_key, S_OR(logical_agent_map ? logical_agent_map->service_key : NULL, config->service_key)); 624 | ast_string_field_set(pvt, endpoint, S_OR(logical_agent_map ? logical_agent_map->endpoint : NULL, config->endpoint)); 625 | ast_string_field_set(pvt, model, S_OR(logical_agent_map ? logical_agent_map->model : NULL, config->model)); 626 | calculate_effective_sentiment_analysis_state(pvt, config, logical_agent_map); 627 | calculate_effective_hints(pvt, config, logical_agent_map); 628 | pvt->use_internal_endpointer_for_end_of_speech = logical_agent_map ? logical_agent_map->use_internal_endpointer_for_end_of_speech : config->use_internal_endpointer_for_end_of_speech; 629 | ao2_unlock(pvt); 630 | if (logical_agent_map) { 631 | ao2_ref(logical_agent_map, -1); 632 | } 633 | ao2_ref(config, -1); 634 | } else { 635 | ao2_lock(pvt); 636 | ast_string_field_set(pvt, project_id, pvt->logical_agent_name); 637 | ast_string_field_set(pvt, model, ""); 638 | calculate_effective_sentiment_analysis_state(pvt, NULL, NULL); 639 | pvt->use_internal_endpointer_for_end_of_speech = 1; 640 | ao2_unlock(pvt); 641 | } 642 | 643 | if (!ast_strlen_zero(event)) { 644 | ast_log(LOG_DEBUG, "Activating project %s ('%s'), event %s on %s\n", 645 | pvt->project_id, pvt->logical_agent_name, pvt->event, pvt->session_id); 646 | } else { 647 | ast_log(LOG_DEBUG, "Activating project %s ('%s') on %s\n", pvt->project_id, pvt->logical_agent_name, 648 | pvt->session_id); 649 | } 650 | } 651 | 652 | static void activate_new_style_grammar(struct gdf_pvt *pvt, const char *grammar_name) 653 | { 654 | const char *name_part = grammar_name + BUILTIN_COLON_GRAMMAR_SLASH_LEN; 655 | const char *event_part = ""; 656 | size_t name_len; 657 | const char *question_mark; 658 | 659 | if ((question_mark = strchr(name_part, '?'))) { 660 | name_len = question_mark - name_part; 661 | event_part = question_mark + 1; 662 | } else { 663 | name_len = strlen(name_part); 664 | } 665 | 666 | activate_agent_for_name(pvt, name_part, name_len, event_part); 667 | } 668 | 669 | /** activate is used in this context to prime DFE with an event for 'detection' 670 | * this is typically used when starting up (e.g. event:welcome) 671 | */ 672 | static int gdf_activate(struct ast_speech *speech, const char *grammar_name) 673 | { 674 | struct gdf_pvt *pvt = speech->data; 675 | if (is_grammar_old_style_event(grammar_name)) { 676 | activate_old_style_event(pvt, grammar_name); 677 | } else if (is_grammar_new_style_format(grammar_name)) { 678 | activate_new_style_grammar(pvt, grammar_name); 679 | } else { 680 | ast_log(LOG_WARNING, "Do not understand grammar name %s on %s\n", grammar_name, pvt->session_id); 681 | return -1; 682 | } 683 | return 0; 684 | } 685 | 686 | static int gdf_deactivate(struct ast_speech *speech, const char *grammar_name) 687 | { 688 | return 0; 689 | } 690 | 691 | static int calculate_audio_level(const short *slin, int len) 692 | { 693 | int i; 694 | long long sum = 0; 695 | for (i = 0; i < len; i++) { 696 | short sample = slin[i]; 697 | sum += abs(sample); 698 | } 699 | #ifdef RES_SPEECH_GDFE_DEBUG_VAD 700 | ast_log(LOG_DEBUG, "packet sum = %lld, average = %d\n", sum, (int)(sum / len)); 701 | #endif 702 | return sum / len; 703 | } 704 | 705 | static long long tvdiff_ms_or_zero(struct timeval end, struct timeval start) 706 | { 707 | if (ast_tvzero(end) || ast_tvzero(start)) { 708 | return 0; 709 | } else { 710 | return ast_tvdiff_ms(end, start); 711 | } 712 | } 713 | 714 | static void write_end_of_recognition_call_event(struct gdf_request *req) 715 | { 716 | char duration_buffer[32] = ""; 717 | char utterance_duration_buffer[32] = ""; 718 | char speech_rec_duration_buffer[32] = ""; 719 | char dialogflow_response_time_buffer[32] = ""; 720 | char intent_latency_time_buffer[32] = ""; 721 | char endpointer_barge_in_time_buffer[32] = ""; 722 | char dialogflow_barge_in_time_buffer[32] = ""; 723 | struct timeval dialogflow_start_time = df_get_session_start_time(req->session); 724 | struct timeval last_transcription_time = df_get_session_last_transcription_time(req->session); 725 | struct timeval intent_detect_time = df_get_session_intent_detected_time(req->session); 726 | struct dialogflow_log_data log_data[] = { 727 | { "duration_ms", duration_buffer }, 728 | { "utterance_duration_ms", utterance_duration_buffer }, 729 | { "speech_rec_duration_ms", speech_rec_duration_buffer }, 730 | { "dialogflow_response_time_ms", dialogflow_response_time_buffer }, 731 | { "intent_latency_time_ms", intent_latency_time_buffer }, 732 | { "endpointer_barge_in_ms", endpointer_barge_in_time_buffer }, 733 | { "dialogflow_barge_in_ms", dialogflow_barge_in_time_buffer }, 734 | { "pre_recording", req->pre_recording_filename }, 735 | { "post_recording", req->post_recording_filename } 736 | }; 737 | 738 | sprintf(duration_buffer, "%lld", req->last_request_duration_ms); 739 | sprintf(utterance_duration_buffer, "%lld", req->last_audio_duration_ms); 740 | sprintf(speech_rec_duration_buffer, "%lld", tvdiff_ms_or_zero(last_transcription_time, dialogflow_start_time)); 741 | sprintf(dialogflow_response_time_buffer, "%lld", tvdiff_ms_or_zero(intent_detect_time, last_transcription_time)); 742 | sprintf(intent_latency_time_buffer, "%lld", tvdiff_ms_or_zero(intent_detect_time, req->endpointer_end_of_speech_time)); 743 | sprintf(endpointer_barge_in_time_buffer, "%lld", tvdiff_ms_or_zero(req->endpointer_barge_in_time, req->request_start)); 744 | sprintf(dialogflow_barge_in_time_buffer, "%lld", tvdiff_ms_or_zero(req->dialogflow_barge_in_time, req->request_start)); 745 | 746 | gdf_log_call_event(req->pvt, req, CALL_LOG_TYPE_RECOGNITION, "stop", ARRAY_LEN(log_data), log_data); 747 | } 748 | 749 | static int open_preendpointed_recording_file(struct gdf_request *req) 750 | { 751 | struct ast_str *path = build_log_related_filename_to_thread_local_str(req->pvt, req, "pre", "ul"); 752 | FILE *record_file; 753 | 754 | ao2_lock(req); 755 | req->utterance_preendpointer_recording_open_already_attempted = 1; 756 | ao2_unlock(req); 757 | 758 | record_file = fopen(ast_str_buffer(path), "w"); 759 | if (record_file) { 760 | struct dialogflow_log_data log_data[] = { 761 | { "filename", ast_str_buffer(path) } 762 | }; 763 | gdf_log_call_event(req->pvt, req, CALL_LOG_TYPE_ENDPOINTER, "pre_recording_start", ARRAY_LEN(log_data), log_data); 764 | ast_log(LOG_DEBUG, "Opened %s for preendpointer recording for %d@%s\n", ast_str_buffer(path), req->current_utterance_number, req->pvt->session_id); 765 | ao2_lock(req); 766 | ast_string_field_set(req, pre_recording_filename, ast_str_buffer(path)); 767 | req->utterance_preendpointer_recording_file_handle = record_file; 768 | ao2_unlock(req); 769 | } else { 770 | ast_log(LOG_WARNING, "Unable to open %s for preendpointer recording for %d@%s -- %d: %s\n", ast_str_buffer(path), req->current_utterance_number, req->pvt->session_id, errno, strerror(errno)); 771 | } 772 | 773 | return (record_file == NULL ? -1 : 0); 774 | } 775 | 776 | static int open_postendpointed_recording_file(struct gdf_request *req) 777 | { 778 | struct ast_str *path = build_log_related_filename_to_thread_local_str(req->pvt, req, "post", "ul"); 779 | FILE *record_file; 780 | 781 | ao2_lock(req); 782 | req->utterance_postendpointer_recording_open_already_attempted = 1; 783 | ao2_unlock(req); 784 | 785 | record_file = fopen(ast_str_buffer(path), "w"); 786 | if (record_file) { 787 | struct dialogflow_log_data log_data[] = { 788 | { "filename", ast_str_buffer(path) } 789 | }; 790 | gdf_log_call_event(req->pvt, req, CALL_LOG_TYPE_ENDPOINTER, "post_recording_start", ARRAY_LEN(log_data), log_data); 791 | ast_log(LOG_DEBUG, "Opened %s for postendpointer recording for %d@%s\n", ast_str_buffer(path), req->current_utterance_number, req->pvt->session_id); 792 | ao2_lock(req); 793 | ast_string_field_set(req, post_recording_filename, ast_str_buffer(path)); 794 | req->utterance_postendpointer_recording_file_handle = record_file; 795 | ao2_unlock(req); 796 | } else { 797 | ast_log(LOG_WARNING, "Unable to open %s for postendpointer recording for %d@%s -- %d: %s\n", ast_str_buffer(path), req->current_utterance_number, req->pvt->session_id, errno, strerror(errno)); 798 | } 799 | 800 | return (record_file == NULL ? -1 : 0); 801 | } 802 | 803 | static void coalesce_cached_audio_for_writing(struct gdf_request *req) 804 | { 805 | ao2_lock(req); 806 | if (req->mulaw_endpointer_audio_cache) { 807 | size_t end_amount_at_beginning_of_buffer; 808 | 809 | if (req->mulaw_endpointer_audio_cache_start == 0) { 810 | end_amount_at_beginning_of_buffer = 0; 811 | } else if (req->mulaw_endpointer_audio_cache_start + req->mulaw_endpointer_audio_cache_len > req->mulaw_endpointer_audio_cache_size) { 812 | end_amount_at_beginning_of_buffer = (req->mulaw_endpointer_audio_cache_start + req->mulaw_endpointer_audio_cache_len) - req->mulaw_endpointer_audio_cache_size; 813 | } else { 814 | ast_log(LOG_DEBUG, "Audio cache buffer for %d@%s not a full buffer but starts in middle\n", req->current_utterance_number, req->pvt->session_id); 815 | end_amount_at_beginning_of_buffer = 0; 816 | } 817 | 818 | if (end_amount_at_beginning_of_buffer == 0) { 819 | if (req->mulaw_endpointer_audio_cache_start == 0) { 820 | /* nothing to do */ 821 | } else { 822 | char *start_of_cached_audio = req->mulaw_endpointer_audio_cache + req->mulaw_endpointer_audio_cache_start; 823 | memmove(req->mulaw_endpointer_audio_cache, start_of_cached_audio, req->mulaw_endpointer_audio_cache_len); 824 | req->mulaw_endpointer_audio_cache_start = 0; 825 | } 826 | } else { 827 | char *cache = alloca(end_amount_at_beginning_of_buffer); 828 | char *start_of_cached_audio = req->mulaw_endpointer_audio_cache + req->mulaw_endpointer_audio_cache_start; 829 | size_t amount_of_audio_at_end_of_buffer = req->mulaw_endpointer_audio_cache_size - req->mulaw_endpointer_audio_cache_start; 830 | char *where_end_of_audio_will_go = req->mulaw_endpointer_audio_cache + amount_of_audio_at_end_of_buffer; 831 | 832 | memcpy(cache, req->mulaw_endpointer_audio_cache, end_amount_at_beginning_of_buffer); 833 | memmove(req->mulaw_endpointer_audio_cache, start_of_cached_audio, amount_of_audio_at_end_of_buffer); 834 | memcpy(where_end_of_audio_will_go, cache, end_amount_at_beginning_of_buffer); 835 | 836 | req->mulaw_endpointer_audio_cache_start = 0; 837 | } 838 | } 839 | ao2_unlock(req); 840 | } 841 | 842 | static void maybe_record_audio(struct gdf_request *req, const char *mulaw, size_t mulaw_len, enum VAD_STATE current_vad_state) 843 | { 844 | struct gdf_config *config = gdf_get_config(); 845 | int enable_preendpointer_recordings = 0; 846 | int enable_postendpointer_recordings = 0; 847 | int record_preendpointer_on_demand = 0; 848 | int currently_recording_preendpointed_audio = 0; 849 | int currently_recording_postendpointed_audio = 0; 850 | int already_attempted_open_for_preendpointed_audio = 0; 851 | int already_attempted_open_for_postendpointed_audio = 0; 852 | 853 | if (config) { 854 | enable_preendpointer_recordings = config->enable_preendpointer_recordings; 855 | enable_postendpointer_recordings = config->enable_postendpointer_recordings; 856 | record_preendpointer_on_demand = config->record_preendpointer_on_demand; 857 | ao2_t_ref(config, -1, "done with config checking for recording"); 858 | } 859 | 860 | enable_postendpointer_recordings |= req->record_utterance; 861 | if (record_preendpointer_on_demand) { 862 | enable_preendpointer_recordings |= req->record_utterance; 863 | } 864 | 865 | if (enable_postendpointer_recordings || enable_preendpointer_recordings) { 866 | int have_call_log_path; 867 | ao2_lock(req); 868 | have_call_log_path = !ast_strlen_zero(req->pvt->call_log_path); 869 | if (have_call_log_path) { 870 | currently_recording_preendpointed_audio = (req->utterance_preendpointer_recording_file_handle != NULL); 871 | already_attempted_open_for_preendpointed_audio = req->utterance_preendpointer_recording_open_already_attempted; 872 | currently_recording_postendpointed_audio = (req->utterance_postendpointer_recording_file_handle != NULL); 873 | already_attempted_open_for_postendpointed_audio = req->utterance_postendpointer_recording_open_already_attempted; 874 | } 875 | ao2_unlock(req); 876 | } 877 | 878 | if (enable_preendpointer_recordings) { 879 | if (!currently_recording_preendpointed_audio && !already_attempted_open_for_preendpointed_audio) { 880 | if (!open_preendpointed_recording_file(req)) { 881 | currently_recording_preendpointed_audio = 1; 882 | } 883 | } 884 | if (currently_recording_preendpointed_audio) { 885 | size_t written = fwrite(mulaw, sizeof(char), mulaw_len, req->utterance_preendpointer_recording_file_handle); 886 | if (written < mulaw_len) { 887 | ast_log(LOG_WARNING, "Only wrote %d of %d bytes for pre-endpointed recording for%d@%s\n", 888 | (int) written, (int) mulaw_len, req->current_utterance_number, req->pvt->session_id); 889 | } 890 | } 891 | } 892 | 893 | if (enable_postendpointer_recordings && current_vad_state != VAD_STATE_START) { 894 | int need_to_dump_cached_audio = 0; 895 | if (!currently_recording_postendpointed_audio && !already_attempted_open_for_postendpointed_audio) { 896 | if (!open_postendpointed_recording_file(req)) { 897 | currently_recording_postendpointed_audio = 1; 898 | need_to_dump_cached_audio = 1; 899 | } 900 | } 901 | if (need_to_dump_cached_audio) { 902 | size_t written; 903 | coalesce_cached_audio_for_writing(req); 904 | written = fwrite(req->mulaw_endpointer_audio_cache, sizeof(char), req->mulaw_endpointer_audio_cache_len, req->utterance_postendpointer_recording_file_handle); 905 | if (written < req->mulaw_endpointer_audio_cache_len) { 906 | ast_log(LOG_WARNING, "Only wrote %d of %d bytes for cached post-endpointed recording for %d@%s\n", 907 | (int) written, (int) req->mulaw_endpointer_audio_cache_len, req->current_utterance_number, req->pvt->session_id); 908 | } 909 | } 910 | if (currently_recording_postendpointed_audio) { 911 | size_t written = fwrite(mulaw, sizeof(char), mulaw_len, req->utterance_postendpointer_recording_file_handle); 912 | if (written < mulaw_len) { 913 | ast_log(LOG_WARNING, "Only wrote %d of %d bytes for post-endpointed recording for %d@%s\n", 914 | (int) written, (int) mulaw_len, req->current_utterance_number, req->pvt->session_id); 915 | } 916 | } 917 | } 918 | } 919 | 920 | static void maybe_cache_preendpointed_audio(struct gdf_request *req, const char *mulaw, size_t mulaw_len, enum VAD_STATE vad_state) 921 | { 922 | if (vad_state == VAD_STATE_START) { 923 | ao2_lock(req); 924 | if (req->mulaw_endpointer_audio_cache) { 925 | size_t relative_write_location; 926 | char *write_location; 927 | 928 | if (req->mulaw_endpointer_audio_cache_len + mulaw_len > req->mulaw_endpointer_audio_cache_size) { 929 | size_t space_needed = req->mulaw_endpointer_audio_cache_len + mulaw_len - req->mulaw_endpointer_audio_cache_size; 930 | req->mulaw_endpointer_audio_cache_start += space_needed; 931 | req->mulaw_endpointer_audio_cache_len -= space_needed; 932 | if (req->mulaw_endpointer_audio_cache_start >= req->mulaw_endpointer_audio_cache_size) { 933 | req->mulaw_endpointer_audio_cache_start -= req->mulaw_endpointer_audio_cache_size; 934 | } 935 | } 936 | 937 | relative_write_location = req->mulaw_endpointer_audio_cache_start + req->mulaw_endpointer_audio_cache_len; 938 | if (relative_write_location >= req->mulaw_endpointer_audio_cache_size) { 939 | relative_write_location -= req->mulaw_endpointer_audio_cache_size; 940 | } 941 | 942 | write_location = req->mulaw_endpointer_audio_cache + relative_write_location; 943 | 944 | memcpy(write_location, mulaw, mulaw_len); 945 | req->mulaw_endpointer_audio_cache_len += mulaw_len; 946 | } 947 | ao2_unlock(req); 948 | } 949 | } 950 | 951 | static void close_preendpointed_audio_recording(struct gdf_request *req) 952 | { 953 | if (req->utterance_preendpointer_recording_file_handle) { 954 | fclose(req->utterance_preendpointer_recording_file_handle); 955 | req->utterance_preendpointer_recording_file_handle = NULL; 956 | gdf_log_call_event_only(req->pvt, req, CALL_LOG_TYPE_ENDPOINTER, "pre_recording_stop"); 957 | } 958 | req->utterance_preendpointer_recording_open_already_attempted = 0; 959 | } 960 | 961 | static void close_postendpointed_audio_recording(struct gdf_request *req) 962 | { 963 | if (req->utterance_postendpointer_recording_file_handle) { 964 | fclose(req->utterance_postendpointer_recording_file_handle); 965 | req->utterance_postendpointer_recording_file_handle = NULL; 966 | gdf_log_call_event_only(req->pvt, req, CALL_LOG_TYPE_ENDPOINTER, "post_recording_stop"); 967 | } 968 | req->utterance_postendpointer_recording_open_already_attempted = 0; 969 | } 970 | 971 | static int gdf_stop_recognition(struct gdf_request *req) 972 | { 973 | close_preendpointed_audio_recording(req); 974 | close_postendpointed_audio_recording(req); 975 | df_stop_recognition(req->session); 976 | ao2_lock(req->pvt); 977 | if (req == req->pvt->current_request) { 978 | if (req->pvt->speech) { 979 | ast_speech_change_state(req->pvt->speech, AST_SPEECH_STATE_DONE); /* okay to call this locked */ 980 | } 981 | } 982 | ao2_unlock(req->pvt); 983 | req->last_request_duration_ms = ast_tvdiff_ms(ast_tvnow(), req->request_start); 984 | write_end_of_recognition_call_event(req); 985 | return 0; 986 | } 987 | 988 | /* speech structure is locked */ 989 | static int gdf_write(struct ast_speech *speech, void *data, int len) 990 | { 991 | struct gdf_pvt *pvt = speech->data; 992 | int res = 0; 993 | struct ast_frame f; 994 | struct ast_frame *iso; 995 | 996 | memset(&f, 0, sizeof(f)); 997 | 998 | f.frametype = AST_FRAME_VOICE; 999 | #ifdef ASTERISK_13_OR_LATER 1000 | f.subclass.format = ast_format_slin; 1001 | #else 1002 | f.subclass.codec = AST_FORMAT_SLINEAR; 1003 | #endif 1004 | f.data.ptr = data; 1005 | f.datalen = len; 1006 | f.samples = len / 2; 1007 | f.src = "gdf_write"; 1008 | 1009 | iso = ast_frisolate(&f); 1010 | if (iso) { 1011 | ao2_lock(pvt); 1012 | if (pvt->current_request) { 1013 | ao2_lock(pvt->current_request); 1014 | AST_LIST_INSERT_TAIL(&pvt->current_request->frame_queue, iso, frame_list); 1015 | pvt->current_request->frame_queue_len++; 1016 | ao2_unlock(pvt->current_request); 1017 | } else { 1018 | ast_frfree(iso); 1019 | } 1020 | ao2_unlock(pvt); 1021 | } else { 1022 | ast_log(LOG_WARNING, "Error isolating frame for write to %s\n", 1023 | pvt->session_id); 1024 | } 1025 | 1026 | return res; 1027 | } 1028 | 1029 | static int gdf_dtmf(struct ast_speech *speech, const char *dtmf) 1030 | { 1031 | return -1; 1032 | } 1033 | 1034 | static int should_start_call_log(struct gdf_pvt *pvt) 1035 | { 1036 | int should_start; 1037 | ao2_lock(pvt); 1038 | should_start = !pvt->call_log_open_already_attempted; 1039 | ao2_unlock(pvt); 1040 | if (should_start) { 1041 | struct gdf_config *cfg; 1042 | cfg = gdf_get_config(); 1043 | if (cfg) { 1044 | should_start &= cfg->enable_call_logs; 1045 | ao2_t_ref(cfg, -1, "done checking for starting call log"); 1046 | } 1047 | } 1048 | return should_start; 1049 | } 1050 | 1051 | AST_THREADSTORAGE(call_log_path); 1052 | static void calculate_log_path(struct gdf_pvt *pvt) 1053 | { 1054 | struct varshead var_head = { .first = NULL, .last = NULL }; 1055 | struct ast_var_t *var; 1056 | struct gdf_config *cfg; 1057 | 1058 | ao2_lock(pvt); 1059 | var = ast_var_assign("APPLICATION", pvt->call_logging_application_name); 1060 | ao2_unlock(pvt); 1061 | 1062 | AST_LIST_INSERT_HEAD(&var_head, var, entries); 1063 | 1064 | cfg = gdf_get_config(); 1065 | if (cfg) { 1066 | struct ast_str *path = ast_str_thread_get(&call_log_path, 256); 1067 | 1068 | ast_str_substitute_variables_varshead(&path, 0, &var_head, cfg->call_log_location); 1069 | 1070 | ao2_lock(pvt); 1071 | ast_string_field_set(pvt, call_log_path, ast_str_buffer(path)); 1072 | ao2_unlock(pvt); 1073 | 1074 | ao2_t_ref(cfg, -1, "done with config in calculating call log path"); 1075 | } 1076 | 1077 | ast_var_delete(var); 1078 | } 1079 | 1080 | static void calculate_log_file_basename(struct gdf_pvt *pvt) 1081 | { 1082 | struct timeval t; 1083 | struct ast_tm now; 1084 | 1085 | t = ast_tvnow(); 1086 | ast_localtime(&t, &now, NULL); 1087 | ast_string_field_build(pvt, call_log_file_basename, "%02d%02d_%s", now.tm_min, now.tm_sec, pvt->session_id); 1088 | } 1089 | 1090 | static void mkdir_log_path(struct gdf_pvt *pvt) 1091 | { 1092 | ast_mkdir(pvt->call_log_path, 0755); 1093 | } 1094 | 1095 | static struct ast_str *build_log_related_filename_to_thread_local_str(struct gdf_pvt *pvt, struct gdf_request *req, const char *type, const char *extension) 1096 | { 1097 | struct ast_str *path; 1098 | path = ast_str_thread_get(&call_log_path, 256); 1099 | ao2_lock(pvt); 1100 | ast_str_set(&path, 0, "%s", pvt->call_log_path); 1101 | ast_str_append(&path, 0, "%s", pvt->call_log_file_basename); 1102 | ast_str_append(&path, 0, "_%s", type); 1103 | if (req) { 1104 | ast_str_append(&path, 0, "_%d", req->current_utterance_number); 1105 | } 1106 | ast_str_append(&path, 0, ".%s" , extension); 1107 | ao2_unlock(pvt); 1108 | return path; 1109 | } 1110 | 1111 | static void start_call_log(struct gdf_pvt *pvt) 1112 | { 1113 | ao2_lock(pvt); 1114 | if (pvt->call_log_open_already_attempted) { 1115 | ao2_unlock(pvt); 1116 | return; 1117 | } 1118 | pvt->call_log_open_already_attempted = 1; 1119 | ao2_unlock(pvt); 1120 | 1121 | calculate_log_path(pvt); 1122 | calculate_log_file_basename(pvt); 1123 | 1124 | if (!ast_strlen_zero(pvt->call_log_path)) { 1125 | struct ast_str *path; 1126 | FILE *log_file; 1127 | 1128 | mkdir_log_path(pvt); 1129 | 1130 | path = build_log_related_filename_to_thread_local_str(pvt, NULL, "log", "jsonl"); 1131 | 1132 | log_file = fopen(ast_str_buffer(path), "w"); 1133 | if (log_file) { 1134 | char hostname[HOST_NAME_MAX] = ""; 1135 | struct dialogflow_log_data log_data[] = { 1136 | { "application", pvt->call_logging_application_name }, 1137 | { "hostname", hostname } 1138 | }; 1139 | 1140 | gethostname(hostname, sizeof(hostname) - 1); 1141 | 1142 | ast_log(LOG_DEBUG, "Opened %s for call log for %s\n", ast_str_buffer(path), pvt->session_id); 1143 | ao2_lock(pvt); 1144 | pvt->call_log_file_handle = log_file; 1145 | ao2_unlock(pvt); 1146 | 1147 | gdf_log_call_event(pvt, NULL, CALL_LOG_TYPE_SESSION, "start", ARRAY_LEN(log_data), log_data); 1148 | } else { 1149 | ast_log(LOG_WARNING, "Unable to open %s for writing call log for %s -- %d: %s\n", ast_str_buffer(path), pvt->session_id, errno, strerror(errno)); 1150 | } 1151 | } else { 1152 | ast_log(LOG_WARNING, "Not starting call log, path is empty\n"); 1153 | } 1154 | } 1155 | 1156 | static void log_endpointer_start_event(struct gdf_request *req) 1157 | { 1158 | char threshold[11]; 1159 | char voice_duration[11]; 1160 | char silence_duration[11]; 1161 | char barge_duration[11]; 1162 | char end_of_speech_duration[11]; 1163 | struct dialogflow_log_data log_data[] = { 1164 | { VAD_PROP_VOICE_THRESHOLD, threshold }, 1165 | { VAD_PROP_VOICE_DURATION, voice_duration }, 1166 | { VAD_PROP_SILENCE_DURATION, silence_duration }, 1167 | { VAD_PROP_BARGE_DURATION, barge_duration }, 1168 | { VAD_PROP_END_OF_SPEECH_DURATION, end_of_speech_duration } 1169 | }; 1170 | 1171 | sprintf(threshold, "%d", req->voice_threshold); 1172 | sprintf(voice_duration, "%d", req->voice_minimum_duration); 1173 | sprintf(silence_duration, "%d", req->silence_minimum_duration); 1174 | sprintf(barge_duration, "%d", req->barge_in_minimum_duration); 1175 | sprintf(end_of_speech_duration, "%d", req->end_of_speech_minimum_silence); 1176 | 1177 | gdf_log_call_event(req->pvt, req, CALL_LOG_TYPE_ENDPOINTER, "start", ARRAY_LEN(log_data), log_data); 1178 | } 1179 | 1180 | static int gdf_start(struct ast_speech *speech) 1181 | { 1182 | struct gdf_pvt *pvt = speech->data; 1183 | 1184 | if (should_start_call_log(pvt)) { 1185 | start_call_log(pvt); 1186 | } 1187 | 1188 | ao2_lock(pvt); 1189 | if (pvt->current_request) { 1190 | ao2_t_ref(pvt->current_request, -1, "Cancel in-progress request for a new one"); 1191 | pvt->current_request = NULL; 1192 | } 1193 | pvt->current_request = create_new_request(pvt, pvt->utterance_counter++); 1194 | ao2_unlock(pvt); 1195 | if (pvt->current_request == NULL) { 1196 | ast_speech_change_state(pvt->speech, AST_SPEECH_STATE_DONE); 1197 | return -1; 1198 | } 1199 | 1200 | return 0; 1201 | } 1202 | 1203 | static void maybe_signal_speaking(struct gdf_request *req, enum dialogflow_session_state state) 1204 | { 1205 | ao2_lock(req->pvt); 1206 | if (req->pvt->current_request == req) { 1207 | if (state == DF_STATE_STARTED && req->pvt->speech && !ast_test_flag(req->pvt->speech, AST_SPEECH_SPOKE)) { 1208 | ast_log(LOG_DEBUG, "Setting heard speech on %d@%s\n", req->current_utterance_number, req->pvt->session_id); 1209 | ast_set_flag(req->pvt->speech, AST_SPEECH_QUIET); 1210 | ast_set_flag(req->pvt->speech, AST_SPEECH_SPOKE); 1211 | } 1212 | } 1213 | ao2_unlock(req->pvt); 1214 | } 1215 | 1216 | static void mark_request_done(struct gdf_request *req) 1217 | { 1218 | ao2_lock(req); 1219 | req->state = GDFE_STATE_DONE; 1220 | ao2_unlock(req); 1221 | } 1222 | 1223 | static int write_audio_frame(struct gdf_request *req, void *data, int len) 1224 | { 1225 | enum dialogflow_session_state state; 1226 | enum VAD_STATE vad_state; 1227 | #ifdef RES_SPEECH_GDFE_DEBUG_VAD 1228 | enum VAD_STATE orig_vad_state; 1229 | #endif 1230 | int threshold; 1231 | int cur_duration; 1232 | int change_duration; 1233 | int avg_level; 1234 | int voice_duration; 1235 | int silence_duration; 1236 | int barge_duration; 1237 | int end_of_speech_duration; 1238 | int heard_speech; 1239 | milliseconds_t incomplete_timeout; 1240 | milliseconds_t no_speech_timeout; 1241 | milliseconds_t maximum_speech_timeout; 1242 | int datasamples; 1243 | int datams; 1244 | int mulaw_len; 1245 | char *mulaw; 1246 | int i; 1247 | int start_recognition_on_start = 0; 1248 | int recognition_start_failure_retries = 0; 1249 | int recognition_start_failure_retry_max_time_ms = 0; 1250 | const char *start_failure_retry_codes = ""; 1251 | struct gdf_config *cfg; 1252 | int signal_end_of_speech = 0; 1253 | 1254 | datasamples = len / sizeof(short); /* 2 bytes per sample for slin */; 1255 | datams = datasamples / 8; /* 8 samples per millisecond */; 1256 | mulaw_len = datasamples * sizeof(char); 1257 | mulaw = alloca(mulaw_len); 1258 | 1259 | ao2_lock(req); 1260 | #ifdef RES_SPEECH_GDFE_DEBUG_VAD 1261 | orig_vad_state = req->vad_state; 1262 | #endif 1263 | vad_state = req->vad_state; 1264 | threshold = req->voice_threshold; 1265 | cur_duration = req->vad_state_duration; 1266 | change_duration = req->vad_change_duration; 1267 | voice_duration = req->voice_minimum_duration; 1268 | silence_duration = req->silence_minimum_duration; 1269 | barge_duration = req->barge_in_minimum_duration; 1270 | end_of_speech_duration = req->end_of_speech_minimum_silence; 1271 | heard_speech = req->heard_speech; 1272 | incomplete_timeout = req->incomplete_timeout; 1273 | no_speech_timeout = req->no_speech_timeout; 1274 | maximum_speech_timeout = req->maximum_speech_timeout; 1275 | ao2_unlock(req); 1276 | 1277 | cfg = gdf_get_config(); 1278 | if (cfg) { 1279 | start_recognition_on_start = cfg->start_recognition_on_start; 1280 | recognition_start_failure_retries = cfg->recognition_start_failure_retries; 1281 | recognition_start_failure_retry_max_time_ms = cfg->recognition_start_failure_retry_max_time_ms; 1282 | start_failure_retry_codes = ast_strdupa(cfg->start_failure_retry_codes); 1283 | ao2_t_ref(cfg, -1, "done checking for starting rec on call start"); 1284 | } 1285 | 1286 | state = df_get_state(req->session); 1287 | 1288 | cur_duration += datams; 1289 | 1290 | avg_level = calculate_audio_level((short *)data, datasamples); 1291 | if (avg_level >= threshold) { 1292 | if (vad_state != VAD_STATE_SPEAK) { 1293 | change_duration += datams; 1294 | } else { 1295 | change_duration = 0; 1296 | } 1297 | } else { 1298 | if (vad_state != VAD_STATE_SPEAK) { 1299 | change_duration = 0; 1300 | } else { 1301 | change_duration += datams; 1302 | } 1303 | } 1304 | 1305 | if (vad_state == VAD_STATE_START) { 1306 | if (change_duration >= voice_duration) { 1307 | /* speaking */ 1308 | vad_state = VAD_STATE_SPEAK; 1309 | cur_duration = change_duration; 1310 | change_duration = 0; 1311 | ao2_lock(req); 1312 | req->speech_start = ast_tvnow(); 1313 | ao2_unlock(req); 1314 | gdf_log_call_event_only(req->pvt, req, CALL_LOG_TYPE_ENDPOINTER, "start_of_speech"); 1315 | } 1316 | } else if (vad_state == VAD_STATE_SPEAK) { 1317 | if (change_duration >= silence_duration) { 1318 | vad_state = VAD_STATE_SILENT; 1319 | cur_duration = change_duration; 1320 | change_duration = 0; 1321 | } else if (cur_duration >= barge_duration) { 1322 | if (!heard_speech) { 1323 | heard_speech = 1; 1324 | gdf_log_call_event_only(req->pvt, req, CALL_LOG_TYPE_ENDPOINTER, "barge_in"); 1325 | maybe_signal_speaking(req, state); 1326 | } 1327 | if (ast_tvzero(req->endpointer_barge_in_time)) { 1328 | req->endpointer_barge_in_time = ast_tvnow(); 1329 | } 1330 | } 1331 | } else if (vad_state == VAD_STATE_SILENT) { 1332 | if (change_duration >= voice_duration) { 1333 | vad_state = VAD_STATE_SPEAK; 1334 | cur_duration = change_duration; 1335 | change_duration = 0; 1336 | } else if (heard_speech && cur_duration >= end_of_speech_duration) { 1337 | vad_state = VAD_STATE_END; 1338 | gdf_log_call_event_only(req->pvt, req, CALL_LOG_TYPE_ENDPOINTER, "end_of_speech"); 1339 | ao2_lock(req); 1340 | req->endpointer_end_of_speech_time = ast_tvnow(); 1341 | signal_end_of_speech = req->pvt->use_internal_endpointer_for_end_of_speech; 1342 | ao2_unlock(req); 1343 | } 1344 | } 1345 | 1346 | ao2_lock(req); 1347 | req->vad_state = vad_state; 1348 | req->vad_state_duration = cur_duration; 1349 | req->vad_change_duration = change_duration; 1350 | req->heard_speech = heard_speech; 1351 | ao2_unlock(req); 1352 | 1353 | #ifdef RES_SPEECH_GDFE_DEBUG_VAD 1354 | ast_log(LOG_DEBUG, "avg: %d thr: %d dur: %d chg: %d vce: %d sil: %d old: %d new: %d\n", 1355 | avg_level, threshold, cur_duration, change_duration, voice_duration, silence_duration, 1356 | orig_vad_state, vad_state); 1357 | #endif 1358 | 1359 | if (state == DF_STATE_READY && start_recognition_on_start) { 1360 | if (ast_tvzero(req->recognition_initial_attempt)) { 1361 | req->recognition_initial_attempt = ast_tvnow(); 1362 | } 1363 | if (df_start_recognition(req->session, req->language, 0, (const char **)req->pvt->hints, req->pvt->hint_count)) { 1364 | int will_retry = 0; 1365 | long long retry_duration = ast_tvdiff_ms(ast_tvnow(), req->recognition_initial_attempt); 1366 | if (req->current_start_retry < recognition_start_failure_retries && retry_duration < recognition_start_failure_retry_max_time_ms) { 1367 | int results; 1368 | int result_number; 1369 | ast_log(LOG_DEBUG, "Error pre-starting recognition on %d@%s -- might retry\n", req->current_utterance_number, req->pvt->session_id); 1370 | df_stop_recognition(req->session); 1371 | results = df_get_result_count(req->session); 1372 | for (result_number = 0; result_number < results; result_number++) { 1373 | struct dialogflow_result *df_result = df_get_result(req->session, result_number); 1374 | if (!strcasecmp(df_result->slot, "error_code")) { 1375 | size_t code_len = strlen(df_result->value); 1376 | char *comma_code = alloca(code_len + 3); 1377 | 1378 | sprintf(comma_code, ",%s,", df_result->value); 1379 | if (strstr(start_failure_retry_codes, comma_code)) { 1380 | will_retry = 1; 1381 | } 1382 | break; 1383 | } 1384 | } 1385 | } 1386 | if (will_retry) { 1387 | ast_log(LOG_DEBUG, "Error pre-starting recognition on %d@%s -- will retry\n", req->current_utterance_number, req->pvt->session_id); 1388 | req->current_start_retry++; 1389 | vad_state = VAD_STATE_START; 1390 | } else { 1391 | ast_log(LOG_WARNING, "Error pre-starting recognition on %d@%s\n", req->current_utterance_number, req->pvt->session_id); 1392 | mark_request_done(req); 1393 | } 1394 | } 1395 | ao2_lock(req); 1396 | req->last_audio_duration_ms = 0; 1397 | ao2_unlock(req); 1398 | } 1399 | 1400 | for (i = 0; i < datasamples; i++) { 1401 | mulaw[i] = AST_LIN2MU(((short *)data)[i]); 1402 | } 1403 | 1404 | maybe_record_audio(req, mulaw, mulaw_len, vad_state); 1405 | maybe_cache_preendpointed_audio(req, mulaw, mulaw_len, vad_state); 1406 | 1407 | state = df_get_state(req->session); 1408 | if (vad_state != VAD_STATE_START) { 1409 | if (state == DF_STATE_READY) { 1410 | if (!start_recognition_on_start) { 1411 | if (ast_tvzero(req->recognition_initial_attempt)) { 1412 | req->recognition_initial_attempt = ast_tvnow(); 1413 | } 1414 | if (df_start_recognition(req->session, req->language, 0, (const char **)req->pvt->hints, req->pvt->hint_count)) { 1415 | int will_retry = 0; 1416 | long long retry_duration = ast_tvdiff_ms(ast_tvnow(), req->recognition_initial_attempt); 1417 | if (req->current_start_retry < recognition_start_failure_retries && retry_duration < recognition_start_failure_retry_max_time_ms) { 1418 | int results; 1419 | int result_number; 1420 | ast_log(LOG_DEBUG, "Error starting recognition on %d@%s -- might retry\n", req->current_utterance_number, req->pvt->session_id); 1421 | df_stop_recognition(req->session); 1422 | results = df_get_result_count(req->session); 1423 | for (result_number = 0; result_number < results; result_number++) { 1424 | struct dialogflow_result *df_result = df_get_result(req->session, result_number); 1425 | if (!strcasecmp(df_result->slot, "error_code")) { 1426 | size_t code_len = strlen(df_result->value); 1427 | char *comma_code = alloca(code_len + 3); 1428 | 1429 | sprintf(comma_code, ",%s,", df_result->value); 1430 | if (strstr(start_failure_retry_codes, comma_code)) { 1431 | will_retry = 1; 1432 | } 1433 | break; 1434 | } 1435 | } 1436 | } 1437 | if (will_retry) { 1438 | size_t new_audio_cache_size; 1439 | char *new_audio_cache; 1440 | ast_log(LOG_DEBUG, "Error starting recognition on %d@%s -- will retry\n", req->current_utterance_number, req->pvt->session_id); 1441 | req->current_start_retry++; 1442 | coalesce_cached_audio_for_writing(req); 1443 | new_audio_cache_size = req->mulaw_endpointer_audio_cache_size + 160; /* 20 ms more */ 1444 | new_audio_cache = ast_realloc(req->mulaw_endpointer_audio_cache, new_audio_cache_size); 1445 | if (new_audio_cache) { 1446 | req->mulaw_endpointer_audio_cache_size = new_audio_cache_size; 1447 | req->mulaw_endpointer_audio_cache = new_audio_cache; 1448 | } else { 1449 | ast_log(LOG_WARNING, "Unable to resize audio cache for %d@%s -- will lose audio\n", req->current_utterance_number, req->pvt->session_id); 1450 | } 1451 | } else { 1452 | ast_log(LOG_WARNING, "Error starting recognition on %d@%s\n", req->current_utterance_number, req->pvt->session_id); 1453 | mark_request_done(req); 1454 | } 1455 | } 1456 | ao2_lock(req); 1457 | req->last_audio_duration_ms = 0; 1458 | ao2_unlock(req); 1459 | } 1460 | 1461 | state = df_get_state(req->session); 1462 | 1463 | if (state == DF_STATE_STARTED) { 1464 | size_t flush_start = 0; 1465 | 1466 | coalesce_cached_audio_for_writing(req); 1467 | 1468 | while (flush_start < req->mulaw_endpointer_audio_cache_len && state != DF_STATE_FINISHED && state != DF_STATE_ERROR) { 1469 | if (flush_start + mulaw_len <= req->mulaw_endpointer_audio_cache_len) { 1470 | state = df_write_audio(req->session, req->mulaw_endpointer_audio_cache + flush_start, mulaw_len); 1471 | flush_start += mulaw_len; 1472 | } else { 1473 | size_t partial_write_size = req->mulaw_endpointer_audio_cache_len - flush_start; 1474 | state = df_write_audio(req->session, req->mulaw_endpointer_audio_cache + flush_start, partial_write_size); 1475 | flush_start += partial_write_size; 1476 | } 1477 | } 1478 | 1479 | ao2_lock(req); 1480 | req->last_audio_duration_ms += flush_start / 8; 1481 | ao2_unlock(req); 1482 | } 1483 | } 1484 | 1485 | if (state == DF_STATE_STARTED) { 1486 | int response_count; 1487 | state = df_write_audio(req->session, mulaw, mulaw_len); 1488 | 1489 | response_count = df_get_response_count(req->session); 1490 | 1491 | if (!heard_speech && response_count > 0) { 1492 | heard_speech = 1; 1493 | gdf_log_call_event_only(req->pvt, req, CALL_LOG_TYPE_ENDPOINTER, "auto_barge_in"); 1494 | maybe_signal_speaking(req, state); 1495 | } 1496 | if (ast_tvzero(req->dialogflow_barge_in_time) && response_count > 0) { 1497 | req->dialogflow_barge_in_time = ast_tvnow(); 1498 | } 1499 | 1500 | ao2_lock(req); 1501 | req->last_audio_duration_ms += mulaw_len / 8; 1502 | req->heard_speech = heard_speech; 1503 | ao2_unlock(req); 1504 | 1505 | if (incomplete_timeout > 0 && response_count > 0) { 1506 | struct timeval now = ast_tvnow(); 1507 | struct timeval last_transcription_time = df_get_session_last_transcription_time(req->session); 1508 | 1509 | if (ast_tvdiff_ms(now, last_transcription_time) > incomplete_timeout) { 1510 | gdf_log_call_event_only(req->pvt, req, CALL_LOG_TYPE_RECOGNITION, "cancelled_incomplete"); 1511 | mark_request_done(req); 1512 | } 1513 | } 1514 | if (no_speech_timeout > 0 && heard_speech && response_count == 0) { 1515 | /* we heard speech but have gotten no speech responses */ 1516 | struct timeval now = ast_tvnow(); 1517 | struct timeval barge_in_time = req->endpointer_barge_in_time; 1518 | 1519 | if (ast_tvdiff_ms(now, barge_in_time) > no_speech_timeout) { 1520 | gdf_log_call_event_only(req->pvt, req, CALL_LOG_TYPE_RECOGNITION, "cancelled_no_speech"); 1521 | mark_request_done(req); 1522 | } 1523 | } 1524 | if (maximum_speech_timeout > 0 && !ast_tvzero(req->speech_start)) { 1525 | struct timeval now = ast_tvnow(); 1526 | struct timeval speech_start = req->speech_start; 1527 | 1528 | if (ast_tvdiff_ms(now, speech_start) > maximum_speech_timeout) { 1529 | gdf_log_call_event_only(req->pvt, req, CALL_LOG_TYPE_RECOGNITION, "cancelled_max_speech"); 1530 | mark_request_done(req); 1531 | } 1532 | } 1533 | } 1534 | } 1535 | if (signal_end_of_speech || state == DF_STATE_FINISHED || state == DF_STATE_ERROR) { 1536 | mark_request_done(req); 1537 | } 1538 | 1539 | return 0; 1540 | } 1541 | 1542 | static int start_dialogflow_recognition(struct gdf_request *req) 1543 | { 1544 | char *event = NULL; 1545 | char *language = NULL; 1546 | char *project_id = NULL; 1547 | char *endpoint = NULL; 1548 | char *service_key = NULL; 1549 | char *model = NULL; 1550 | int request_sentiment_analysis; 1551 | int use_internal_endpointer_for_end_of_speech; 1552 | enum SENTIMENT_ANALYSIS_STATE sentiment_analysis_state; 1553 | 1554 | ao2_lock(req->pvt); 1555 | if (req->pvt->current_request != req) { 1556 | ao2_unlock(req->pvt); 1557 | return -1; 1558 | } 1559 | ao2_unlock(req->pvt); 1560 | 1561 | event = ast_strdupa(req->event); 1562 | language = ast_strdupa(req->language); 1563 | project_id = ast_strdupa(req->project_id); 1564 | endpoint = ast_strdupa(req->endpoint); 1565 | service_key = ast_strdupa(req->service_key); 1566 | model = ast_strdupa(req->model); 1567 | request_sentiment_analysis = req->pvt->request_sentiment_analysis; 1568 | sentiment_analysis_state = req->pvt->effective_sentiment_analysis_state; 1569 | use_internal_endpointer_for_end_of_speech = req->pvt->use_internal_endpointer_for_end_of_speech; 1570 | 1571 | req->vad_state = VAD_STATE_START; 1572 | req->vad_state_duration = 0; 1573 | req->vad_change_duration = 0; 1574 | req->request_start = ast_tvnow(); 1575 | req->state = GDFE_STATE_PROCESSING; 1576 | 1577 | df_set_project_id(req->session, project_id); 1578 | df_set_endpoint(req->session, endpoint); 1579 | df_set_auth_key(req->session, service_key); 1580 | 1581 | if (request_sentiment_analysis) { 1582 | if (sentiment_analysis_state == SENTIMENT_ANALYSIS_NEVER) { 1583 | ast_log(LOG_DEBUG, "Refusing to do sentiment analysis on %d@%s due to configuration prohibition.\n", req->current_utterance_number, req->pvt->session_id); 1584 | request_sentiment_analysis = 0; 1585 | } 1586 | } else { 1587 | if (sentiment_analysis_state == SENTIMENT_ANALYSIS_ALWAYS) { 1588 | ast_log(LOG_DEBUG, "Forcing sentiment analysis on %d@%s due to configuration.\n", req->current_utterance_number, req->pvt->session_id); 1589 | request_sentiment_analysis = 1; 1590 | } 1591 | } 1592 | ast_log(LOG_DEBUG, "%sequesting sentiment analysis on %d@%s\n", request_sentiment_analysis ? "R" : "Not r", req->current_utterance_number, req->pvt->session_id); 1593 | df_set_request_sentiment_analysis(req->session, request_sentiment_analysis); 1594 | df_set_use_external_endpointer(req->session, use_internal_endpointer_for_end_of_speech); 1595 | df_set_model(req->session, model); 1596 | 1597 | { 1598 | char utterance_number[11]; 1599 | struct dialogflow_log_data log_data[] = { 1600 | { "event", event }, 1601 | { "language", language }, 1602 | { "project_id", project_id }, 1603 | { "logical_agent_name", req->pvt->logical_agent_name }, 1604 | { "utterance", utterance_number }, 1605 | { "context", req->pvt->call_logging_context }, 1606 | { "application", req->pvt->call_logging_application_name } 1607 | }; 1608 | sprintf(utterance_number, "%d", req->current_utterance_number); 1609 | gdf_log_call_event(req->pvt, req, CALL_LOG_TYPE_RECOGNITION, "start", ARRAY_LEN(log_data), log_data); 1610 | } 1611 | log_endpointer_start_event(req); 1612 | 1613 | if (!ast_strlen_zero(event)) { 1614 | if (df_recognize_event(req->session, event, language, 0)) { 1615 | ast_log(LOG_WARNING, "Error recognizing event on %d@%s\n", req->current_utterance_number, req->pvt->session_id); 1616 | ao2_lock(req->pvt); 1617 | if (req->pvt->current_request == req && req->pvt->speech) { 1618 | ast_speech_change_state(req->pvt->speech, AST_SPEECH_STATE_DONE); 1619 | } 1620 | req->state = GDFE_STATE_HAVE_RESULTS; 1621 | ao2_unlock(req->pvt); 1622 | } else { 1623 | ao2_lock(req); 1624 | req->state = GDFE_STATE_DONE; 1625 | ao2_unlock(req); 1626 | } 1627 | } else { 1628 | df_connect(req->session); 1629 | } 1630 | 1631 | return 0; 1632 | } 1633 | 1634 | static void *gdf_exec(void *arg) 1635 | { 1636 | struct gdf_request *req = arg; 1637 | 1638 | ast_log(LOG_DEBUG, "Starting background thread for GDF %d@%s\n", req->current_utterance_number, req->pvt->session_id); 1639 | 1640 | ao2_lock(req); 1641 | while (req->state != GDFE_STATE_DONE) 1642 | { 1643 | struct timespec ts; 1644 | int time_sleep_ms = 20; 1645 | int cancelled; 1646 | 1647 | ao2_unlock(req); 1648 | 1649 | ts.tv_sec = time_sleep_ms / 1000; 1650 | ts.tv_nsec = (time_sleep_ms % 1000) * 1000000; 1651 | 1652 | nanosleep(&ts, NULL); 1653 | 1654 | ao2_lock(req); 1655 | if (req->state == GDFE_STATE_START) { 1656 | ao2_unlock(req); 1657 | start_dialogflow_recognition(req); 1658 | ao2_lock(req); 1659 | } 1660 | ao2_unlock(req); 1661 | ao2_lock(req->pvt); 1662 | cancelled = req->pvt->current_request != req || (req->pvt->speech && req->pvt->speech->state == AST_SPEECH_STATE_NOT_READY); 1663 | ao2_unlock(req->pvt); 1664 | ao2_lock(req); 1665 | if (req->state == GDFE_STATE_PROCESSING && cancelled) { 1666 | ao2_unlock(req); 1667 | gdf_log_call_event_only(req->pvt, req, CALL_LOG_TYPE_RECOGNITION, "cancelled"); 1668 | ao2_lock(req); 1669 | req->state = GDFE_STATE_DONE; 1670 | } 1671 | while (req->state == GDFE_STATE_PROCESSING && req->frame_queue_len > 0) { 1672 | struct ast_frame *f = AST_LIST_REMOVE_HEAD(&req->frame_queue, frame_list); 1673 | req->frame_queue_len--; 1674 | if (f) { 1675 | ao2_unlock(req); 1676 | write_audio_frame(req, f->data.ptr, f->datalen); 1677 | ast_frfree(f); 1678 | ao2_lock(req); 1679 | } 1680 | } 1681 | } 1682 | ao2_unlock(req); 1683 | 1684 | gdf_stop_recognition(req); 1685 | 1686 | ast_log(LOG_DEBUG, "Exiting background thread for GDF %d@%s\n", req->current_utterance_number, req->pvt->session_id); 1687 | 1688 | ao2_t_ref(req, -1, "Done with exec loop"); 1689 | 1690 | return NULL; 1691 | } 1692 | 1693 | static int gdf_change(struct ast_speech *speech, const char *name, const char *value) 1694 | { 1695 | struct gdf_pvt *pvt = speech->data; 1696 | 1697 | if (!strcasecmp(name, GDF_PROP_SESSION_ID_NAME) || !strcasecmp(name, GDF_PROP_ALTERNATE_SESSION_NAME)) { 1698 | if (ast_strlen_zero(value)) { 1699 | ast_log(LOG_WARNING, "Session ID must have a value, refusing to set to nothing (remains %s)\n", pvt->session_id); 1700 | return -1; 1701 | } 1702 | ao2_lock(pvt); 1703 | ast_string_field_set(pvt, session_id, value); 1704 | ao2_unlock(pvt); 1705 | } else if (!strcasecmp(name, GDF_PROP_PROJECT_ID_NAME)) { 1706 | if (ast_strlen_zero(value)) { 1707 | ast_log(LOG_WARNING, "Project ID must have a value, refusing to set to nothing (remains %s)\n", pvt->session_id); 1708 | return -1; 1709 | } 1710 | ao2_lock(pvt); 1711 | ast_string_field_set(pvt, project_id, value); 1712 | ao2_unlock(pvt); 1713 | } else if (!strcasecmp(name, GDF_PROP_LANGUAGE_NAME)) { 1714 | ao2_lock(pvt); 1715 | ast_string_field_set(pvt, language, value); 1716 | ao2_unlock(pvt); 1717 | } else if (!strcasecmp(name, GDF_PROP_LOG_CONTEXT) || !strcasecmp(name, GDF_PROP_ALTERNATE_LOG_CONTEXT)) { 1718 | ao2_lock(pvt); 1719 | ast_string_field_set(pvt, call_logging_context, value); 1720 | ao2_unlock(pvt); 1721 | } else if (!strcasecmp(name, GDF_PROP_APPLICATION_CONTEXT)) { 1722 | ao2_lock(pvt); 1723 | ast_string_field_set(pvt, call_logging_application_name, value); 1724 | ao2_unlock(pvt); 1725 | } else if (!strcasecmp(name, VAD_PROP_VOICE_THRESHOLD)) { 1726 | int i; 1727 | if (ast_strlen_zero(value)) { 1728 | ast_log(LOG_WARNING, "Cannot set " VAD_PROP_VOICE_THRESHOLD " to an empty value\n"); 1729 | return -1; 1730 | } else if (sscanf(value, "%d", &i) == 1) { 1731 | ao2_lock(pvt); 1732 | pvt->voice_threshold = i; 1733 | ao2_unlock(pvt); 1734 | } else { 1735 | ast_log(LOG_WARNING, "Invalid value for " VAD_PROP_VOICE_THRESHOLD " -- '%s'\n", value); 1736 | return -1; 1737 | } 1738 | } else if (!strcasecmp(name, VAD_PROP_VOICE_DURATION)) { 1739 | int i; 1740 | if (ast_strlen_zero(value)) { 1741 | ast_log(LOG_WARNING, "Cannot set " VAD_PROP_VOICE_DURATION " to an empty value\n"); 1742 | return -1; 1743 | } else if (sscanf(value, "%d", &i) == 1) { 1744 | ao2_lock(pvt); 1745 | if ((i % 20) != 0) { 1746 | i = ((i / 20) + 1) * 20; 1747 | } 1748 | pvt->voice_minimum_duration = i; 1749 | ao2_unlock(pvt); 1750 | } else { 1751 | ast_log(LOG_WARNING, "Invalid value for " VAD_PROP_VOICE_DURATION " -- '%s'\n", value); 1752 | return -1; 1753 | } 1754 | } else if (!strcasecmp(name, VAD_PROP_SILENCE_DURATION)) { 1755 | int i; 1756 | if (ast_strlen_zero(value)) { 1757 | ast_log(LOG_WARNING, "Cannot set " VAD_PROP_SILENCE_DURATION " to an empty value\n"); 1758 | return -1; 1759 | } else if (sscanf(value, "%d", &i) == 1) { 1760 | ao2_lock(pvt); 1761 | pvt->silence_minimum_duration = i; 1762 | ao2_unlock(pvt); 1763 | } else { 1764 | ast_log(LOG_WARNING, "Invalid value for " VAD_PROP_SILENCE_DURATION " -- '%s'\n", value); 1765 | return -1; 1766 | } 1767 | } else if (!strcasecmp(name, VAD_PROP_BARGE_DURATION)) { 1768 | int i; 1769 | if (ast_strlen_zero(value)) { 1770 | ast_log(LOG_WARNING, "Cannot set " VAD_PROP_BARGE_DURATION " to an empty value\n"); 1771 | return -1; 1772 | } else if (sscanf(value, "%d", &i) == 1) { 1773 | ao2_lock(pvt); 1774 | pvt->barge_in_minimum_duration = i; 1775 | ao2_unlock(pvt); 1776 | } else { 1777 | ast_log(LOG_WARNING, "Invalid value for " VAD_PROP_BARGE_DURATION " -- '%s'\n", value); 1778 | return -1; 1779 | } 1780 | } else if (!strcasecmp(name, VAD_PROP_END_OF_SPEECH_DURATION)) { 1781 | int i; 1782 | if (ast_strlen_zero(value)) { 1783 | ast_log(LOG_WARNING, "Cannot set " VAD_PROP_END_OF_SPEECH_DURATION " to an empty value\n"); 1784 | return -1; 1785 | } else if (sscanf(value, "%d", &i) == 1) { 1786 | ao2_lock(pvt); 1787 | pvt->end_of_speech_minimum_silence = i; 1788 | ao2_unlock(pvt); 1789 | } else { 1790 | ast_log(LOG_WARNING, "Invalid value for " VAD_PROP_END_OF_SPEECH_DURATION " -- '%s'\n", value); 1791 | return -1; 1792 | } 1793 | } else if (!strcasecmp(name, GDF_PROP_REQUEST_SENTIMENT_ANALYSIS)) { 1794 | ao2_lock(pvt); 1795 | pvt->request_sentiment_analysis = ast_true(value); 1796 | ao2_unlock(pvt); 1797 | } else if (!strcasecmp(name, "logPromptStart")) { 1798 | struct dialogflow_log_data log_data[] = { 1799 | { "context", pvt->call_logging_context }, 1800 | { "prompt", S_OR(value, "") } 1801 | }; 1802 | gdf_log_call_event(pvt, pvt->current_request, CALL_LOG_TYPE_RECOGNITION, "prompt_start", ARRAY_LEN(log_data), log_data); 1803 | } else if (!strcasecmp(name, "logPromptStop")) { 1804 | struct dialogflow_log_data log_data[] = { 1805 | { "reason", S_OR(value, "none") } 1806 | }; 1807 | gdf_log_call_event(pvt, pvt->current_request, CALL_LOG_TYPE_RECOGNITION, "prompt_stop", ARRAY_LEN(log_data), log_data); 1808 | } else if (!strcasecmp(name, "logDtmf")) { 1809 | struct dialogflow_log_data log_data[] = { 1810 | { "digits", S_OR(value, "") } 1811 | }; 1812 | gdf_log_call_event(pvt, pvt->current_request, CALL_LOG_TYPE_RECOGNITION, "digits", ARRAY_LEN(log_data), log_data); 1813 | } else if (!strcasecmp(name, "record") || !strcasecmp(name, "recordUtterance")) { 1814 | ao2_lock(pvt); 1815 | pvt->record_next_utterance = ast_true(value); 1816 | ao2_unlock(pvt); 1817 | } else if (!strcasecmp(name, "incompleteTimeout")) { 1818 | ao2_lock(pvt); 1819 | pvt->incomplete_timeout = atoi(value); 1820 | ao2_unlock(pvt); 1821 | } else if (!strcasecmp(name, "noSpeechTimeout")) { 1822 | ao2_lock(pvt); 1823 | pvt->no_speech_timeout = atoi(value); 1824 | ao2_unlock(pvt); 1825 | } else if (!strcasecmp(name, "maximumSpeechTimeout")) { 1826 | ao2_lock(pvt); 1827 | pvt->maximum_speech_timeout = atoi(value); 1828 | ao2_unlock(pvt); 1829 | } else { 1830 | ast_log(LOG_DEBUG, "Unknown property '%s'\n", name); 1831 | return -1; 1832 | } 1833 | 1834 | return 0; 1835 | } 1836 | 1837 | #ifdef AST_SPEECH_HAVE_GET_SETTING 1838 | static int gdf_get_setting(struct ast_speech *speech, const char *name, char *buf, size_t len) 1839 | { 1840 | struct gdf_pvt *pvt = speech->data; 1841 | 1842 | if (!strcasecmp(name, GDF_PROP_UTTERANCE_DURATION_MS)) { 1843 | long long last_audio_duration_ms = 0; 1844 | ao2_lock(pvt); 1845 | if (pvt->current_request) { 1846 | last_audio_duration_ms = pvt->current_request->last_audio_duration_ms; 1847 | } 1848 | ao2_unlock(pvt); 1849 | ast_build_string(&buf, &len, "%lld", last_audio_duration_ms); 1850 | } else if (!strcasecmp(name, GDF_PROP_SESSION_ID_NAME)) { 1851 | ast_copy_string(buf, pvt->session_id, len); 1852 | } else if (!strcasecmp(name, GDF_PROP_PROJECT_ID_NAME)) { 1853 | ast_copy_string(buf, pvt->session_id, len); 1854 | } else if (!strcasecmp(name, GDF_PROP_LANGUAGE_NAME)) { 1855 | ao2_lock(pvt); 1856 | ast_copy_string(buf, pvt->language, len); 1857 | ao2_unlock(pvt); 1858 | } else if (!strcasecmp(name, VAD_PROP_VOICE_THRESHOLD)) { 1859 | ao2_lock(pvt); 1860 | ast_build_string(&buf, &len, "%d", pvt->voice_threshold); 1861 | ao2_unlock(pvt); 1862 | } else if (!strcasecmp(name, VAD_PROP_VOICE_DURATION)) { 1863 | ao2_lock(pvt); 1864 | ast_build_string(&buf, &len, "%d", pvt->voice_minimum_duration); 1865 | ao2_unlock(pvt); 1866 | } else if (!strcasecmp(name, VAD_PROP_SILENCE_DURATION)) { 1867 | ao2_lock(pvt); 1868 | ast_build_string(&buf, &len, "%d", pvt->silence_minimum_duration); 1869 | ao2_unlock(pvt); 1870 | } else if (!strcasecmp(name, VAD_PROP_BARGE_DURATION)) { 1871 | ao2_lock(pvt); 1872 | ast_build_string(&buf, &len, "%d", pvt->barge_in_minimum_duration); 1873 | ao2_unlock(pvt); 1874 | } else if (!strcasecmp(name, VAD_PROP_END_OF_SPEECH_DURATION)) { 1875 | ao2_lock(pvt); 1876 | ast_build_string(&buf, &len, "%d", pvt->end_of_speech_minimum_silence); 1877 | ao2_unlock(pvt); 1878 | } else { 1879 | ast_log(LOG_WARNING, "Unknown property '%s'\n", name); 1880 | return -1; 1881 | } 1882 | 1883 | return 0; 1884 | } 1885 | #endif 1886 | 1887 | static int gdf_change_results_type(struct ast_speech *speech, enum ast_speech_results_type results_type) 1888 | { 1889 | return 0; 1890 | } 1891 | 1892 | static void add_speech_result(struct ast_speech_result **start, struct ast_speech_result **end, 1893 | const char *grammar, int score, const char *text) 1894 | { 1895 | struct ast_speech_result *new = ast_calloc(1, sizeof(*new)); 1896 | if (new) { 1897 | new->text = ast_strdup(text); 1898 | new->score = score; 1899 | new->grammar = ast_strdup(grammar); 1900 | } 1901 | 1902 | if (*end) { 1903 | AST_LIST_NEXT(*end, list) = new; 1904 | *end = new; 1905 | } else { 1906 | *start = *end = new; 1907 | } 1908 | } 1909 | 1910 | static struct ast_speech_result *gdf_get_results(struct ast_speech *speech) 1911 | { 1912 | /* speech is not locked */ 1913 | struct gdf_pvt *pvt = speech->data; 1914 | struct gdf_request *req = pvt->current_request; 1915 | int results; 1916 | int i; 1917 | struct ast_speech_result *start = NULL; 1918 | struct ast_speech_result *end = NULL; 1919 | static int last_resort = 0; 1920 | 1921 | struct dialogflow_result *fulfillment_text = NULL; 1922 | struct dialogflow_result *output_audio = NULL; 1923 | 1924 | const char *audioFile = NULL; 1925 | 1926 | struct gdf_config *cfg; 1927 | 1928 | if (!req || !req->session) { 1929 | return NULL; 1930 | } 1931 | 1932 | cfg = gdf_get_config(); 1933 | results = df_get_result_count(req->session); 1934 | 1935 | for (i = 0; i < results; i++) { 1936 | struct dialogflow_result *df_result = df_get_result(req->session, i); /* this is a borrowed reference */ 1937 | if (df_result) { 1938 | if (!strcasecmp(df_result->slot, "output_audio")) { 1939 | /* this is fine for now, but we really need a flag on the structure that says it's binary vs. text */ 1940 | output_audio = df_result; 1941 | } else { 1942 | add_speech_result(&start, &end, df_result->slot, df_result->score, df_result->value); 1943 | 1944 | if (!strcasecmp(df_result->slot, "fulfillment_text")) { 1945 | fulfillment_text = df_result; 1946 | } 1947 | } 1948 | } 1949 | } 1950 | 1951 | ao2_lock(pvt); 1952 | if (pvt->current_request) { 1953 | char buffer[32]; 1954 | char *b = buffer; 1955 | size_t l = sizeof(buffer); 1956 | *b = '\0'; 1957 | ast_build_string(&b, &l, "%lld", pvt->current_request->last_audio_duration_ms); 1958 | add_speech_result(&start, &end, "waveformDuration", 0, buffer); 1959 | } 1960 | ao2_unlock(pvt); 1961 | 1962 | if (output_audio) { 1963 | struct ast_speech_result *new; 1964 | char tmpFilename[128]; 1965 | int fd; 1966 | ssize_t written; 1967 | 1968 | ast_copy_string(tmpFilename, "/tmp/res_speech_gdfe_fulfillment_XXXXXX.wav", sizeof(tmpFilename)); 1969 | fd = mkstemps(tmpFilename, 4); 1970 | 1971 | if (fd < 0) { 1972 | ast_log(LOG_WARNING, "Unable to create temporary file for fulfillment message\n"); 1973 | sprintf(tmpFilename, "/tmp/res_speech_gdfe_fulfillment_%d.wav", ast_atomic_fetchadd_int(&last_resort, 1)); 1974 | fd = open(tmpFilename, O_WRONLY | O_CREAT, 0600); 1975 | } 1976 | written = write(fd, output_audio->value, output_audio->valueLen); 1977 | if (written < output_audio->valueLen) { 1978 | ast_log(LOG_WARNING, "Short write to temporary file for fulfillment message\n"); 1979 | } 1980 | close(fd); 1981 | 1982 | audioFile = tmpFilename; 1983 | 1984 | new = ast_calloc(1, sizeof(*new)); 1985 | if (new) { 1986 | new->text = ast_strdup(tmpFilename); 1987 | new->score = 100; 1988 | new->grammar = ast_strdup("fulfillment_audio"); 1989 | 1990 | if (end) { 1991 | AST_LIST_NEXT(end, list) = new; 1992 | end = new; 1993 | } else { 1994 | start = end = new; 1995 | } 1996 | } else { 1997 | ast_log(LOG_WARNING, "Unable to allocate speech result slot for fulfillment audio\n"); 1998 | } 1999 | } else if (cfg->synthesize_fulfillment_text && fulfillment_text && !ast_strlen_zero(fulfillment_text->value)) { 2000 | char tmpFilename[128]; 2001 | int fd; 2002 | struct gdf_config *cfg; 2003 | char *key; 2004 | char *language; 2005 | 2006 | cfg = gdf_get_config(); 2007 | key = ast_strdupa(cfg->service_key); 2008 | ao2_t_ref(cfg, -1, "done with creating session"); 2009 | 2010 | ao2_lock(pvt); 2011 | language = ast_strdupa(pvt->language); 2012 | ao2_unlock(pvt); 2013 | 2014 | ast_copy_string(tmpFilename, "/tmp/res_speech_gdfe_fulfillment_XXXXXX.wav", sizeof(tmpFilename)); 2015 | fd = mkstemps(tmpFilename, 4); 2016 | 2017 | if (fd >= 0) { 2018 | close(fd); 2019 | } else { 2020 | ast_log(LOG_WARNING, "Unable to create temporary file for fulfillment message\n"); 2021 | sprintf(tmpFilename, "/tmp/res_speech_gdfe_fulfillment_%d.wav", ast_atomic_fetchadd_int(&last_resort, 1)); 2022 | } 2023 | 2024 | audioFile = tmpFilename; 2025 | 2026 | if (google_synth_speech(NULL, key, fulfillment_text->value, language, NULL, tmpFilename)) { 2027 | ast_log(LOG_WARNING, "Failed to synthesize fulfillment text to %s\n", tmpFilename); 2028 | } else { 2029 | struct ast_speech_result *new = ast_calloc(1, sizeof(*new)); 2030 | if (new) { 2031 | new->text = ast_strdup(tmpFilename); 2032 | new->score = 100; 2033 | new->grammar = ast_strdup("fulfillment_audio"); 2034 | 2035 | if (end) { 2036 | AST_LIST_NEXT(end, list) = new; 2037 | end = new; 2038 | } else { 2039 | start = end = new; 2040 | } 2041 | } else { 2042 | ast_log(LOG_WARNING, "Unable to allocate speech result slot for synthesized fulfillment text\n"); 2043 | } 2044 | } 2045 | } 2046 | 2047 | ao2_t_ref(cfg, -1, "done with config"); 2048 | 2049 | if (!ast_strlen_zero(pvt->lastAudioResponse)) { 2050 | unlink(pvt->lastAudioResponse); 2051 | ast_string_field_set(pvt, lastAudioResponse, ""); 2052 | } 2053 | if (!ast_strlen_zero(audioFile)) { 2054 | ast_string_field_set(pvt, lastAudioResponse, audioFile); 2055 | } 2056 | 2057 | return start; 2058 | } 2059 | 2060 | static void gdf_config_destroy(void *o) 2061 | { 2062 | struct gdf_config *conf = o; 2063 | 2064 | ast_string_field_free_memory(conf); 2065 | 2066 | if (conf->logical_agents) { 2067 | ao2_ref(conf->logical_agents, -1); 2068 | } 2069 | } 2070 | 2071 | static struct gdf_config *gdf_get_config(void) 2072 | { 2073 | struct gdf_config *cfg; 2074 | #ifdef ASTERISK_13_OR_LATER 2075 | ao2_rdlock(config); 2076 | #else 2077 | ao2_lock(config); 2078 | #endif 2079 | cfg = ao2_find(config, NULL, 0); 2080 | ao2_unlock(config); 2081 | return cfg; 2082 | } 2083 | 2084 | static void logical_agent_destructor(void *obj) 2085 | { 2086 | struct gdf_logical_agent *agent = (struct gdf_logical_agent *) obj; 2087 | if (agent->hints) { 2088 | ao2_t_ref(agent->hints, -1, "destroying agent"); 2089 | } 2090 | ast_string_field_free_memory(agent); 2091 | } 2092 | 2093 | static void hint_destructor(void *obj) 2094 | { 2095 | /* noop */ 2096 | } 2097 | 2098 | static void parse_hints(struct ao2_container *hints, const char *val) 2099 | { 2100 | if (!ast_strlen_zero(val)) { 2101 | char *val_copy = ast_strdupa(val); 2102 | char *saved = NULL; 2103 | char *hint; 2104 | 2105 | for (hint = strtok_r(val_copy, ",", &saved); hint; hint = strtok_r(NULL, ",", &saved)) { 2106 | char *ao2_hint; 2107 | hint = ast_strip(hint); 2108 | if (!ast_strlen_zero(hint)) { 2109 | size_t len = strlen(hint); 2110 | ao2_hint = ao2_alloc(len + 1, hint_destructor); 2111 | if (ao2_hint) { 2112 | ast_copy_string(ao2_hint, hint, len + 1); 2113 | ao2_link(hints, ao2_hint); 2114 | ao2_t_ref(ao2_hint, -1, "linked hint on general load"); 2115 | } 2116 | } 2117 | } 2118 | } 2119 | } 2120 | 2121 | static struct gdf_logical_agent *logical_agent_alloc(const char *name, const char *project_id, 2122 | const char *service_key, const char *endpoint, enum SENTIMENT_ANALYSIS_STATE sentiment_analysis_state, 2123 | const char *hints, int use_internal_endpointer_for_end_of_speech, const char *model) 2124 | { 2125 | size_t name_len = strlen(name); 2126 | size_t project_id_len = strlen(project_id); 2127 | size_t service_key_len = strlen(service_key); 2128 | size_t endpoint_len = strlen(endpoint); 2129 | size_t model_len = strlen(model); 2130 | size_t space_needed = name_len + 1 + 2131 | project_id_len + 1 + 2132 | service_key_len + 1 + 2133 | endpoint_len + 1 + 2134 | model_len + 1; 2135 | struct gdf_logical_agent *agent; 2136 | 2137 | agent = ao2_alloc(sizeof(struct gdf_logical_agent), logical_agent_destructor); 2138 | if (!agent) { 2139 | ast_log(LOG_WARNING, "Failed to allocate logical agent for %s\n", name); 2140 | return NULL; 2141 | } 2142 | 2143 | if (ast_string_field_init(agent, space_needed)) { 2144 | ast_log(LOG_WARNING, "Failed to allocate string fields for logical agent %s\n", name); 2145 | ao2_t_ref(agent, -1, "Failed to allocate string fields"); 2146 | return NULL; 2147 | } 2148 | 2149 | ast_string_field_set(agent, name, name); 2150 | ast_string_field_set(agent, project_id, project_id); 2151 | ast_string_field_set(agent, service_key, service_key); 2152 | ast_string_field_set(agent, endpoint, endpoint); 2153 | agent->enable_sentiment_analysis = sentiment_analysis_state; 2154 | agent->use_internal_endpointer_for_end_of_speech = use_internal_endpointer_for_end_of_speech; 2155 | 2156 | agent->hints = ao2_container_alloc(1, NULL, NULL); 2157 | if (agent->hints && !ast_strlen_zero(hints)) { 2158 | parse_hints(agent->hints, hints); 2159 | } 2160 | 2161 | return agent; 2162 | } 2163 | 2164 | static int logical_agent_hash_callback(const void *obj, const int flags) 2165 | { 2166 | const struct gdf_logical_agent *agent = obj; 2167 | return ast_str_case_hash(agent->name); 2168 | } 2169 | 2170 | static int logical_agent_compare_callback(void *obj, void *other, int flags) 2171 | { 2172 | const struct gdf_logical_agent *agentA = obj; 2173 | const struct gdf_logical_agent *agentB = other; 2174 | return (!strcasecmp(agentA->name, agentB->name) ? CMP_MATCH | CMP_STOP : 0); 2175 | } 2176 | 2177 | static struct gdf_logical_agent *get_logical_agent_by_name(struct gdf_config *config, const char *name) 2178 | { 2179 | struct gdf_logical_agent tmpAgent = { .name = name }; 2180 | return ao2_find(config->logical_agents, &tmpAgent, OBJ_POINTER); 2181 | } 2182 | 2183 | static struct ast_str *load_service_key(const char *val) 2184 | { 2185 | struct ast_str *buffer = ast_str_create(3 * 1024); /* big enough for the typical key size */ 2186 | if (!buffer) { 2187 | ast_log(LOG_WARNING, "Memory allocation failure allocating ast_str for loading service key\n"); 2188 | return NULL; 2189 | } 2190 | 2191 | if (strchr(val, '{')) { 2192 | ast_str_set(&buffer, 0, "%s", val); 2193 | } else { 2194 | FILE *f; 2195 | ast_log(LOG_DEBUG, "Loading service key data from %s\n", val); 2196 | f = fopen(val, "r"); 2197 | if (f) { 2198 | char readbuffer[512]; 2199 | size_t read = fread(readbuffer, sizeof(char), sizeof(readbuffer), f); 2200 | while (read > 0) { 2201 | ast_str_append_substr(&buffer, 0, readbuffer, read); 2202 | read = fread(readbuffer, sizeof(char), sizeof(readbuffer), f); 2203 | } 2204 | if (ferror(f)) { 2205 | ast_log(LOG_WARNING, "Error reading %s -- %d\n", val, errno); 2206 | } 2207 | fclose(f); 2208 | } else { 2209 | ast_log(LOG_ERROR, "Unable to open service key file %s -- %d\n", val, errno); 2210 | } 2211 | } 2212 | 2213 | return buffer; 2214 | } 2215 | 2216 | #define CONFIGURATION_FILENAME "res_speech_gdfe.conf" 2217 | static int load_config(int reload) 2218 | { 2219 | struct ast_config *cfg = NULL; 2220 | struct ast_flags config_flags = { reload ? CONFIG_FLAG_FILEUNCHANGED : 0 }; 2221 | 2222 | cfg = ast_config_load(CONFIGURATION_FILENAME, config_flags); 2223 | if (cfg == CONFIG_STATUS_FILEUNCHANGED) { 2224 | ast_log(LOG_DEBUG, "Configuration unchanged.\n"); 2225 | } else { 2226 | struct gdf_config *conf; 2227 | const char *val; 2228 | const char *category; 2229 | 2230 | if (cfg == CONFIG_STATUS_FILEINVALID) { 2231 | ast_log(LOG_WARNING, "Configuration file invalid\n"); 2232 | cfg = ast_config_new(); 2233 | } else if (cfg == CONFIG_STATUS_FILEMISSING) { 2234 | ast_log(LOG_WARNING, "Configuration not found, using defaults\n"); 2235 | cfg = ast_config_new(); 2236 | } 2237 | 2238 | conf = ao2_alloc(sizeof(*conf), gdf_config_destroy); 2239 | if (!conf) { 2240 | ast_log(LOG_WARNING, "Failed to allocate config record for speech gdf\n"); 2241 | ast_config_destroy(cfg); 2242 | return AST_MODULE_LOAD_FAILURE; 2243 | } 2244 | 2245 | if (ast_string_field_init(conf, 3 * 1024)) { 2246 | ast_log(LOG_WARNING, "Failed to allocate string fields for config for speech gdf\n"); 2247 | ao2_ref(conf, -1); 2248 | ast_config_destroy(cfg); 2249 | return AST_MODULE_LOAD_FAILURE; 2250 | } 2251 | 2252 | conf->logical_agents = ao2_container_alloc(32, logical_agent_hash_callback, logical_agent_compare_callback); 2253 | if (!conf->logical_agents) { 2254 | ast_log(LOG_WARNING, "Failed to allocate logical agent container for speech gdf\n"); 2255 | ao2_ref(conf, -1); 2256 | ast_config_destroy(cfg); 2257 | } 2258 | 2259 | conf->hints = ao2_container_alloc(1, NULL, NULL); 2260 | if (!conf->hints) { 2261 | ast_log(LOG_WARNING, "Failed to allocate hint container for speech gdf\n"); 2262 | } 2263 | 2264 | val = ast_variable_retrieve(cfg, "general", "service_key"); 2265 | if (ast_strlen_zero(val)) { 2266 | ast_log(LOG_VERBOSE, "Service key not provided -- will use default credentials.\n"); 2267 | } else { 2268 | struct ast_str *buffer = load_service_key(val); 2269 | ast_string_field_set(conf, service_key, ast_str_buffer(buffer)); 2270 | ast_free(buffer); 2271 | } 2272 | 2273 | val = ast_variable_retrieve(cfg, "general", "endpoint"); 2274 | if (!ast_strlen_zero(val)) { 2275 | ast_string_field_set(conf, endpoint, val); 2276 | } 2277 | 2278 | conf->vad_voice_threshold = 1024; 2279 | val = ast_variable_retrieve(cfg, "general", "vad_voice_threshold"); 2280 | if (!ast_strlen_zero(val)) { 2281 | int i; 2282 | if (sscanf(val, "%d", &i) == 1) { 2283 | conf->vad_voice_threshold = i; 2284 | } else { 2285 | ast_log(LOG_WARNING, "Invalid value for vad_voice_threshold\n"); 2286 | } 2287 | } 2288 | 2289 | conf->vad_voice_minimum_duration = 100; /* ms */ 2290 | val = ast_variable_retrieve(cfg, "general", "vad_voice_minimum_duration"); 2291 | if (!ast_strlen_zero(val)) { 2292 | int i; 2293 | if (sscanf(val, "%d", &i) == 1) { 2294 | if ((i % 20) != 0) { 2295 | i = ((i / 20) + 1) * 20; 2296 | } 2297 | conf->vad_voice_minimum_duration = i; 2298 | } else { 2299 | ast_log(LOG_WARNING, "Invalid value for vad_voice_minimum_duration\n"); 2300 | } 2301 | } 2302 | 2303 | conf->vad_silence_minimum_duration = 100; /* ms */ 2304 | val = ast_variable_retrieve(cfg, "general", "vad_silence_minimum_duration"); 2305 | if (!ast_strlen_zero(val)) { 2306 | int i; 2307 | if (sscanf(val, "%d", &i) == 1) { 2308 | if ((i % 20) != 0) { 2309 | i = ((i / 20) + 1) * 20; 2310 | } 2311 | conf->vad_silence_minimum_duration = i; 2312 | } else { 2313 | ast_log(LOG_WARNING, "Invalid value for vad_silence_minimum_duration\n"); 2314 | } 2315 | } 2316 | 2317 | conf->vad_barge_minimum_duration = 300; /* ms */ 2318 | val = ast_variable_retrieve(cfg, "general", "vad_barge_minimum_duration"); 2319 | if (!ast_strlen_zero(val)) { 2320 | int i; 2321 | if (sscanf(val, "%d", &i) == 1) { 2322 | if ((i % 20) != 0) { 2323 | i = ((i / 20) + 1) * 20; 2324 | } 2325 | conf->vad_barge_minimum_duration = i; 2326 | } else { 2327 | ast_log(LOG_WARNING, "Invalid value for vad_barge_minimum_duration\n"); 2328 | } 2329 | } 2330 | 2331 | conf->vad_end_of_speech_silence_duration = 500; /* ms */ 2332 | val = ast_variable_retrieve(cfg, "general", "vad_end_of_speech_silence_duration"); 2333 | if (!ast_strlen_zero(val)) { 2334 | int i; 2335 | if (sscanf(val, "%d", &i) == 1) { 2336 | if ((i % 20) != 0) { 2337 | i = ((i / 20) + 1) * 20; 2338 | } 2339 | conf->vad_end_of_speech_silence_duration = i; 2340 | } else { 2341 | ast_log(LOG_WARNING, "Invalid value for vad_end_of_speech_silence_duration\n"); 2342 | } 2343 | } 2344 | 2345 | conf->endpointer_cache_audio_pretrigger_ms = 100; 2346 | val = ast_variable_retrieve(cfg, "general", "endpointer_cache_audio_pretrigger_ms"); 2347 | if (!ast_strlen_zero(val)) { 2348 | int i; 2349 | if (sscanf(val, "%d", &i) == 1) { 2350 | if (i % 20 != 0) { 2351 | int new_i = ((i / 20) + 1) * 20; 2352 | ast_log(LOG_WARNING, "Rounding endpointer_cache_audio_pretrigger_ms from %d to %d to match packet size\n", 2353 | i, new_i); 2354 | i = new_i; 2355 | } 2356 | conf->endpointer_cache_audio_pretrigger_ms = i; 2357 | } else { 2358 | ast_log(LOG_WARNING, "Invalid value for endpointer_cache_audio_pretrigger_ms\n"); 2359 | } 2360 | } 2361 | 2362 | ast_string_field_set(conf, call_log_location, "/var/log/dialogflow/${APPLICATION}/${STRFTIME(,,%Y/%m/%d/%H)}/"); 2363 | val = ast_variable_retrieve(cfg, "general", "call_log_location"); 2364 | if (!ast_strlen_zero(val)) { 2365 | ast_string_field_set(conf, call_log_location, val); 2366 | } 2367 | 2368 | conf->enable_call_logs = 1; 2369 | val = ast_variable_retrieve(cfg, "general", "enable_call_logs"); 2370 | if (!ast_strlen_zero(val)) { 2371 | conf->enable_call_logs = ast_true(val); 2372 | } 2373 | 2374 | conf->enable_preendpointer_recordings = 0; 2375 | val = ast_variable_retrieve(cfg, "general", "enable_preendpointer_recordings"); 2376 | if (!ast_strlen_zero(val)) { 2377 | conf->enable_preendpointer_recordings = ast_true(val); 2378 | } 2379 | 2380 | conf->enable_postendpointer_recordings = 0; 2381 | val = ast_variable_retrieve(cfg, "general", "enable_postendpointer_recordings"); 2382 | if (!ast_strlen_zero(val)) { 2383 | conf->enable_postendpointer_recordings = ast_true(val); 2384 | } 2385 | 2386 | conf->record_preendpointer_on_demand = 0; 2387 | val = ast_variable_retrieve(cfg, "general", "record_preendpointer_on_demand"); 2388 | if (!ast_strlen_zero(val)) { 2389 | conf->record_preendpointer_on_demand = ast_true(val); 2390 | } 2391 | 2392 | conf->enable_sentiment_analysis = SENTIMENT_ANALYSIS_DEFAULT; 2393 | val = ast_variable_retrieve(cfg, "general", "enable_sentiment_analysis"); 2394 | if (!ast_strlen_zero(val)) { 2395 | if (ast_true(val) || !strcasecmp(val, "always")) { 2396 | conf->enable_sentiment_analysis = SENTIMENT_ANALYSIS_ALWAYS; 2397 | } else if (!strcasecmp(val, "default")) { 2398 | conf->enable_sentiment_analysis = SENTIMENT_ANALYSIS_DEFAULT; 2399 | } else { 2400 | conf->enable_sentiment_analysis = SENTIMENT_ANALYSIS_NEVER; 2401 | } 2402 | } 2403 | 2404 | conf->stop_writes_on_final_transcription = 1; 2405 | val = ast_variable_retrieve(cfg, "general", "stop_writes_on_final_transcription"); 2406 | if (!ast_strlen_zero(val)) { 2407 | conf->stop_writes_on_final_transcription = ast_true(val); 2408 | } 2409 | 2410 | conf->start_recognition_on_start = 0; 2411 | val = ast_variable_retrieve(cfg, "general", "start_recognition_on_start"); 2412 | if (!ast_strlen_zero(val)) { 2413 | conf->start_recognition_on_start = ast_true(val); 2414 | } 2415 | 2416 | conf->synthesize_fulfillment_text = 0; 2417 | val = ast_variable_retrieve(cfg, "general", "synthesize_fulfillment_text"); 2418 | if (!ast_strlen_zero(val)) { 2419 | conf->synthesize_fulfillment_text = ast_true(val); 2420 | } 2421 | 2422 | conf->use_internal_endpointer_for_end_of_speech = 0; 2423 | val = ast_variable_retrieve(cfg, "general", "use_internal_endpointer_for_end_of_speech"); 2424 | if (!ast_strlen_zero(val)) { 2425 | conf->use_internal_endpointer_for_end_of_speech = ast_true(val); 2426 | } 2427 | 2428 | conf->recognition_start_failure_retries = 4; 2429 | val = ast_variable_retrieve(cfg, "general", "recognition_start_failure_retries"); 2430 | if (!ast_strlen_zero(val)) { 2431 | int i; 2432 | if (1 == sscanf(val, "%d", &i)) { 2433 | conf->recognition_start_failure_retries = i; 2434 | } else { 2435 | ast_log(LOG_WARNING, "Invalid value '%s' for recognition_start_failure_retries\n", val); 2436 | } 2437 | } 2438 | 2439 | conf->recognition_start_failure_retry_max_time_ms = 1000; 2440 | val = ast_variable_retrieve(cfg, "general", "recognition_start_failure_retry_max_time_ms"); 2441 | if (!ast_strlen_zero(val)) { 2442 | int i; 2443 | if (1 == sscanf(val, "%d", &i)) { 2444 | conf->recognition_start_failure_retry_max_time_ms = i; 2445 | } else { 2446 | ast_log(LOG_WARNING, "Invalid value '%s' for recognition_start_failure_retry_max_time_ms\n", val); 2447 | } 2448 | } 2449 | 2450 | ast_string_field_set(conf, start_failure_retry_codes, ",14,"); 2451 | val = ast_variable_retrieve(cfg, "general", "start_failure_retry_codes"); 2452 | if (!ast_strlen_zero(val)) { 2453 | ast_string_field_build(conf, start_failure_retry_codes, ",%s,", val); 2454 | } 2455 | 2456 | val = ast_variable_retrieve(cfg, "general", "model"); 2457 | if (!ast_strlen_zero(val)) { 2458 | ast_string_field_set(conf, model, val); 2459 | } 2460 | 2461 | if (conf->hints) { 2462 | val = ast_variable_retrieve(cfg, "general", "hints"); 2463 | if (!ast_strlen_zero(val)) { 2464 | parse_hints(conf->hints, val); 2465 | } 2466 | } 2467 | 2468 | conf->default_incomplete_timeout = 0; 2469 | val = ast_variable_retrieve(cfg, "general", "default_incomplete_timeout"); 2470 | if (!ast_strlen_zero(val)) { 2471 | int i; 2472 | if (1 == sscanf(val, "%d", &i)) { 2473 | conf->default_incomplete_timeout = i; 2474 | } else { 2475 | ast_log(LOG_WARNING, "Invalid value '%s' for default_incomplete_timeout\n", val); 2476 | } 2477 | } 2478 | 2479 | conf->default_no_speech_timeout = 0; 2480 | val = ast_variable_retrieve(cfg, "general", "default_no_speech_timeout"); 2481 | if (!ast_strlen_zero(val)) { 2482 | int i; 2483 | if (1 == sscanf(val, "%d", &i)) { 2484 | conf->default_no_speech_timeout = i; 2485 | } else { 2486 | ast_log(LOG_WARNING, "Invalid value '%s' for default_no_speech_timeout\n", val); 2487 | } 2488 | } 2489 | 2490 | conf->default_maximum_speech_timeout = 90 * 1000; 2491 | val = ast_variable_retrieve(cfg, "general", "default_maximum_speech_timeout"); 2492 | if (!ast_strlen_zero(val)) { 2493 | int i; 2494 | if (1 == sscanf(val, "%d", &i)) { 2495 | conf->default_maximum_speech_timeout = i; 2496 | } else { 2497 | ast_log(LOG_WARNING, "Invalid value '%s' for default_maximum_speech_timeout\n", val); 2498 | } 2499 | } 2500 | 2501 | category = NULL; 2502 | while ((category = ast_category_browse(cfg, category))) { 2503 | if (strcasecmp("general", category)) { 2504 | const char *name = category; 2505 | const char *project_id = ast_variable_retrieve(cfg, category, "project_id"); 2506 | const char *endpoint = ast_variable_retrieve(cfg, category, "endpoint"); 2507 | const char *service_key = ast_variable_retrieve(cfg, category, "service_key"); 2508 | const char *enable_sentiment_analysis = ast_variable_retrieve(cfg, category, "enable_sentiment_analysis"); 2509 | const char *hints = ast_variable_retrieve(cfg, category, "hints"); 2510 | const char *use_internal_endpointer_for_end_of_speech_str = ast_variable_retrieve(cfg, category, "use_internal_endpointer_for_end_of_speech"); 2511 | const char *model = ast_variable_retrieve(cfg, category, "model"); 2512 | int use_internal_endpointer_for_end_of_speech; 2513 | enum SENTIMENT_ANALYSIS_STATE sentiment_analysis_state = SENTIMENT_ANALYSIS_DEFAULT; 2514 | 2515 | if (!ast_strlen_zero(service_key)) { 2516 | struct ast_str *buffer = load_service_key(service_key); 2517 | if (buffer) { 2518 | service_key = ast_strdupa(ast_str_buffer(buffer)); 2519 | ast_free(buffer); 2520 | } 2521 | } 2522 | 2523 | use_internal_endpointer_for_end_of_speech = conf->use_internal_endpointer_for_end_of_speech; 2524 | if (!ast_strlen_zero(use_internal_endpointer_for_end_of_speech_str)) { 2525 | use_internal_endpointer_for_end_of_speech = ast_true(use_internal_endpointer_for_end_of_speech_str); 2526 | } 2527 | 2528 | if (!ast_strlen_zero(enable_sentiment_analysis)) { 2529 | if (ast_true(enable_sentiment_analysis) || !strcasecmp(enable_sentiment_analysis, "always")) { 2530 | sentiment_analysis_state = SENTIMENT_ANALYSIS_ALWAYS; 2531 | } else if (!strcasecmp(enable_sentiment_analysis, "default")) { 2532 | sentiment_analysis_state = SENTIMENT_ANALYSIS_DEFAULT; 2533 | } else { 2534 | sentiment_analysis_state = SENTIMENT_ANALYSIS_NEVER; 2535 | } 2536 | } 2537 | 2538 | if (!ast_strlen_zero(project_id)) { 2539 | struct gdf_logical_agent *agent; 2540 | 2541 | agent = logical_agent_alloc(name, project_id, S_OR(service_key, ""), S_OR(endpoint, ""), sentiment_analysis_state, 2542 | hints, use_internal_endpointer_for_end_of_speech, S_OR(model, conf->model)); 2543 | if (agent) { 2544 | ao2_link(conf->logical_agents, agent); 2545 | ao2_ref(agent, -1); 2546 | } else { 2547 | ast_log(LOG_WARNING, "Memory allocation failed creating logical agent %s\n", name); 2548 | } 2549 | } else { 2550 | ast_log(LOG_WARNING, "Mapped project_id is required for %s\n", name); 2551 | } 2552 | } 2553 | } 2554 | 2555 | /* swap out the configs */ 2556 | #ifdef ASTERISK_13_OR_LATER 2557 | ao2_wrlock(config); 2558 | #else 2559 | ao2_lock(config); 2560 | #endif 2561 | { 2562 | struct gdf_config *old_config = gdf_get_config(); 2563 | ao2_unlink(config, old_config); 2564 | ao2_ref(old_config, -1); 2565 | } 2566 | ao2_link(config, conf); 2567 | ao2_unlock(config); 2568 | ao2_ref(conf, -1); 2569 | 2570 | if (cfg) { 2571 | ast_config_destroy(cfg); 2572 | } 2573 | } 2574 | 2575 | return AST_MODULE_LOAD_SUCCESS; 2576 | } 2577 | 2578 | static char *gdfe_reload(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a) 2579 | { 2580 | switch (cmd) { 2581 | case CLI_INIT: 2582 | e->command = "gdfe reload"; 2583 | e->usage = 2584 | "Usage: gdfe reload\n" 2585 | " Reload res_speech_gdfe configuration.\n"; 2586 | return NULL; 2587 | case CLI_GENERATE: 2588 | return NULL; 2589 | default: 2590 | ast_cli(a->fd, "Reloading res_speech_gdfe config from " CONFIGURATION_FILENAME "\n"); 2591 | load_config(1); 2592 | ast_cli(a->fd, "Reload complete\n"); 2593 | ast_cli(a->fd, "\n\n"); 2594 | return CLI_SUCCESS; 2595 | } 2596 | } 2597 | 2598 | static char *gdfe_show_config(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a) 2599 | { 2600 | struct gdf_config *config; 2601 | 2602 | switch (cmd) { 2603 | case CLI_INIT: 2604 | e->command = "gdfe show config"; 2605 | e->usage = 2606 | "Usage: gdfe show config\n" 2607 | " Show current gdfe configuration.\n"; 2608 | return NULL; 2609 | case CLI_GENERATE: 2610 | return NULL; 2611 | default: 2612 | config = gdf_get_config(); 2613 | if (config) { 2614 | struct ao2_iterator i; 2615 | struct ao2_iterator h; 2616 | char *hint; 2617 | struct gdf_logical_agent *agent; 2618 | 2619 | ast_cli(a->fd, "[general]\n"); 2620 | ast_cli(a->fd, "service_key = %s\n", config->service_key); 2621 | ast_cli(a->fd, "endpoint = %s\n", config->endpoint); 2622 | ast_cli(a->fd, "vad_voice_threshold = %d\n", config->vad_voice_threshold); 2623 | ast_cli(a->fd, "vad_voice_minimum_duration = %d\n", config->vad_voice_minimum_duration); 2624 | ast_cli(a->fd, "vad_silence_minimum_duration = %d\n", config->vad_silence_minimum_duration); 2625 | ast_cli(a->fd, "vad_barge_minimum_duration = %d\n", config->vad_barge_minimum_duration); 2626 | ast_cli(a->fd, "vad_end_of_speech_silence_duration = %d\n", config->vad_end_of_speech_silence_duration); 2627 | ast_cli(a->fd, "endpointer_cache_audio_pretrigger_ms = %d\n", config->endpointer_cache_audio_pretrigger_ms); 2628 | ast_cli(a->fd, "default_incomplete_timeout = %d\n", config->default_incomplete_timeout); 2629 | ast_cli(a->fd, "default_no_speech_timeout = %d\n", config->default_no_speech_timeout); 2630 | ast_cli(a->fd, "default_maximum_speech_timeout = %d\n", config->default_maximum_speech_timeout); 2631 | ast_cli(a->fd, "call_log_location = %s\n", config->call_log_location); 2632 | ast_cli(a->fd, "enable_call_logs = %s\n", AST_CLI_YESNO(config->enable_call_logs)); 2633 | ast_cli(a->fd, "enable_preendpointer_recordings = %s\n", AST_CLI_YESNO(config->enable_preendpointer_recordings)); 2634 | ast_cli(a->fd, "enable_postendpointer_recordings = %s\n", AST_CLI_YESNO(config->enable_postendpointer_recordings)); 2635 | ast_cli(a->fd, "record_preendpointer_on_demand = %s\n", AST_CLI_YESNO(config->record_preendpointer_on_demand)); 2636 | ast_cli(a->fd, "enable_sentiment_analysis = %s\n", config->enable_sentiment_analysis == SENTIMENT_ANALYSIS_ALWAYS ? "always" : 2637 | config->enable_sentiment_analysis == SENTIMENT_ANALYSIS_DEFAULT ? "default" : "never"); 2638 | ast_cli(a->fd, "stop_writes_on_final_transcription = %s\n", AST_CLI_YESNO(config->stop_writes_on_final_transcription)); 2639 | ast_cli(a->fd, "start_recognition_on_start = %s\n", AST_CLI_YESNO(config->start_recognition_on_start)); 2640 | ast_cli(a->fd, "recognition_start_failure_retries = %d\n", config->recognition_start_failure_retries); 2641 | ast_cli(a->fd, "recognition_start_failure_retry_max_time_ms = %d\n", config->recognition_start_failure_retry_max_time_ms); 2642 | ast_cli(a->fd, "start_failure_retry_codes = %s\n", config->start_failure_retry_codes); 2643 | ast_cli(a->fd, "synthesize_fulfillment_text = %s\n", AST_CLI_YESNO(config->synthesize_fulfillment_text)); 2644 | ast_cli(a->fd, "model = %s\n", config->model); 2645 | ast_cli(a->fd, "use_internal_endpointer_for_end_of_speech = %s\n", AST_CLI_YESNO(config->use_internal_endpointer_for_end_of_speech)); 2646 | ast_cli(a->fd, "hints = "); 2647 | h = ao2_iterator_init(config->hints, 0); 2648 | while ((hint = ao2_iterator_next(&h))) { 2649 | ast_cli(a->fd, "%s, ", hint); 2650 | ao2_ref(hint, -1); 2651 | } 2652 | ast_cli(a->fd, "\n"); 2653 | ao2_iterator_destroy(&h); 2654 | i = ao2_iterator_init(config->logical_agents, 0); 2655 | while ((agent = ao2_iterator_next(&i))) { 2656 | ast_cli(a->fd, "\n[%s]\n", agent->name); 2657 | ast_cli(a->fd, "project_id = %s\n", agent->project_id); 2658 | ast_cli(a->fd, "endpoint = %s\n", agent->endpoint); 2659 | ast_cli(a->fd, "service_key = %s\n", agent->service_key); 2660 | ast_cli(a->fd, "model = %s\n", agent->model); 2661 | ast_cli(a->fd, "use_internal_endpointer_for_end_of_speech = %s\n", AST_CLI_YESNO(agent->use_internal_endpointer_for_end_of_speech)); 2662 | ast_cli(a->fd, "enable_sentiment_analysis = %s\n", agent->enable_sentiment_analysis == SENTIMENT_ANALYSIS_ALWAYS ? "always" : 2663 | agent->enable_sentiment_analysis == SENTIMENT_ANALYSIS_DEFAULT ? "default" : "never"); 2664 | ast_cli(a->fd, "hints = "); 2665 | h = ao2_iterator_init(agent->hints, 0); 2666 | while ((hint = ao2_iterator_next(&h))) { 2667 | ast_cli(a->fd, "%s, ", hint); 2668 | ao2_ref(hint, -1); 2669 | } 2670 | ast_cli(a->fd, "\n"); 2671 | ao2_iterator_destroy(&h); 2672 | ao2_ref(agent, -1); 2673 | } 2674 | ao2_iterator_destroy(&i); 2675 | ao2_ref(config, -1); 2676 | } else { 2677 | ast_cli(a->fd, "Unable to retrieve configuration\n"); 2678 | } 2679 | ast_cli(a->fd, "\n"); 2680 | return CLI_SUCCESS; 2681 | } 2682 | } 2683 | 2684 | static struct ast_cli_entry gdfe_cli[] = { 2685 | AST_CLI_DEFINE(gdfe_reload, "Reload gdfe configuration"), 2686 | AST_CLI_DEFINE(gdfe_show_config, "Show current gdfe configuration"), 2687 | }; 2688 | 2689 | static int call_log_enabled_for_pvt(struct gdf_pvt *pvt) 2690 | { 2691 | struct gdf_config *config; 2692 | int log_enabled = 0; 2693 | 2694 | config = gdf_get_config(); 2695 | if (config) { 2696 | log_enabled = config->enable_call_logs; 2697 | if (log_enabled) { 2698 | ao2_lock(pvt); 2699 | log_enabled = (pvt->call_log_file_handle != NULL); 2700 | ao2_unlock(pvt); 2701 | } 2702 | 2703 | ao2_t_ref(config, -1, "done with config in log check"); 2704 | } 2705 | return log_enabled; 2706 | } 2707 | 2708 | #ifndef ASTERISK_13_OR_LATER 2709 | #define AST_ISO8601_LEN 29 2710 | #endif 2711 | 2712 | static void gdf_log_call_event(struct gdf_pvt *pvt, struct gdf_request *req, enum gdf_call_log_type type, const char *event, size_t log_data_size, const struct dialogflow_log_data *log_data) 2713 | { 2714 | struct timeval timeval_now; 2715 | struct ast_tm tm_now = {}; 2716 | char char_now[AST_ISO8601_LEN]; 2717 | const char *char_type; 2718 | char *log_line; 2719 | size_t i; 2720 | #ifdef ASTERISK_13_OR_LATER 2721 | RAII_VAR(struct ast_json *, log_message, ast_json_object_create(), ast_json_unref); 2722 | #else 2723 | json_t *log_message; 2724 | #endif 2725 | 2726 | if (!call_log_enabled_for_pvt(pvt)) { 2727 | return; 2728 | } 2729 | 2730 | timeval_now = ast_tvnow(); 2731 | ast_localtime(&timeval_now, &tm_now, NULL); 2732 | 2733 | ast_strftime(char_now, sizeof(char_now), "%FT%T.%q%z", &tm_now); 2734 | 2735 | if (type == CALL_LOG_TYPE_SESSION) { 2736 | char_type = "SESSION"; 2737 | } else if (type == CALL_LOG_TYPE_RECOGNITION) { 2738 | char_type = "RECOGNITION"; 2739 | } else if (type == CALL_LOG_TYPE_ENDPOINTER) { 2740 | char_type = "ENDPOINTER"; 2741 | } else if (type == CALL_LOG_TYPE_DIALOGFLOW) { 2742 | char_type = "DIALOGFLOW"; 2743 | } else { 2744 | char_type = "UNKNOWN"; 2745 | } 2746 | 2747 | #ifdef ASTERISK_13_OR_LATER 2748 | ast_json_object_set(log_message, "log_timestamp", ast_json_string_create(char_now)); 2749 | ast_json_object_set(log_message, "log_type", ast_json_string_create(char_type)); 2750 | ast_json_object_set(log_message, "log_event", ast_json_string_create(event)); 2751 | for (i = 0; i < log_data_size; i++) { 2752 | if (log_data[i].value_type == dialogflow_log_data_value_type_string) { 2753 | ast_json_object_set(log_message, log_data[i].name, ast_json_string_create((const char *)log_data[i].value)); 2754 | } else if (log_data[i].value_type == dialogflow_log_data_value_type_array_of_string) { 2755 | size_t j; 2756 | RAII_VAR(struct ast_json *, array, ast_json_array_create(), ast_json_unref); 2757 | 2758 | for (j = 0; j < log_data[i].value_count; j++) { 2759 | ast_json_array_append(log_message, ast_json_string_create(((const char **)log_data[i].value)[j])); 2760 | } 2761 | ast_json_object_set(log_message, log_data[i].name, ast_json_ref(array)); 2762 | } 2763 | } 2764 | log_line = ast_json_dump_string(log_message); 2765 | #else 2766 | log_message = json_object(); 2767 | json_object_set_new(log_message, "log_timestamp", json_string(char_now)); 2768 | json_object_set_new(log_message, "log_type", json_string(char_type)); 2769 | json_object_set_new(log_message, "log_event", json_string(event)); 2770 | if (req) { 2771 | json_object_set_new(log_message, "request_number", json_integer(req->current_utterance_number)); 2772 | } 2773 | for (i = 0; i < log_data_size; i++) { 2774 | if (log_data[i].value_type == dialogflow_log_data_value_type_string) { 2775 | json_object_set_new(log_message, log_data[i].name, json_string((const char *)log_data[i].value)); 2776 | } else if (log_data[i].value_type == dialogflow_log_data_value_type_array_of_string) { 2777 | size_t j; 2778 | json_t *array = json_array(); 2779 | 2780 | for (j = 0; j < log_data[i].value_count; j++) { 2781 | json_array_append_new(array, json_string(((const char **)log_data[i].value)[j])); 2782 | } 2783 | json_object_set_new(log_message, log_data[i].name, array); 2784 | } 2785 | } 2786 | log_line = json_dumps(log_message, JSON_COMPACT | JSON_PRESERVE_ORDER); 2787 | #endif 2788 | 2789 | ao2_lock(pvt); 2790 | fprintf(pvt->call_log_file_handle, "%s\n", log_line); 2791 | fflush(pvt->call_log_file_handle); 2792 | ao2_unlock(pvt); 2793 | 2794 | #ifdef ASTERISK_13_OR_LATER 2795 | ast_json_free(log_line); 2796 | #else 2797 | json_decref(log_message); 2798 | ast_free(log_line); 2799 | #endif 2800 | } 2801 | 2802 | static void libdialogflow_general_logging_callback(enum dialogflow_log_level level, 2803 | const char *file, int line, const char *function, const char *fmt, va_list args) 2804 | __attribute__ ((format(printf, 5, 0))); 2805 | 2806 | static void libdialogflow_general_logging_callback(enum dialogflow_log_level level, 2807 | const char *file, int line, const char *function, const char *fmt, va_list args) 2808 | { 2809 | size_t len; 2810 | char *buff; 2811 | va_list args2; 2812 | va_copy(args2, args); 2813 | len = vsnprintf(NULL, 0, fmt, args2); 2814 | va_end(args2); 2815 | buff = alloca(len + 1); 2816 | vsnprintf(buff, len + 1, fmt, args); 2817 | 2818 | ast_log((int) level, file, line, function, "%s", buff); 2819 | } 2820 | 2821 | static void libdialogflow_call_logging_callback(void *user_data, const char *event, size_t log_data_size, const struct dialogflow_log_data *data) 2822 | { 2823 | struct gdf_request *req = (struct gdf_request *) user_data; 2824 | gdf_log_call_event(req->pvt, req, CALL_LOG_TYPE_DIALOGFLOW, event, log_data_size, data); 2825 | } 2826 | 2827 | static char gdf_engine_name[] = "GoogleDFE"; 2828 | 2829 | static struct ast_speech_engine gdf_engine = { 2830 | .name = gdf_engine_name, 2831 | .create = gdf_create, 2832 | .destroy = gdf_destroy, 2833 | .load = gdf_load, 2834 | .unload = gdf_unload, 2835 | .activate = gdf_activate, 2836 | .deactivate = gdf_deactivate, 2837 | .write = gdf_write, 2838 | .dtmf = gdf_dtmf, 2839 | .start = gdf_start, 2840 | .change = gdf_change, 2841 | #ifdef AST_SPEECH_HAVE_GET_SETTING 2842 | .get_setting = gdf_get_setting, 2843 | #endif 2844 | .change_results_type = gdf_change_results_type, 2845 | .get = gdf_get_results, 2846 | }; 2847 | 2848 | #ifndef ASTERISK_13_OR_LATER 2849 | static void *json_custom_malloc(size_t sz) 2850 | { 2851 | return ast_malloc(sz); 2852 | } 2853 | static void json_custom_free(void *ptr) 2854 | { 2855 | ast_free(ptr); 2856 | } 2857 | #endif 2858 | 2859 | #pragma GCC diagnostic ignored "-Wmissing-format-attribute" 2860 | static enum ast_module_load_result load_module(void) 2861 | { 2862 | struct gdf_config *cfg; 2863 | 2864 | #ifndef ASTERISK_13_OR_LATER 2865 | json_set_alloc_funcs(json_custom_malloc, json_custom_free); 2866 | #endif 2867 | 2868 | config = ao2_container_alloc(1, NULL, NULL); 2869 | if (!config) { 2870 | ast_log(LOG_ERROR, "Failed to allocate configuration container\n"); 2871 | return AST_MODULE_LOAD_FAILURE; 2872 | } 2873 | 2874 | cfg = ao2_alloc(sizeof(*cfg), gdf_config_destroy); 2875 | if (!cfg) { 2876 | ast_log(LOG_ERROR, "Failed to allocate blank configuration\n"); 2877 | ao2_ref(config, -1); 2878 | return AST_MODULE_LOAD_FAILURE; 2879 | } 2880 | 2881 | ao2_link(config, cfg); 2882 | ao2_ref(cfg, -1); 2883 | 2884 | if (load_config(0)) { 2885 | ast_log(LOG_WARNING, "Failed to load configuration\n"); 2886 | } 2887 | 2888 | #ifdef ASTERISK_13_OR_LATER 2889 | gdf_engine.formats = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT); 2890 | 2891 | if (!gdf_engine.formats) { 2892 | ast_log(LOG_ERROR, "DFE speech could not create format caps\n"); 2893 | ao2_ref(config, -1); 2894 | return AST_MODULE_LOAD_FAILURE; 2895 | } 2896 | 2897 | ast_format_cap_append(gdf_engine.formats, ast_format_ulaw, 20); 2898 | #else 2899 | gdf_engine.formats = AST_FORMAT_SLINEAR; 2900 | #endif 2901 | 2902 | if (ast_speech_register(&gdf_engine)) { 2903 | ast_log(LOG_WARNING, "DFE speech failed to register with speech subsystem\n"); 2904 | ao2_ref(config, -1); 2905 | return AST_MODULE_LOAD_FAILURE; 2906 | } 2907 | 2908 | if (df_init(libdialogflow_general_logging_callback, libdialogflow_call_logging_callback)) { 2909 | ast_log(LOG_WARNING, "Failed to initialize dialogflow library\n"); 2910 | ao2_ref(config, -1); 2911 | return AST_MODULE_LOAD_FAILURE; 2912 | } 2913 | 2914 | ast_cli_register_multiple(gdfe_cli, ARRAY_LEN(gdfe_cli)); 2915 | 2916 | return AST_MODULE_LOAD_SUCCESS; 2917 | } 2918 | 2919 | static int unload_module(void) 2920 | { 2921 | if (ast_speech_unregister(gdf_engine.name)) { 2922 | ast_log(LOG_WARNING, "Failed to unregister GDF speech engine\n"); 2923 | return -1; 2924 | } 2925 | 2926 | ast_cli_unregister_multiple(gdfe_cli, ARRAY_LEN(gdfe_cli)); 2927 | 2928 | #ifdef ASTERISK_13_OR_LATER 2929 | ao2_t_ref(gdf_engine.formats, -1, "unloading module"); 2930 | #endif 2931 | 2932 | return 0; 2933 | } 2934 | 2935 | AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Google DialogFlow for Enterprise (DFE) Speech Engine"); --------------------------------------------------------------------------------