├── app-engine-front-end ├── img │ ├── favicon.ico │ ├── icon_3_pdf_x32.png │ ├── file-preview-pdf.png │ ├── icon_1_word_x32.png │ └── sample-outline-map.jpg ├── fonts │ ├── fontawesome-webfont.eot │ ├── fontawesome-webfont.ttf │ ├── fontawesome-webfont.woff │ └── fontawesome-webfont.woff2 ├── README.md ├── ep-upcomingMeetings.py ├── ep-jumbotronContent.py ├── ep-segmentMarkers.py ├── app.yaml ├── utilities.py ├── ep-relatedFiles.py ├── ep-searchVideo.py ├── ep-meetingDetails.py ├── ep-meetingArchive.py └── ep-searchArchive.py ├── generate-wordcloud ├── fonts │ └── LilitaOne-Regular.ttf ├── procfile ├── requirements.txt ├── monitor.py ├── Dockerfile ├── README.md ├── worker.py └── stopwords-20180109-133115.json ├── publish-pdf-transcript ├── fonts │ ├── Roboto-Black.ttf │ ├── Roboto-Bold.ttf │ ├── Roboto-Italic.ttf │ ├── Roboto-Light.ttf │ ├── Roboto-Medium.ttf │ ├── Roboto-Thin.ttf │ ├── Roboto-Regular.ttf │ ├── Roboto-BlackItalic.ttf │ ├── Roboto-BoldItalic.ttf │ ├── Roboto-LightItalic.ttf │ ├── Roboto-ThinItalic.ttf │ └── Roboto-MediumItalic.ttf ├── procfile ├── requirements.txt ├── monitor.py ├── README.md ├── Dockerfile └── worker.py ├── create-word-list ├── procfile ├── requirements.txt ├── monitor.py ├── Dockerfile ├── README.md └── worker.py ├── index-meeting ├── procfile ├── requirements.txt ├── monitor.py ├── README.md ├── Dockerfile └── worker.py ├── in-video-search ├── requirements.txt ├── app.yaml ├── main_test.py ├── README.md └── main.py ├── transcode-video-to-audio ├── procfile ├── requirements.txt ├── monitor.py ├── README.md ├── Dockerfile └── worker.py ├── archive-video-search ├── requirements.txt ├── main_test.py ├── app.yaml ├── README.md └── main.py ├── app-engine-utility-service ├── requirements.txt ├── main.py ├── dispatch.yaml ├── cron.yaml ├── msgPublish.py ├── toggleIndex.py ├── toggleTranscode.py ├── toggleTranscriptErr.py ├── README.md ├── app.yaml ├── idWordcloud.py ├── idTranscript.py ├── idTranscode.py ├── meetingDetails.py ├── runRecognize.py ├── speechJobs.py ├── utilities.py └── receiveResults.py ├── CONTRIBUTING.md ├── README.md └── LICENSE.md /app-engine-front-end/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/app-engine-front-end/img/favicon.ico -------------------------------------------------------------------------------- /app-engine-front-end/img/icon_3_pdf_x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/app-engine-front-end/img/icon_3_pdf_x32.png -------------------------------------------------------------------------------- /app-engine-front-end/img/file-preview-pdf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/app-engine-front-end/img/file-preview-pdf.png -------------------------------------------------------------------------------- /app-engine-front-end/img/icon_1_word_x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/app-engine-front-end/img/icon_1_word_x32.png -------------------------------------------------------------------------------- /generate-wordcloud/fonts/LilitaOne-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/generate-wordcloud/fonts/LilitaOne-Regular.ttf -------------------------------------------------------------------------------- /publish-pdf-transcript/fonts/Roboto-Black.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/publish-pdf-transcript/fonts/Roboto-Black.ttf -------------------------------------------------------------------------------- /publish-pdf-transcript/fonts/Roboto-Bold.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/publish-pdf-transcript/fonts/Roboto-Bold.ttf -------------------------------------------------------------------------------- /publish-pdf-transcript/fonts/Roboto-Italic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/publish-pdf-transcript/fonts/Roboto-Italic.ttf -------------------------------------------------------------------------------- /publish-pdf-transcript/fonts/Roboto-Light.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/publish-pdf-transcript/fonts/Roboto-Light.ttf -------------------------------------------------------------------------------- /publish-pdf-transcript/fonts/Roboto-Medium.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/publish-pdf-transcript/fonts/Roboto-Medium.ttf -------------------------------------------------------------------------------- /publish-pdf-transcript/fonts/Roboto-Thin.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/publish-pdf-transcript/fonts/Roboto-Thin.ttf -------------------------------------------------------------------------------- /app-engine-front-end/img/sample-outline-map.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/app-engine-front-end/img/sample-outline-map.jpg -------------------------------------------------------------------------------- /publish-pdf-transcript/fonts/Roboto-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/publish-pdf-transcript/fonts/Roboto-Regular.ttf -------------------------------------------------------------------------------- /app-engine-front-end/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/app-engine-front-end/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /app-engine-front-end/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/app-engine-front-end/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /app-engine-front-end/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/app-engine-front-end/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /publish-pdf-transcript/fonts/Roboto-BlackItalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/publish-pdf-transcript/fonts/Roboto-BlackItalic.ttf -------------------------------------------------------------------------------- /publish-pdf-transcript/fonts/Roboto-BoldItalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/publish-pdf-transcript/fonts/Roboto-BoldItalic.ttf -------------------------------------------------------------------------------- /publish-pdf-transcript/fonts/Roboto-LightItalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/publish-pdf-transcript/fonts/Roboto-LightItalic.ttf -------------------------------------------------------------------------------- /publish-pdf-transcript/fonts/Roboto-ThinItalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/publish-pdf-transcript/fonts/Roboto-ThinItalic.ttf -------------------------------------------------------------------------------- /app-engine-front-end/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/app-engine-front-end/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /publish-pdf-transcript/fonts/Roboto-MediumItalic.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/gov-meetings-made-searchable/HEAD/publish-pdf-transcript/fonts/Roboto-MediumItalic.ttf -------------------------------------------------------------------------------- /create-word-list/procfile: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | worker: python /app/worker.py 21 | monitor: python monitor.py 22 | -------------------------------------------------------------------------------- /index-meeting/procfile: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | worker: python /app/worker.py 21 | monitor: python monitor.py 22 | -------------------------------------------------------------------------------- /generate-wordcloud/procfile: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | worker: python /app/worker.py 21 | monitor: python monitor.py 22 | -------------------------------------------------------------------------------- /in-video-search/requirements.txt: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | requests 21 | Flask==1.0.2 22 | elasticsearch 23 | gunicorn==19.9.0 -------------------------------------------------------------------------------- /publish-pdf-transcript/procfile: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | worker: python /app/worker.py 21 | monitor: python monitor.py 22 | -------------------------------------------------------------------------------- /transcode-video-to-audio/procfile: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | worker: python /app/worker.py 20 | monitor: python monitor.py 21 | -------------------------------------------------------------------------------- /archive-video-search/requirements.txt: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | requests 21 | Flask==1.0.2 22 | elasticsearch 23 | gunicorn==19.9.0 -------------------------------------------------------------------------------- /app-engine-utility-service/requirements.txt: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | google-api-python-client 21 | google-cloud-pubsub 22 | BeautifulSoup 23 | feedparser 24 | gcloud 25 | httplib2 -------------------------------------------------------------------------------- /app-engine-utility-service/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | def main(): 23 | 24 | print "Content-type: text/plain; charset=UTF-8\n\n" 25 | print "Nothing To See Here" 26 | 27 | 28 | if __name__ == '__main__': 29 | main() -------------------------------------------------------------------------------- /transcode-video-to-audio/requirements.txt: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | Flask==0.10.1 21 | requests==2.18.4 22 | google-cloud-storage==1.6.0 23 | urllib3==1.22 24 | gunicorn==19.6.0 25 | PyMySQL==0.7.3 26 | six==1.10.0 27 | honcho==0.7.1 28 | oauth2client==4.1.2 -------------------------------------------------------------------------------- /create-word-list/requirements.txt: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | Flask==0.10.1 21 | requests==2.18.4 22 | google-cloud-pubsub==0.30.1 23 | google-cloud-storage==1.6.0 24 | urllib3==1.22 25 | gunicorn==19.6.0 26 | six==1.10.0 27 | honcho==0.7.1 28 | oauth2client==4.1.2 29 | httplib2==0.10.3 -------------------------------------------------------------------------------- /app-engine-utility-service/dispatch.yaml: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | dispatch: 21 | - url: "__website_URL__/*" 22 | service: engaged-citizens 23 | - url: "*/__website_URL__/*" 24 | service: engaged-citizens 25 | - url: "www.__website_URL__/*" 26 | service: engaged-citizens -------------------------------------------------------------------------------- /publish-pdf-transcript/requirements.txt: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | Flask==0.10.1 21 | requests==2.18.4 22 | google-cloud-pubsub==0.30.1 23 | google-cloud-storage==1.6.0 24 | urllib3==1.22 25 | gunicorn==19.6.0 26 | six==1.10.0 27 | honcho==0.7.1 28 | oauth2client==4.1.2 29 | pdfkit==0.6.1 30 | -------------------------------------------------------------------------------- /index-meeting/requirements.txt: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | Flask==0.10.1 21 | requests==2.18.4 22 | google-cloud-pubsub==0.30.1 23 | google-cloud-storage==1.6.0 24 | urllib3==1.22 25 | gunicorn==19.6.0 26 | PyMySQL==0.7.3 27 | six==1.10.0 28 | honcho==0.7.1 29 | oauth2client==4.1.2 30 | elasticsearch==6.3.1 -------------------------------------------------------------------------------- /in-video-search/app.yaml: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | runtime: python 21 | env: flex 22 | service: in-video-search 23 | entrypoint: gunicorn -b :$PORT main:app 24 | 25 | runtime_config: 26 | python_version: 3 27 | 28 | manual_scaling: 29 | instances: 1 30 | 31 | resources: 32 | cpu: 1 33 | memory_gb: 0.5 34 | disk_size_gb: 10 35 | -------------------------------------------------------------------------------- /in-video-search/main_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import main 23 | 24 | 25 | def test_index(): 26 | main.app.testing = True 27 | client = main.app.test_client() 28 | 29 | r = client.get("/") 30 | assert r.status_code == 200 31 | assert "Hello World" in r.data.decode("utf-8") 32 | -------------------------------------------------------------------------------- /archive-video-search/main_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import main 23 | 24 | 25 | def test_index(): 26 | main.app.testing = True 27 | client = main.app.test_client() 28 | 29 | r = client.get("/") 30 | assert r.status_code == 200 31 | assert "Hello World" in r.data.decode("utf-8") 32 | -------------------------------------------------------------------------------- /archive-video-search/app.yaml: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | runtime: python 21 | env: flex 22 | service: archive-video-search 23 | entrypoint: gunicorn -b :$PORT main:app 24 | 25 | runtime_config: 26 | python_version: 3 27 | 28 | manual_scaling: 29 | instances: 1 30 | 31 | resources: 32 | cpu: 1 33 | memory_gb: 0.5 34 | disk_size_gb: 10 35 | -------------------------------------------------------------------------------- /generate-wordcloud/requirements.txt: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | Flask==0.10.1 21 | requests==2.18.4 22 | google-cloud-pubsub==0.30.1 23 | google-cloud-storage==1.6.0 24 | urllib3==1.22 25 | gunicorn==19.6.0 26 | six==1.10.0 27 | honcho==0.7.1 28 | oauth2client==4.1.2 29 | scipy==1.0.0 30 | nltk==3.2.4 31 | Pillow==4.3.0 32 | wordcloud==1.3.1 33 | matplotlib==2.1.0 -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows 28 | [Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). -------------------------------------------------------------------------------- /create-word-list/monitor.py: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | from flask import Flask 21 | 22 | monitor_app = Flask(__name__) 23 | 24 | 25 | # The health check reads the PID file created by psqworker and checks the proc 26 | # filesystem to see if the worker is running. This same pattern can be used for 27 | # rq and celery. 28 | @monitor_app.route('/_ah/health') 29 | def health(): 30 | return 'healthy', 200 31 | 32 | 33 | @monitor_app.route('/') 34 | def index(): 35 | return health() 36 | 37 | if __name__ == '__main__': 38 | monitor_app.run('0.0.0.0', 8080) 39 | -------------------------------------------------------------------------------- /app-engine-utility-service/cron.yaml: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | cron: 21 | - description: runRecognize 22 | url: /runRecognize 23 | schedule: every 2 minutes 24 | retry_parameters: 25 | min_backoff_seconds: 2.5 26 | max_doublings: 5 27 | 28 | - description: receiveResults 29 | url: /receiveResults 30 | schedule: every 2 minutes 31 | retry_parameters: 32 | min_backoff_seconds: 2.5 33 | max_doublings: 5 34 | 35 | - description: speechJobs 36 | url: /speechJobs 37 | schedule: every 5 minutes 38 | retry_parameters: 39 | min_backoff_seconds: 2.5 40 | max_doublings: 5 -------------------------------------------------------------------------------- /index-meeting/monitor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | from flask import Flask 23 | 24 | monitor_app = Flask(__name__) 25 | 26 | 27 | # The health check reads the PID file created by psqworker and checks the proc 28 | # filesystem to see if the worker is running. This same pattern can be used for 29 | # rq and celery. 30 | @monitor_app.route('/_ah/health') 31 | def health(): 32 | return 'healthy', 200 33 | 34 | 35 | @monitor_app.route('/') 36 | def index(): 37 | return health() 38 | 39 | if __name__ == '__main__': 40 | monitor_app.run('0.0.0.0', 8080) 41 | -------------------------------------------------------------------------------- /generate-wordcloud/monitor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | from flask import Flask 23 | 24 | monitor_app = Flask(__name__) 25 | 26 | 27 | # The health check reads the PID file created by psqworker and checks the proc 28 | # filesystem to see if the worker is running. This same pattern can be used for 29 | # rq and celery. 30 | @monitor_app.route('/_ah/health') 31 | def health(): 32 | return 'healthy', 200 33 | 34 | 35 | @monitor_app.route('/') 36 | def index(): 37 | return health() 38 | 39 | if __name__ == '__main__': 40 | monitor_app.run('0.0.0.0', 8080) 41 | -------------------------------------------------------------------------------- /publish-pdf-transcript/monitor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | from flask import Flask 23 | 24 | monitor_app = Flask(__name__) 25 | 26 | 27 | # The health check reads the PID file created by psqworker and checks the proc 28 | # filesystem to see if the worker is running. This same pattern can be used for 29 | # rq and celery. 30 | @monitor_app.route('/_ah/health') 31 | def health(): 32 | return 'healthy', 200 33 | 34 | 35 | @monitor_app.route('/') 36 | def index(): 37 | return health() 38 | 39 | if __name__ == '__main__': 40 | monitor_app.run('0.0.0.0', 8080) 41 | -------------------------------------------------------------------------------- /transcode-video-to-audio/monitor.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | from flask import Flask 23 | 24 | monitor_app = Flask(__name__) 25 | 26 | 27 | # The health check reads the PID file created by psqworker and checks the proc 28 | # filesystem to see if the worker is running. This same pattern can be used for 29 | # rq and celery. 30 | @monitor_app.route('/_ah/health') 31 | def health(): 32 | return 'healthy', 200 33 | 34 | 35 | @monitor_app.route('/') 36 | def index(): 37 | return health() 38 | 39 | if __name__ == '__main__': 40 | monitor_app.run('0.0.0.0', 8080) 41 | -------------------------------------------------------------------------------- /in-video-search/README.md: -------------------------------------------------------------------------------- 1 | ## App Engine Flex Service for "In" Video Search 2 | 3 | This is not an officially supported Google product, though support will be provided on a best-effort basis. 4 | 5 | Copyright 2018 Google LLC 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | 20 | ### Introduction 21 | 22 | This is a Google App Engine Flex app that provides a wrapper and proxy for requests 23 | to and responses from an Elastic Search instance. This service handles searches for 24 | materials in a particular meeting. Meetings are identified with a `urlId` parameter. 25 | 26 | 27 | ### Assumptions 28 | 29 | * You have deployed an Elastic Search instance 30 | * You have deployed the `app-engine-front-end` service (`ep-searchVideo.py`) 31 | 32 | 33 | ### Deploy this App 34 | 35 | Deploy to App Engine `gcloud app deploy app.yaml` -------------------------------------------------------------------------------- /archive-video-search/README.md: -------------------------------------------------------------------------------- 1 | ## App Engine Flex Service for "In" Video Search 2 | 3 | This is not an officially supported Google product, though support will be provided on a best-effort basis. 4 | 5 | Copyright 2018 Google LLC 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | 20 | ### Introduction 21 | 22 | This is a Google App Engine Flex app that provides a wrapper and proxy for requests 23 | to and responses from an Elastic Search instance. This service handles searches for 24 | materials in a particular meeting. Meetings are identified with a `urlId` parameter. 25 | 26 | 27 | ### Assumptions 28 | 29 | * You have deployed an Elastic Search instance 30 | * You have deployed the `app-engine-front-end` service (`ep-searchVideo.py`) 31 | 32 | 33 | ### Deploy this App 34 | 35 | Deploy to App Engine `gcloud app deploy app.yaml` -------------------------------------------------------------------------------- /app-engine-front-end/README.md: -------------------------------------------------------------------------------- 1 | ## App Engine Front End Service 2 | 3 | This is not an officially supported Google product, though support will be provided on a best-effort basis. 4 | 5 | Copyright 2018 Google LLC 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | 20 | ### Introduction 21 | 22 | This is a Google App Engine app that provides a front end to the project. It uses the 23 | Material Design for Bootstrap component library and surfaces backend data via several 24 | endpoint (ep) API services. 25 | 26 | 27 | ### Assumptions 28 | 29 | * You have created a Google Cloud SQL database called `prodDb` 30 | * You have created a table in the `prodDb` database called `meetingRegistry` 31 | 32 | 33 | ### Test and Deploy this App 34 | 35 | Test Locally 36 | `dev_appserver.py app.yaml` 37 | 38 | Deploy to App Engine 39 | `gcloud app deploy --promote` -------------------------------------------------------------------------------- /app-engine-front-end/ep-upcomingMeetings.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | from google.appengine.ext import ndb 24 | 25 | from google.appengine.ext import vendor 26 | vendor.add(os.path.join(os.path.dirname(os.path.realpath(__file__)), "lib")) 27 | import ujson as json 28 | 29 | 30 | class upcomingMeetings(ndb.Model): 31 | scheduleJson = ndb.JsonProperty() 32 | 33 | 34 | def main(): 35 | keyObj = ndb.Key(upcomingMeetings, "currentSchedule") 36 | storageObj = keyObj.get() 37 | outputObj = storageObj.scheduleJson 38 | 39 | print "Content-Type: application/json\n" 40 | print json.dumps(outputObj) 41 | 42 | 43 | if __name__ == "__main__": 44 | main() 45 | -------------------------------------------------------------------------------- /index-meeting/README.md: -------------------------------------------------------------------------------- 1 | ## Container to Index the Transcript of a Meeting to Elastic Search 2 | 3 | This is not an officially supported Google product, though support will be provided on a best-effort basis. 4 | 5 | Copyright 2018 Google LLC 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | 20 | ### Introduction 21 | 22 | This is a container that receives pubsub messages with `globalId` identifiers and then 23 | parses Google Speech API responses and writes the contents to an Elastic Search index 24 | in a batch process. 25 | 26 | ### Assumptions 27 | 28 | * You have created a Pubsub topic called `indexQueue` 29 | * You have created a Pubsub subscription called `index-meeting-subscription` 30 | * You have deployed a utility service to handle database lookups and updates 31 | 32 | ### Build and Run this container 33 | 34 | Build 35 | `docker build -t index-meeting:latest .` 36 | 37 | Run 38 | `docker run -it index-meeting` -------------------------------------------------------------------------------- /app-engine-utility-service/msgPublish.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import cgi 23 | import utilities 24 | 25 | projectId = "__GCP_Project_ID__" 26 | 27 | 28 | def main(): 29 | passedArgs = cgi.FieldStorage() 30 | 31 | print "Content-type: text/plain; charset=UTF-8\n\n" 32 | 33 | if 1==1: 34 | gId = passedArgs["gId"].value 35 | gId = utilities.cleanInput(gId, 5) 36 | 37 | msgAction = passedArgs["msgAction"].value 38 | msgAction = utilities.cleanInput(msgAction, 22) 39 | 40 | topicName = passedArgs["topicName"].value 41 | topicName = utilities.cleanInput(topicName, 30) 42 | 43 | messageObj = utilities.publishMsg(projectId, gId, topicName, msgAction) 44 | print messageObj.get("messageIds")[0] 45 | 46 | 47 | if __name__ == '__main__': 48 | main() -------------------------------------------------------------------------------- /publish-pdf-transcript/README.md: -------------------------------------------------------------------------------- 1 | ## Container to Create a PDF Transcript from Google Speech API Results 2 | 3 | This is not an officially supported Google product, though support will be provided on a best-effort basis. 4 | 5 | Copyright 2018 Google LLC 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | 20 | ### Introduction 21 | 22 | This is a container that receives pubsub messages with `globalId` identifiers and then 23 | parses Google Speech API responses to produce a human readable PDF transcript. The PDF 24 | transcript is saved to Google Cloud Storage. 25 | 26 | ### Assumptions 27 | 28 | * You have created a Pubsub topic called `publish-transcript-queue` 29 | * You have created a Pubsub subscription called `publish-transcript-subscription` 30 | * You have deployed a utility service to handle database lookups and updates 31 | 32 | ### Build and Run this container 33 | 34 | Build 35 | `docker build -t publish-pdf-transcript:latest .` 36 | 37 | Run 38 | `docker run -it publish-pdf-transcript` -------------------------------------------------------------------------------- /transcode-video-to-audio/README.md: -------------------------------------------------------------------------------- 1 | ## Container to Transcode Video to Audio 2 | 3 | This is not an officially supported Google product, though support will be provided on a best-effort basis. 4 | 5 | Copyright 2018 Google LLC 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | 20 | ### Introduction 21 | 22 | This is a container that receives pubsub messages with `globalId` identifiers and then 23 | kicks off a workflow to lookup the video associated with that identifier, download the 24 | corresponding video from Google Cloud Storage, transcode that video to audio in a FLAC 25 | format, then upload the audio segments to Google Cloud Storage. 26 | 27 | 28 | ### Assumptions 29 | 30 | * You have created a Pubsub topic called `transcodeQueue` 31 | * You have created a Pubsub subscription called `media-transcode-subscription` 32 | * You have deployed a utility service to handle database lookups and updates 33 | 34 | ### Build and Run this container 35 | 36 | Build 37 | `docker build -t transcode-video-to-audio:latest .` 38 | 39 | Run 40 | `docker run -it transcode-video-to-audio` -------------------------------------------------------------------------------- /app-engine-front-end/ep-jumbotronContent.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import cgi 23 | import utilities 24 | import ujson as json 25 | 26 | 27 | def meetingCount(orgIdentifier): 28 | sqlCmd = """select 29 | count(*) from meetingRegistry 30 | where orgIdentifier = %s 31 | and youtubeId is not NULL""" 32 | sqlData = (orgIdentifier) 33 | resultList = utilities.dbExecution(sqlCmd, sqlData) 34 | 35 | return resultList[2][0][0] 36 | 37 | 38 | def main(): 39 | passedArgs = cgi.FieldStorage() 40 | orgIdentifier = passedArgs["orgId"].value 41 | 42 | outputObj = {} 43 | outputObj["headerTxt"] = "__Jumbotron_Display_Name_for_Municipality__" 44 | outputObj["supportingTxt"] = "Public meetings and hearings" 45 | outputObj["headerImg"] = "img/sample-outline-map.jpg" 46 | outputObj["videoCnt"] = meetingCount(orgIdentifier) 47 | 48 | print "Content-Type: application/json\n" 49 | print json.dumps(outputObj) 50 | 51 | 52 | if __name__ == "__main__": 53 | main() -------------------------------------------------------------------------------- /index-meeting/Dockerfile: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | FROM gcr.io/google_appengine/python 21 | 22 | RUN apt-get --allow-unauthenticated -y update 23 | 24 | RUN apt-get install curl 25 | 26 | RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz 27 | RUN mkdir -p /usr/local/gcloud 28 | RUN tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz 29 | RUN /usr/local/gcloud/google-cloud-sdk/install.sh 30 | ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin 31 | 32 | ADD __GCP_Project_ID__ /app/__GCP_Project_ID__ 33 | RUN gcloud auth activate-service-account --key-file=__GCP_Project_ID__ 34 | ENV GOOGLE_APPLICATION_CREDENTIALS /app/__GCP_Project_ID__ 35 | 36 | RUN virtualenv /env 37 | ENV VIRTUAL_ENV /env 38 | ENV PATH /env/bin:$PATH 39 | ADD requirements.txt /app/requirements.txt 40 | RUN pip install -r /app/requirements.txt 41 | ADD procfile /app/procfile 42 | ADD . /app 43 | CMD honcho start -f /app/procfile worker monitor 44 | ENV https_proxy "" 45 | ENV http_proxy "" -------------------------------------------------------------------------------- /app-engine-utility-service/toggleIndex.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import ujson 24 | import webapp2 25 | import utilities 26 | 27 | 28 | class main(webapp2.RequestHandler): 29 | def get(self): 30 | self.response.headers["Content-Type"] = "application/json" 31 | self.response.headers.add_header( 32 | "Cache-Control", 33 | "no-cache, no-store, must-revalidate, max-age=0" 34 | ) 35 | self.response.headers.add_header( 36 | "Expires", 37 | "0" 38 | ) 39 | 40 | try: 41 | globalId = self.request.get("gId") 42 | sqlCmd = "update meetingRegistry set beenIndexed = %s where globalId = %s" 43 | sqlData = (1, globalId) 44 | resultList = utilities.dbExecution(sqlCmd, sqlData) 45 | outputStr = str(resultList) 46 | except: 47 | outputStr = None 48 | 49 | resultObj = {} 50 | resultObj["response"] = outputStr 51 | 52 | self.response.out.write(ujson.dumps(resultObj)) 53 | 54 | 55 | app = webapp2.WSGIApplication([ 56 | ("/toggleIndex", main)], debug = True 57 | ) 58 | -------------------------------------------------------------------------------- /app-engine-utility-service/toggleTranscode.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import ujson 24 | import webapp2 25 | import utilities 26 | 27 | 28 | class main(webapp2.RequestHandler): 29 | def get(self): 30 | self.response.headers["Content-Type"] = "application/json" 31 | self.response.headers.add_header( 32 | "Cache-Control", 33 | "no-cache, no-store, must-revalidate, max-age=0" 34 | ) 35 | self.response.headers.add_header( 36 | "Expires", 37 | "0" 38 | ) 39 | 40 | try: 41 | globalId = self.request.get("gId") 42 | sqlCmd = "update meetingRegistry set beenTranscoded = %s where globalId = %s" 43 | sqlData = (1, globalId) 44 | resultList = utilities.dbExecution(sqlCmd, sqlData) 45 | outputStr = str(resultList) 46 | except: 47 | outputStr = None 48 | 49 | resultObj = {} 50 | resultObj["response"] = outputStr 51 | 52 | self.response.out.write(ujson.dumps(resultObj)) 53 | 54 | 55 | app = webapp2.WSGIApplication([ 56 | ("/toggleTranscode", main)], debug = True 57 | ) 58 | -------------------------------------------------------------------------------- /app-engine-utility-service/toggleTranscriptErr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import ujson 24 | import webapp2 25 | import utilities 26 | 27 | 28 | class main(webapp2.RequestHandler): 29 | def get(self): 30 | self.response.headers["Content-Type"] = "application/json" 31 | self.response.headers.add_header( 32 | "Cache-Control", 33 | "no-cache, no-store, must-revalidate, max-age=0" 34 | ) 35 | self.response.headers.add_header( 36 | "Expires", 37 | "0" 38 | ) 39 | 40 | try: 41 | globalId = self.request.get("gId") 42 | sqlCmd = "update meetingRegistry set transcriptErr = %s where globalId = %s" 43 | sqlData = (1, globalId) 44 | resultList = utilities.dbExecution(sqlCmd, sqlData) 45 | outputStr = str(resultList) 46 | except: 47 | outputStr = None 48 | 49 | resultObj = {} 50 | resultObj["response"] = outputStr 51 | 52 | self.response.out.write(ujson.dumps(resultObj)) 53 | 54 | 55 | app = webapp2.WSGIApplication([ 56 | ("/toggleTranscriptErr", main)], debug = True 57 | ) 58 | -------------------------------------------------------------------------------- /create-word-list/Dockerfile: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | FROM gcr.io/google_appengine/python 21 | 22 | RUN apt-get --allow-unauthenticated -y update 23 | 24 | RUN apt-get install curl 25 | 26 | 27 | RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz 28 | RUN mkdir -p /usr/local/gcloud 29 | RUN tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz 30 | RUN /usr/local/gcloud/google-cloud-sdk/install.sh 31 | ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin 32 | 33 | ADD __Credential_JSON_File_Name__ /app/__Credential_JSON_File_Name__ 34 | RUN gcloud auth activate-service-account --key-file=__Credential_JSON_File_Name__ 35 | ENV GOOGLE_APPLICATION_CREDENTIALS /app/__Credential_JSON_File_Name__ 36 | 37 | RUN virtualenv /env 38 | ENV VIRTUAL_ENV /env 39 | ENV PATH /env/bin:$PATH 40 | ADD requirements.txt /app/requirements.txt 41 | RUN pip install -r /app/requirements.txt 42 | 43 | ADD procfile /app/procfile 44 | ADD . /app 45 | CMD honcho start -f /app/procfile worker monitor 46 | ENV https_proxy "" 47 | ENV http_proxy "" -------------------------------------------------------------------------------- /create-word-list/README.md: -------------------------------------------------------------------------------- 1 | ## Container to Create a List of Words from Speech API Responses 2 | 3 | This is not an officially supported Google product, though support will be provided on a best-effort basis. 4 | 5 | Copyright 2018 Google LLC 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | 20 | ### Introduction 21 | 22 | This is a container that receives pubsub messages with `globalId` identifiers and then 23 | kicks off a workflow to retrieve all associated Google Speech API responses from Google 24 | Cloud Storage, parse these responses, and then output a file with a master list of words 25 | (one word per line) to Google Cloud Storage. This master list will be used in the 26 | development of a word cloud. 27 | 28 | In the `results_files_to_string` function there is workflow to replace words or phrases 29 | from the Speech API response with a different entry. 30 | 31 | ### Assumptions 32 | 33 | * You have created a Pubsub topic called `wordlistQueue` 34 | * You have created a Pubsub subscription called `word-list-creation-subscription` 35 | * You have deployed a utility service to handle database lookups and updates 36 | 37 | ### Build and Run this container 38 | 39 | Build 40 | `docker build -t create-word-list:latest .` 41 | 42 | Run 43 | `docker run -it create-word-list` -------------------------------------------------------------------------------- /app-engine-utility-service/README.md: -------------------------------------------------------------------------------- 1 | ## App Engine Utility Service 2 | 3 | This is not an officially supported Google product, though support will be provided on a best-effort basis. 4 | 5 | Copyright 2018 Google LLC 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | 20 | ### Introduction 21 | 22 | This is a Google App Engine app that facilitates interaction with the Google Cloud SQL 23 | database, and provides common utilities for back end services. 24 | 25 | 26 | ### Assumptions 27 | 28 | * You have created a Google Cloud SQL database called `prodDb` 29 | * You have created a table in the `prodDb` database called `meetingRegistry` 30 | * You have installed the third-party Python libraries in the `lib` directory `pip install -t lib -r requirements.txt` 31 | 32 | 33 | ### Test and Deploy this App 34 | 35 | Test Locally 36 | `dev_appserver.py app.yaml` 37 | 38 | Deploy to App Engine 39 | `gcloud app deploy --promote` 40 | 41 | 42 | ### Note on Security 43 | As implemented, these services and API end points are not secure. Anyone with the right 44 | URL and paths code read and write to the system. Before deploying this app into 45 | production, you will want to implement a custom security schema or standard authentication 46 | protocol (Basic API Authentication w/ TLS, OAuth2, etc). -------------------------------------------------------------------------------- /app-engine-utility-service/app.yaml: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | runtime: python27 21 | api_version: 1 22 | threadsafe: no 23 | 24 | libraries: 25 | - name: ssl 26 | version: latest 27 | - name: ujson 28 | version: "1.35" 29 | - name: MySQLdb 30 | version: "latest" 31 | 32 | handlers: 33 | - url: /meetingDetails 34 | script: meetingDetails.app 35 | 36 | - url: /toggleTranscode 37 | script: toggleTranscode.app 38 | 39 | - url: /toggleIndex 40 | script: toggleIndex.app 41 | 42 | - url: /toggleTranscriptErr 43 | script: toggleTranscriptErr.app 44 | 45 | - url: /idTranscode 46 | script: idTranscode.app 47 | 48 | - url: /idTranscript 49 | script: idTranscript.py 50 | 51 | - url: /idWordcloud 52 | script: idWordcloud.py 53 | 54 | - url: /processNew 55 | script: processNew.py 56 | 57 | - url: /msgPublish 58 | script: msgPublish.py 59 | 60 | - url: /speechJobs 61 | script: speechJobs.py 62 | 63 | - url: /runRecognize 64 | script: runRecognize.py 65 | 66 | - url: /receiveResults 67 | script: receiveResults.py 68 | 69 | - url: .* 70 | script: main.py 71 | 72 | env_variables: 73 | GAE_LONG_APP_ID: "__GCP_Project_ID__" -------------------------------------------------------------------------------- /generate-wordcloud/Dockerfile: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | FROM gcr.io/google_appengine/python 21 | 22 | RUN apt-get --allow-unauthenticated -y update 23 | 24 | RUN apt-get install curl 25 | 26 | RUN apt-get -y install python-tk 27 | 28 | RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz 29 | RUN mkdir -p /usr/local/gcloud 30 | RUN tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz 31 | RUN /usr/local/gcloud/google-cloud-sdk/install.sh 32 | ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin 33 | 34 | ADD __Credential_JSON_File_Name__ /app/__Credential_JSON_File_Name__ 35 | RUN gcloud auth activate-service-account --key-file=__Credential_JSON_File_Name__ 36 | ENV GOOGLE_APPLICATION_CREDENTIALS /app/__Credential_JSON_File_Name__ 37 | 38 | RUN virtualenv /env 39 | ENV VIRTUAL_ENV /env 40 | ENV PATH /env/bin:$PATH 41 | ADD requirements.txt /app/requirements.txt 42 | RUN pip install -r /app/requirements.txt 43 | 44 | RUN python -m nltk.downloader stopwords 45 | 46 | ADD procfile /app/procfile 47 | ADD . /app 48 | CMD honcho start -f /app/procfile worker monitor 49 | ENV https_proxy "" 50 | ENV http_proxy "" -------------------------------------------------------------------------------- /app-engine-front-end/ep-segmentMarkers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | #!/usr/bin/env python 4 | 5 | # This is not an officially supported Google product, though support 6 | # will be provided on a best-effort basis. 7 | 8 | # Copyright 2018 Google LLC 9 | 10 | # Licensed under the Apache License, Version 2.0 (the "License"); you 11 | # may not use this file except in compliance with the License. 12 | 13 | # You may obtain a copy of the License at: 14 | 15 | # https://www.apache.org/licenses/LICENSE-2.0 16 | 17 | # Unless required by applicable law or agreed to in writing, software 18 | # distributed under the License is distributed on an "AS IS" BASIS, 19 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 20 | # See the License for the specific language governing permissions and 21 | # limitations under the License. 22 | 23 | 24 | import os 25 | import cgi 26 | import utilities 27 | import ujson as json 28 | from datetime import datetime 29 | 30 | 31 | def lookupFiles(urlIdentifier): 32 | sqlCmd = """select 33 | segmentJson from videoSegments 34 | where urlIdentifier = %s""" 35 | sqlData = (urlIdentifier) 36 | resultList = utilities.dbExecution(sqlCmd, sqlData) 37 | 38 | segmentJson = resultList[2][0][0] 39 | 40 | return segmentJson 41 | 42 | 43 | def main(): 44 | errorFound = False 45 | 46 | passedArgs = cgi.FieldStorage() 47 | 48 | try: 49 | urlIdentifier = passedArgs["urlIdentifier"].value 50 | resultObj = lookupFiles(urlIdentifier) 51 | except: 52 | meetingId = None 53 | errorFound = True 54 | resultObj = None 55 | 56 | try: 57 | outputObj = json.loads(resultObj) 58 | except: 59 | errorFound = True 60 | 61 | if errorFound is True: 62 | outputObj = {} 63 | outputObj["error"] = "Error" 64 | 65 | print "Content-Type: application/json\n" 66 | print json.dumps(outputObj) 67 | 68 | 69 | if __name__ == "__main__": 70 | main() -------------------------------------------------------------------------------- /generate-wordcloud/README.md: -------------------------------------------------------------------------------- 1 | ## Container to Create a Word Cloud from a List of Words 2 | 3 | This is not an officially supported Google product, though support will be provided on a best-effort basis. 4 | 5 | Copyright 2018 Google LLC 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | 20 | ### Introduction 21 | 22 | This is a container that receives pubsub messages with `globalId` identifiers and then 23 | kicks off a workflow to create a word cloud. The service pulls a word list from Google 24 | Cloud Storage, removes stop words and generates a word cloud. The word cloud is saved to 25 | Google Cloud Storage as a PNG image and the public URL for the image is written to the 26 | `meetingRegistry` database. 27 | 28 | ### Assumptions 29 | 30 | * You have created a Pubsub topic called `wordcloudQueue` 31 | * You have created a Pubsub subscription called `wordcloud-creation-subscription` 32 | * You have deployed a utility service to handle database lookups and updates 33 | 34 | ### Note 35 | 36 | The file `stopwords-20180109-133115.json` contains a sample batch of stop words. You will 37 | want to customize this file based on the frequency of common words and phrases used in 38 | the meetings you are transcribing and analyzing. 39 | 40 | ### Build and Run this container 41 | 42 | Build 43 | `docker build -t generate-wordcloud:latest .` 44 | 45 | Run 46 | `docker run -it generate-wordcloud` -------------------------------------------------------------------------------- /app-engine-front-end/app.yaml: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | service: __Service_Name__ 21 | runtime: python27 22 | api_version: 1 23 | threadsafe: false 24 | 25 | handlers: 26 | - url: /css 27 | static_dir: css 28 | 29 | - url: /img 30 | static_dir: img 31 | 32 | - url: /js 33 | static_dir: js 34 | 35 | - url: /fonts 36 | static_dir: fonts 37 | 38 | - url: /scss 39 | static_dir: scss 40 | 41 | - url: /favicon\.ico 42 | static_files: img/favicon.ico 43 | upload: img/favicon\.ico 44 | 45 | - url: /ep-meetingArchive 46 | script: ep-meetingArchive.py 47 | 48 | - url: /ep-jumbotronContent 49 | script: ep-jumbotronContent.py 50 | 51 | - url: /ep-searchVideo 52 | script: ep-searchVideo.py 53 | 54 | - url: /ep-searchArchive 55 | script: ep-searchArchive.py 56 | 57 | - url: /ep-meetingDetails 58 | script: ep-meetingDetails.py 59 | 60 | - url: /ep-upcomingMeetings 61 | script: ep-upcomingMeetings.py 62 | 63 | - url: .* 64 | static_files: index.html 65 | secure: always 66 | upload: index.html 67 | http_headers: 68 | Cache-Control: no-cache, no-store, must-revalidate 69 | 70 | libraries: 71 | - name: ujson 72 | version: "1.35" 73 | - name: MySQLdb 74 | version: "latest" -------------------------------------------------------------------------------- /app-engine-utility-service/idWordcloud.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import cgi 24 | import utilities 25 | import ujson as json 26 | from datetime import datetime 27 | 28 | 29 | def assignUrl(globalId, wcUrl): 30 | sqlCmd = """update meetingRegistry 31 | set wordCloud = %s 32 | where globalId = %s""" 33 | sqlData = (wcUrl, globalId) 34 | resultList = utilities.dbExecution(sqlCmd, sqlData) 35 | 36 | return resultList 37 | 38 | 39 | def cleanInput(inputVal, inputLen): 40 | inputVal = str(inputVal) 41 | if len(inputVal) > inputLen: 42 | inputVal = inputVal[:inputLen] 43 | inputVal = inputVal.replace("\\", "") 44 | inputVal = inputVal.replace(";", "") 45 | 46 | return inputVal 47 | 48 | 49 | def main(): 50 | errorFound = False 51 | 52 | passedArgs = cgi.FieldStorage() 53 | 54 | try: 55 | globalId = int(passedArgs["gId"].value) 56 | globalId = cleanInput(globalId, 5) 57 | wcUrl = passedArgs["wcUrl"].value 58 | wcUrl = cleanInput(wcUrl, 150) 59 | resultObj = assignUrl(globalId, wcUrl) 60 | except: 61 | globalId = None 62 | errorFound = True 63 | 64 | print "Content-Type: application/json\n" 65 | if errorFound is False: 66 | print [resultObj[0], resultObj[1]] 67 | else: 68 | print "Nada" 69 | 70 | 71 | if __name__ == "__main__": 72 | main() 73 | -------------------------------------------------------------------------------- /app-engine-utility-service/idTranscript.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import cgi 24 | import utilities 25 | import ujson as json 26 | from datetime import datetime 27 | 28 | 29 | def assignUrl(globalId, transcriptUrl): 30 | sqlCmd = """update meetingRegistry 31 | set publishedTranscript = %s 32 | where globalId = %s""" 33 | sqlData = (transcriptUrl, globalId) 34 | resultList = utilities.dbExecution(sqlCmd, sqlData) 35 | 36 | return resultList 37 | 38 | 39 | def cleanInput(inputVal, inputLen): 40 | inputVal = str(inputVal) 41 | if len(inputVal) > inputLen: 42 | inputVal = inputVal[:inputLen] 43 | inputVal = inputVal.replace("\\", "") 44 | inputVal = inputVal.replace(";", "") 45 | 46 | return inputVal 47 | 48 | 49 | def main(): 50 | errorFound = False 51 | 52 | passedArgs = cgi.FieldStorage() 53 | 54 | try: 55 | globalId = int(passedArgs["gId"].value) 56 | globalId = cleanInput(globalId, 5) 57 | transcriptUrl = passedArgs["transcriptUrl"].value 58 | transcriptUrl = cleanInput(transcriptUrl, 250) 59 | resultObj = assignUrl(globalId, transcriptUrl) 60 | except: 61 | globalId = None 62 | errorFound = True 63 | 64 | print "Content-Type: application/json\n" 65 | if errorFound is False: 66 | print resultObj[0] 67 | print resultObj[1] 68 | else: 69 | print "Nada" 70 | 71 | 72 | if __name__ == "__main__": 73 | main() 74 | -------------------------------------------------------------------------------- /app-engine-front-end/utilities.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import MySQLdb 24 | 25 | MySQLdb.escape_string("'") 26 | 27 | 28 | def dbExecution(sqlCmd, sqlData): 29 | 30 | sqlInstance = "__Cloud_SQL_Instance_Connection_Name__" 31 | 32 | if os.getenv("SERVER_SOFTWARE", "").startswith("Google App Engine/"): 33 | connection = MySQLdb.connect( unix_socket = "/cloudsql/" + sqlInstance, 34 | user = "__Cloud_SQL_Username__", 35 | passwd = "__Cloud_SQL_User_Password__", 36 | db = "prodDb") 37 | else: 38 | connection = MySQLdb.connect( host = "__Cloud_SQL_Public_IP_Address__", 39 | user = "__Cloud_SQL_Username__", 40 | passwd = "__Cloud_SQL_User_Password__", 41 | db = "prodDb") 42 | cursor = connection.cursor() 43 | 44 | if len(sqlData) > 0: 45 | try: 46 | cmdExecution = cursor.execute(sqlCmd, *[sqlData]) 47 | except: 48 | cmdExecution = cursor.execute(sqlCmd, [sqlData]) 49 | else: 50 | cmdExecution = cursor.execute(sqlCmd) 51 | 52 | connection.commit() 53 | numResults = cursor.rowcount 54 | resultRows = cursor.fetchall() 55 | cursor.close() 56 | connection.close() 57 | resultList = [cmdExecution, numResults, resultRows] 58 | 59 | return resultList 60 | 61 | 62 | def main(): 63 | print "Content-type: text/plain; charset=UTF-8\n\n" 64 | print "Nothing To See Here" 65 | 66 | 67 | if __name__ == "__main__": 68 | main() 69 | -------------------------------------------------------------------------------- /publish-pdf-transcript/Dockerfile: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | FROM gcr.io/google_appengine/python 21 | 22 | RUN apt-get --allow-unauthenticated -y update 23 | 24 | RUN apt-get install curl 25 | 26 | ADD ./fonts/ ./fonts/ 27 | 28 | RUN apt-get -y install libxext6 29 | RUN apt-get -y install libfontconfig1 libxrender1 30 | 31 | RUN cd ~ 32 | RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.3/wkhtmltox-0.12.3_linux-generic-amd64.tar.xz 33 | RUN tar vxf wkhtmltox-0.12.3_linux-generic-amd64.tar.xz 34 | RUN cp wkhtmltox/bin/wk* /usr/local/bin/ 35 | RUN cd - 36 | 37 | RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz 38 | RUN mkdir -p /usr/local/gcloud 39 | RUN tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz 40 | RUN /usr/local/gcloud/google-cloud-sdk/install.sh 41 | ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin 42 | 43 | ADD __Credential_JSON_File_Name__ /app/__Credential_JSON_File_Name__ 44 | RUN gcloud auth activate-service-account --key-file=__Credential_JSON_File_Name__ 45 | ENV GOOGLE_APPLICATION_CREDENTIALS /app/__Credential_JSON_File_Name__ 46 | 47 | RUN virtualenv /env 48 | ENV VIRTUAL_ENV /env 49 | ENV PATH /env/bin:$PATH 50 | ADD requirements.txt /app/requirements.txt 51 | RUN pip install -r /app/requirements.txt 52 | ADD procfile /app/procfile 53 | ADD . /app 54 | CMD honcho start -f /app/procfile worker monitor 55 | ENV https_proxy "" 56 | ENV http_proxy "" -------------------------------------------------------------------------------- /transcode-video-to-audio/Dockerfile: -------------------------------------------------------------------------------- 1 | # This is not an officially supported Google product, though support 2 | # will be provided on a best-effort basis. 3 | 4 | # Copyright 2018 Google LLC 5 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); you 7 | # may not use this file except in compliance with the License. 8 | 9 | # You may obtain a copy of the License at: 10 | 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | 20 | FROM gcr.io/google_appengine/python 21 | FROM gcr.io/google_appengine/python 22 | RUN echo "deb http://www.deb-multimedia.org jessie main non-free" >> /etc/apt/sources.list 23 | RUN echo "deb-src http://www.deb-multimedia.org jessie main non-free" >> /etc/apt/sources.list 24 | RUN apt-get --allow-unauthenticated -y update 25 | RUN apt-get --allow-unauthenticated -y install deb-multimedia-keyring 26 | RUN apt-get --allow-unauthenticated -y update 27 | RUN apt-get --allow-unauthenticated -y install libav-tools ffmpeg nmap vim 28 | 29 | RUN apt-get install curl 30 | 31 | RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz 32 | RUN mkdir -p /usr/local/gcloud 33 | RUN tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz 34 | RUN /usr/local/gcloud/google-cloud-sdk/install.sh 35 | ENV PATH $PATH:/usr/local/gcloud/google-cloud-sdk/bin 36 | 37 | ADD __Credential_JSON_File_Name__ /app/__Credential_JSON_File_Name__ 38 | RUN gcloud auth activate-service-account --key-file=__Credential_JSON_File_Name__ 39 | ENV GOOGLE_APPLICATION_CREDENTIALS /app/__Credential_JSON_File_Name__ 40 | 41 | RUN virtualenv /env 42 | ENV VIRTUAL_ENV /env 43 | ENV PATH /env/bin:$PATH 44 | ADD requirements.txt /app/requirements.txt 45 | RUN pip install -r /app/requirements.txt 46 | ADD procfile /app/procfile 47 | ADD . /app 48 | CMD honcho start -f /app/procfile worker monitor 49 | ENV https_proxy "" 50 | ENV http_proxy "" 51 | -------------------------------------------------------------------------------- /app-engine-utility-service/idTranscode.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import ujson 24 | import webapp2 25 | import utilities 26 | 27 | 28 | def cleanInput(inputVal, inputLen): 29 | inputVal = str(inputVal) 30 | if len(inputVal) > inputLen: 31 | inputVal = inputVal[:inputLen] 32 | inputVal = inputVal.replace("/", "") 33 | inputVal = inputVal.replace("\\", "") 34 | inputVal = inputVal.replace(";", "") 35 | 36 | return inputVal 37 | 38 | 39 | class main(webapp2.RequestHandler): 40 | def get(self): 41 | self.response.headers["Content-Type"] = "application/json" 42 | self.response.headers.add_header( 43 | "Cache-Control", 44 | "no-cache, no-store, must-revalidate, max-age=0" 45 | ) 46 | self.response.headers.add_header( 47 | "Expires", 48 | "0" 49 | ) 50 | 51 | try: 52 | globalId = self.request.get("gId") 53 | globalId = cleanInput(globalId, 5) 54 | prodTranscode = self.request.get("transcode") 55 | prodTranscode = cleanInput(prodTranscode, 22) 56 | if globalId != "": 57 | if prodTranscode != "": 58 | sqlCmd = "update meetingRegistry set prodTranscode = %s where globalId = %s" 59 | sqlData = (prodTranscode, globalId) 60 | resultList = utilities.dbExecution(sqlCmd, sqlData) 61 | outputStr = str(resultList) 62 | except: 63 | outputStr = None 64 | 65 | resultObj = {} 66 | resultObj["response"] = outputStr 67 | 68 | self.response.out.write(ujson.dumps(resultObj)) 69 | 70 | 71 | app = webapp2.WSGIApplication([ 72 | ("/idTranscode", main)], debug = True 73 | ) 74 | -------------------------------------------------------------------------------- /app-engine-front-end/ep-relatedFiles.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import cgi 24 | import utilities 25 | import ujson as json 26 | from datetime import datetime 27 | 28 | 29 | def lookupFiles(urlIdentifier): 30 | sqlCmd = """select 31 | globalId from meetingRegistry 32 | where urlIdentifier = %s""" 33 | sqlData = (urlIdentifier) 34 | resultList = utilities.dbExecution(sqlCmd, sqlData) 35 | 36 | globalId = resultList[2][0][0] 37 | globalId = str(globalId) 38 | 39 | sqlCmd = """select 40 | fileId, 41 | fileName, 42 | mimeType, 43 | webViewLink, 44 | thumbnailLink, 45 | pageIndex from relatedFiles 46 | where globalId = %s 47 | order by fileName ASC""" 48 | sqlData = (globalId) 49 | resultList = utilities.dbExecution(sqlCmd, sqlData) 50 | 51 | return resultList[2] 52 | 53 | 54 | def main(): 55 | errorFound = False 56 | 57 | passedArgs = cgi.FieldStorage() 58 | 59 | try: 60 | urlIdentifier = passedArgs["urlIdentifier"].value 61 | resultObj = lookupFiles(urlIdentifier) 62 | except: 63 | meetingId = None 64 | errorFound = True 65 | resultObj = None 66 | 67 | try: 68 | outputObj = {} 69 | for eachResult in resultObj: 70 | pageIndex = eachResult[5].zfill(2) 71 | fileObj = {} 72 | fileObj["fileId"] = eachResult[0] 73 | fileObj["fileName"] = eachResult[1] 74 | fileObj["fileType"] = eachResult[2] 75 | fileObj["webViewLink"] = eachResult[3] 76 | fileObj["thumbnailLink"] = eachResult[4] 77 | outputObj[pageIndex] = fileObj 78 | except: 79 | errorFound = True 80 | 81 | if errorFound is True: 82 | outputObj = {} 83 | outputObj["error"] = "Error" 84 | 85 | print "Content-Type: application/json\n" 86 | print json.dumps(outputObj) 87 | 88 | 89 | if __name__ == "__main__": 90 | main() -------------------------------------------------------------------------------- /app-engine-front-end/ep-searchVideo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import cgi 24 | import time 25 | import ujson 26 | import urllib 27 | import logging 28 | from google.appengine.api import search 29 | from google.appengine.api import urlfetch 30 | 31 | in_video_search_service_url = "__In_Video_Search_Service_URL__" 32 | 33 | 34 | def main(): 35 | passedArgs = cgi.FieldStorage() 36 | queryString = passedArgs["q"].value 37 | urlIdentifier = passedArgs["urlId"].value 38 | orgIdentifier = passedArgs["orgId"].value 39 | 40 | logging.info("search") 41 | 42 | payloadObj = { 43 | "urlId": urlIdentifier, 44 | "orgId": orgIdentifier, 45 | "q": queryString 46 | } 47 | 48 | urlParams = urllib.urlencode(payloadObj, doseq=True) 49 | reqUrl = in_video_search_service_url + "/?%s" % urlParams 50 | responseObj = urlfetch.fetch(reqUrl) 51 | responseStr = responseObj.content 52 | 53 | searchObj = ujson.loads(responseStr) 54 | outputObj = {} 55 | for eachEntry in searchObj["hits"]["hits"]: 56 | if "highlight" in eachEntry: 57 | resultObj = {} 58 | 59 | tsVal = eachEntry["_source"]["mediaTimestamp"] 60 | resultObj["length"] = int(eachEntry["_source"]["segmentLength"]) 61 | resultObj["timestamp"] = time.strftime("%H:%M:%S", time.gmtime(eachEntry["_source"]["mediaTimestamp"])) 62 | snippetStr = eachEntry["highlight"]["transcriptStr"][0] 63 | snippetStr = snippetStr.replace(".", "") 64 | snippetStr = snippetStr.lower() 65 | snippetStr = snippetStr.lstrip() 66 | resultObj["result"] = "... " + snippetStr +"..." 67 | outputObj[str(int(tsVal)).zfill(6)] = resultObj 68 | 69 | print "Expires: 0" 70 | print "Cache-Control: no-cache, no-store, must-revalidate, max-age=0" 71 | print "Content-Type: application/json\n" 72 | print ujson.dumps(outputObj) 73 | 74 | 75 | if __name__ == "__main__": 76 | main() 77 | -------------------------------------------------------------------------------- /in-video-search/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import json 23 | import logging 24 | from flask import Flask 25 | from flask import request 26 | from flask import Response 27 | from elasticsearch import Elasticsearch 28 | 29 | app = Flask(__name__) 30 | 31 | 32 | @app.route("/") 33 | def main(): 34 | q = request.args.get("q") 35 | urlId = request.args.get("urlId") 36 | orgId = request.args.get("orgId") 37 | 38 | searchClient = Elasticsearch( 39 | ["__Elastic_Search_Instance_URL__"], 40 | http_auth = ( 41 | "__Elastic_Search_Instance_Username__", 42 | "__Elastic_Search_Instance_Password__" 43 | ) 44 | ) 45 | 46 | queryBody = { 47 | "query": { 48 | "bool": { 49 | "should": { 50 | "match": { 51 | "transcriptStr": { 52 | "query": q, 53 | "operator": "and" 54 | } 55 | }, 56 | }, 57 | "must": [ 58 | { "match": { "urlIdentifier": urlId } } 59 | ] 60 | } 61 | }, 62 | "_source": ["mediaTimestamp", "meetingDate", "segmentLength"], 63 | "highlight": { 64 | "fields" : { 65 | "transcriptStr": { 66 | "type": "plain", 67 | "fragment_size": 38, 68 | "number_of_fragments": 5 69 | } 70 | } 71 | }, 72 | "size": 100, 73 | "min_score": 1.1 74 | } 75 | 76 | try: 77 | searchObj = searchClient.search( 78 | index = orgId, 79 | body = queryBody 80 | ) 81 | outputStr = json.dumps(searchObj) 82 | except: 83 | outputStr = json.dumps( { "None": "None" } ) 84 | 85 | return Response(outputStr, mimetype="application/json") 86 | 87 | 88 | @app.errorhandler(500) 89 | def server_error(e): 90 | logging.exception("An error occurred during a request.") 91 | return """ 92 | An internal error occurred: 
{}
93 | See logs for full stacktrace. 94 | """.format(e), 500 95 | 96 | 97 | if __name__ == "__main__": 98 | app.run(host = "127.0.0.1", port = 8080, debug = True) 99 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Government Meetings Made Searchable 2 | 3 | This is not an officially supported Google product, though support will be provided on a best-effort basis. 4 | 5 | Copyright 2018 Google LLC 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | https://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | 20 | ### Introduction 21 | 22 | This is project to make the contents of public meetings searchable and discoverable. This 23 | repo is a series of utilities and containers you can use to transcode, transcribe, and 24 | publish content from videos of public meetings and hearings. 25 | 26 | 27 | #### app-engine-front-end 28 | This is a Google App Engine app that provides a front end to the project. 29 | 30 | 31 | #### app-engine-utility-service 32 | This is a Google App Engine app that facilitates interaction with the Google Cloud SQL 33 | database, and provides common utilities for back end services. 34 | 35 | 36 | #### in-video-search 37 | This is a Google App Engine Flex app that provides a wrapper and proxy for requests 38 | to an Elastic Search instance. This service handles searches for materials in a 39 | particular meeting. 40 | 41 | 42 | #### archive-video-search 43 | This is a Google App Engine Flex app that provides a wrapper and proxy for requests 44 | to and responses from an Elastic Search instance. This service handles searches for 45 | materials across all meetings in an index. 46 | 47 | 48 | #### transcode-video-to-audio 49 | This is a container that transcodes a video file to an audio file that is compatible with 50 | the Google Speech API. 51 | 52 | 53 | #### create-word-list 54 | This is a container that creates a list of words from the Google Speech API responses that 55 | will be used for creating a word cloud. 56 | 57 | 58 | #### generate-wordcloud 59 | This is a container that creates a word cloud image in PNG format from a list of words 60 | stored on Google Cloud Storage. 61 | 62 | 63 | #### index-meeting 64 | This is a container that parses Google Speech API responses and writes the contents to an 65 | Elastic Search index in a batch process. 66 | 67 | #### publish-pdf-transcript 68 | 69 | This is a container that parses Google Speech API responses and produces a human readable 70 | PDF transcript. 71 | -------------------------------------------------------------------------------- /archive-video-search/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import json 23 | import logging 24 | from flask import Flask 25 | from flask import request 26 | from flask import Response 27 | from elasticsearch import Elasticsearch 28 | 29 | app = Flask(__name__) 30 | 31 | 32 | @app.route("/") 33 | def main(): 34 | q = request.args.get("q") 35 | orgId = request.args.get("orgId") 36 | 37 | searchClient = Elasticsearch( 38 | ["__Elastic_Search_Instance_URL__"], 39 | http_auth = ( 40 | "__Elastic_Search_Instance_Username__", 41 | "__Elastic_Search_Instance_Password__" 42 | ) 43 | ) 44 | queryBody = { 45 | "query": { 46 | "bool": { 47 | "should": { 48 | "match": { 49 | "transcriptStr": { 50 | "query": q, 51 | "operator": "and" 52 | } 53 | } 54 | } 55 | } 56 | }, 57 | "sort": [ 58 | { "_score": { "order": "desc" } }, 59 | ], 60 | "size": 0, 61 | "aggs": { 62 | "group_by_meeting": { 63 | "terms": { 64 | "field": "globalId", 65 | "size": 80, 66 | "min_doc_count": 1 67 | }, 68 | "aggs": { 69 | "meeting_details": { 70 | "top_hits": { 71 | "size": 1, 72 | "_source": { 73 | "includes": ["meetingDate", "meetingDesc", "urlIdentifier"] 74 | } 75 | } 76 | } 77 | } 78 | } 79 | } 80 | } 81 | 82 | try: 83 | searchObj = searchClient.search( 84 | index = orgId, 85 | body = queryBody 86 | ) 87 | outputStr = json.dumps(searchObj) 88 | except: 89 | outputStr = json.dumps( { "None": "None" } ) 90 | 91 | return Response(outputStr, mimetype="application/json") 92 | 93 | 94 | @app.errorhandler(500) 95 | def server_error(e): 96 | logging.exception("An error occurred during a request.") 97 | return """ 98 | An internal error occurred: 
{}
99 | See logs for full stacktrace. 100 | """.format(e), 500 101 | 102 | 103 | if __name__ == "__main__": 104 | app.run(host="127.0.0.1", port=8080, debug=True) 105 | -------------------------------------------------------------------------------- /app-engine-front-end/ep-meetingDetails.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import cgi 24 | import logging 25 | import utilities 26 | import ujson as json 27 | from datetime import datetime 28 | 29 | 30 | def lookupMeeting(urlIdentifier): 31 | globalId = str(urlIdentifier) 32 | sqlCmd = """select 33 | meetingDesc, 34 | meetingDate, 35 | youtubeId, 36 | wordCloud, 37 | publishedVideo, 38 | publishedTranscript, 39 | orgIdentifier, 40 | publishedAgenda, 41 | urlIdentifier, 42 | hasSegments from meetingRegistry 43 | where urlIdentifier = %s 44 | and youtubeId is not NULL 45 | limit 1""" 46 | sqlData = (urlIdentifier) 47 | resultList = utilities.dbExecution(sqlCmd, sqlData) 48 | 49 | return resultList[2][0] 50 | 51 | 52 | def formatDate(meetingDate): 53 | datetimeObj = datetime.strptime(meetingDate, "%Y%m%d") 54 | formattedDate = datetimeObj.strftime("%B %d, %Y") 55 | weekDay = datetimeObj.strftime("%A") 56 | 57 | return formattedDate, weekDay 58 | 59 | 60 | def main(): 61 | errorFound = False 62 | 63 | passedArgs = cgi.FieldStorage() 64 | 65 | try: 66 | urlIdentifier = passedArgs["urlIdentifier"].value 67 | resultObj = lookupMeeting(urlIdentifier) 68 | except: 69 | meetingId = None 70 | errorFound = True 71 | 72 | try: 73 | hasSegments = resultObj[9] 74 | if hasSegments is None: 75 | hasSegments = 0 76 | 77 | formattedDate, weekDay = formatDate(resultObj[1]) 78 | outputObj = {} 79 | outputObj["desc"] = resultObj[0] 80 | outputObj["dow"] = weekDay 81 | outputObj["date"] = formattedDate 82 | outputObj["youtubeId"] = resultObj[2] 83 | outputObj["urlIdentifier"] = urlIdentifier 84 | outputObj["wordCloud"] = resultObj[3] 85 | outputObj["videoUrl"] = resultObj[4] 86 | outputObj["transcriptUrl"] = resultObj[5] 87 | outputObj["agendaUrl"] = resultObj[7] 88 | outputObj["hasSegments"] = hasSegments 89 | orgIdentifier = resultObj[6] 90 | logging.info("loadMeeting") 91 | except: 92 | errorFound = True 93 | 94 | if errorFound is True: 95 | outputObj = {} 96 | outputObj["error"] = "Error" 97 | 98 | print "Content-Type: application/json\n" 99 | print json.dumps(outputObj) 100 | 101 | 102 | if __name__ == "__main__": 103 | main() -------------------------------------------------------------------------------- /app-engine-front-end/ep-meetingArchive.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import cgi 24 | import utilities 25 | import ujson as json 26 | from datetime import datetime 27 | 28 | 29 | def getMeetings(qryLimit, qryOffset, orgIdentifier): 30 | sqlCmd = """select 31 | meetingDesc, 32 | meetingDate, 33 | urlIdentifier, 34 | globalId from meetingRegistry 35 | where orgIdentifier = %s 36 | and youtubeId is not NULL 37 | order by meetingDate DESC 38 | limit %s 39 | offset %s""" 40 | sqlData = (orgIdentifier, int(qryLimit), int(qryOffset)) 41 | resultList = utilities.dbExecution(sqlCmd, sqlData) 42 | 43 | lastDate = None 44 | dateCnt = 0 45 | 46 | meetingDict = {} 47 | for eachEntry in resultList[2]: 48 | meetingDate = eachEntry[1] 49 | 50 | dateIncr = meetingDate 51 | if lastDate == meetingDate: 52 | dateCnt = dateCnt + 1 53 | else: 54 | dateCnt = 0 55 | dateIncr = str(meetingDate) + str(dateCnt).zfill(3) 56 | 57 | formattedDate, weekDay = formatDate(meetingDate) 58 | 59 | meetingObj = {} 60 | meetingObj["desc"] = eachEntry[0] 61 | meetingObj["date"] = formattedDate 62 | meetingObj["dow"] = weekDay 63 | meetingObj["meetingId"] = eachEntry[3] 64 | meetingObj["urlIdentifier"] = eachEntry[2] 65 | 66 | meetingDict[dateIncr] = meetingObj 67 | lastDate = meetingDate 68 | 69 | return meetingDict 70 | 71 | 72 | def formatDate(meetingDate): 73 | datetimeObj = datetime.strptime(meetingDate, "%Y%m%d") 74 | formattedDate = datetimeObj.strftime("%B %d, %Y") 75 | weekDay = datetimeObj.strftime("%A") 76 | 77 | return formattedDate, weekDay 78 | 79 | 80 | def main(): 81 | passedArgs = cgi.FieldStorage() 82 | 83 | try: 84 | lastMeeting = int(passedArgs["lastMeeting"].value) 85 | orgIdentifier = passedArgs["orgId"].value 86 | except: 87 | lastMeeting = 0 88 | orgIdentifier = None 89 | 90 | if lastMeeting == 0: 91 | meetingTotal = 3 92 | else: 93 | meetingTotal = 6 94 | 95 | jsonObj = getMeetings(meetingTotal, lastMeeting, orgIdentifier) 96 | 97 | keyList = jsonObj.keys() 98 | keyList.sort(reverse=True) 99 | targetMeeting = lastMeeting + meetingTotal 100 | 101 | outputObj = {} 102 | outputObj["meetingList"] = jsonObj 103 | 104 | print "Content-Type: application/json\n" 105 | print json.dumps(outputObj) 106 | 107 | 108 | if __name__ == "__main__": 109 | main() 110 | -------------------------------------------------------------------------------- /app-engine-utility-service/meetingDetails.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import ujson 24 | import webapp2 25 | import utilities 26 | 27 | 28 | class main(webapp2.RequestHandler): 29 | def get(self): 30 | self.response.headers["Content-Type"] = "application/json" 31 | self.response.headers.add_header( 32 | "Cache-Control", 33 | "no-cache, no-store, must-revalidate, max-age=0" 34 | ) 35 | self.response.headers.add_header( 36 | "Expires", 37 | "0" 38 | ) 39 | 40 | try: 41 | globalId = self.request.get("gId") 42 | sqlData = (globalId) 43 | sqlCmd = "select videoName, beenTranscribed, beenTranscoded, videoDownloaded, videoLink, orgIdentifier, prodTranscript, meetingDate, meetingDesc, beenIndexed, youtubeId, meetingId, prodTranscode, urlIdentifier from meetingRegistry where globalId = %s" 44 | resultList = utilities.dbExecution(sqlCmd, sqlData) 45 | videoName = resultList[2][0][0] 46 | beenTranscribed = resultList[2][0][1] 47 | beenTranscoded = resultList[2][0][2] 48 | videoDownloaded = resultList[2][0][3] 49 | videoLink = resultList[2][0][4] 50 | orgIdentifier = resultList[2][0][5] 51 | prodTranscript = resultList[2][0][6] 52 | meetingDate = resultList[2][0][7] 53 | meetingDesc = resultList[2][0][8] 54 | beenIndexed = resultList[2][0][9] 55 | youtubeId = resultList[2][0][10] 56 | meetingId = resultList[2][0][11] 57 | prodTranscode = resultList[2][0][12] 58 | urlIdentifier = resultList[2][0][13] 59 | except: 60 | videoName = None 61 | beenTranscribed = None 62 | beenTranscoded = None 63 | videoDownloaded = None 64 | videoLink = None 65 | orgIdentifier = None 66 | prodTranscript = None 67 | meetingDate = None 68 | meetingDesc = None 69 | beenIndexed = None 70 | youtubeId = None 71 | meetingId = None 72 | prodTranscode = None 73 | urlIdentifier= None 74 | 75 | resultObj = {} 76 | resultObj["videoName"] = videoName 77 | resultObj["beenTranscribed"] = beenTranscribed 78 | resultObj["beenTranscoded"] = beenTranscoded 79 | resultObj["videoDownloaded"] = videoDownloaded 80 | resultObj["videoLink"] = videoLink 81 | resultObj["orgIdentifier"] = orgIdentifier 82 | resultObj["prodTranscript"] = prodTranscript 83 | resultObj["meetingDate"] = meetingDate 84 | resultObj["meetingDesc"] = meetingDesc 85 | resultObj["beenIndexed"] = beenIndexed 86 | resultObj["youtubeId"] = youtubeId 87 | resultObj["meetingId"] = meetingId 88 | resultObj["prodTranscode"] = prodTranscode 89 | resultObj["urlIdentifier"] = urlIdentifier 90 | 91 | self.response.out.write(ujson.dumps(resultObj)) 92 | 93 | 94 | app = webapp2.WSGIApplication([ 95 | ("/meetingDetails", main)], debug = True 96 | ) 97 | -------------------------------------------------------------------------------- /app-engine-utility-service/runRecognize.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import sys 23 | import ujson 24 | import urllib 25 | import traceback 26 | import utilities 27 | from googleapiclient.discovery import build 28 | from google.appengine.ext import vendor 29 | vendor.add("lib") 30 | 31 | import httplib2 32 | from oauth2client.service_account import ServiceAccountCredentials 33 | 34 | 35 | def runCycle(gcsLoc, jobId, globalId): 36 | credentialsJson = "__Credential_JSON_File_Name__" 37 | 38 | scopesList = ["https://www.googleapis.com/auth/cloud-platform"] 39 | credentialsObj = ServiceAccountCredentials.from_json_keyfile_name( 40 | credentialsJson, 41 | scopes = scopesList 42 | ) 43 | 44 | payloadObj = { 45 | "audio": { 46 | "uri": gcsLoc 47 | }, 48 | "config": { 49 | "languageCode": "en-US", 50 | "encoding": "FLAC", 51 | "sampleRateHertz": 16000, 52 | "enableWordTimeOffsets": True, 53 | "enableAutomaticPunctuation": True, 54 | "useEnhanced": True, 55 | "model": "video", 56 | "metadata": { 57 | "interaction_type": "DISCUSSION", 58 | "recording_device_type": "OTHER_INDOOR_DEVICE", 59 | "originalMediaType": "VIDEO" 60 | }, 61 | "speechContexts": { 62 | "phrases": [ 63 | "Louisville", "Weldona", "signage", "PROSTAC" 64 | ] 65 | } 66 | } 67 | } 68 | 69 | try: 70 | httpObj = credentialsObj.authorize(httplib2.Http()) 71 | serviceObj = build( 72 | serviceName = "speech", 73 | version = "v1p1beta1", 74 | http = httpObj, 75 | developerKey = "__Google_Speech_API_Key__" 76 | ) 77 | responseObj = serviceObj.speech().longrunningrecognize(body = payloadObj).execute() 78 | 79 | print "job " + str(jobId) + " for global id " + str(globalId) 80 | 81 | apiName = responseObj["name"] 82 | print "request name " + str(apiName) 83 | 84 | sqlCmd = """update speechJobs set apiName = %s, beenProcessed = %s where jobId = %s""" 85 | sqlData = [apiName, 1, jobId] 86 | queryResp = utilities.dbExecution(sqlCmd, sqlData) 87 | except Exception as e: 88 | sqlCmd = """update speechJobs set jobStatus = %s, beenProcessed = %s where jobId = %s""" 89 | sqlData = ["longrunning api call failed", 1, jobId] 90 | queryResp = utilities.dbExecution(sqlCmd, sqlData) 91 | print "longrunning api call failed" 92 | 93 | 94 | def main(): 95 | sqlCmd = """select globalId, jobId, gcsLoc from speechJobs where beenProcessed = %s order by jobId limit 1""" 96 | sqlData = [0] 97 | queryResp = utilities.dbExecution(sqlCmd, sqlData) 98 | 99 | print "Content-type: text/plain; charset=UTF-8\n\n" 100 | 101 | if queryResp[2]: 102 | gcsLoc = queryResp[2][0][2].replace("'","") 103 | jobId = queryResp[2][0][1] 104 | globalId = queryResp[2][0][0] 105 | print globalId 106 | print gcsLoc 107 | print jobId 108 | runCycle(gcsLoc, jobId, globalId) 109 | else: 110 | print "No jobs to run." 111 | 112 | 113 | if __name__ == "__main__": 114 | main() -------------------------------------------------------------------------------- /app-engine-utility-service/speechJobs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import time 23 | import base64 24 | import logging 25 | import calendar 26 | import utilities 27 | 28 | from google.appengine.ext import vendor 29 | vendor.add("lib") 30 | from gcloud import storage 31 | 32 | bucketName = "__GCS_Storage_Bucket_Name__" 33 | 34 | 35 | def lookupMeeting(globalId): 36 | sqlCmd = """select prodTranscode, orgIdentifier from meetingRegistry where globalId = %s""" 37 | sqlData = [globalId] 38 | resultObj = utilities.dbExecution(sqlCmd, sqlData) 39 | 40 | return resultObj[2][0][0], resultObj[2][0][1] 41 | 42 | 43 | def runCycle(globalId, orgIdentifier, prodTranscode, batchId): 44 | clientObj = storage.Client() 45 | bucketObj = clientObj.get_bucket(bucketName) 46 | listObj = bucketObj.list_blobs(prefix="accounts/" + orgIdentifier + "/enrichments/" + str(globalId) + "/transcodes/" + prodTranscode) 47 | fileCnt = 0 48 | for eachEntry in listObj: 49 | 50 | if ".flac" in eachEntry.name: 51 | gcsLoc = "gs://" + eachEntry.bucket.name + "/" + eachEntry.name 52 | sqlCmd = """insert into speechJobs (globalId, orgIdentifier, gcsLoc, beenProcessed, batchId) values (%s, %s, %s, %s, %s)""" 53 | sqlData = [globalId, orgIdentifier, gcsLoc, 0, batchId] 54 | utilities.dbExecution(sqlCmd, sqlData) 55 | fileCnt += 1 56 | 57 | return fileCnt 58 | 59 | 60 | def markTranscript(globalId, prodTranscript, batchId): 61 | prodTranscript = str(batchId) + "-" + prodTranscript 62 | sqlCmd = """update meetingRegistry set beenTranscribed = %s, prodTranscript = %s where globalId = %s""" 63 | sqlData = [1, prodTranscript, globalId] 64 | resultObj = utilities.dbExecution(sqlCmd, sqlData) 65 | 66 | return resultObj 67 | 68 | 69 | def main(): 70 | projectName = "__GCP_Project_ID__" 71 | subName = "speech-job-subscription" 72 | 73 | print "Content-type: text/plain; charset=UTF-8\n\n" 74 | 75 | respObj = utilities.pullMsg(projectName, subName, True) 76 | if respObj: 77 | receivedMessage = respObj.get("receivedMessages")[0] 78 | msgObj = receivedMessage.get("message") 79 | print ".. pubsub message id: " + str(msgObj.get("messageId")) 80 | msgType = base64.b64decode(str(msgObj.get("data"))) 81 | print ".. message type: " + msgType 82 | globalId = msgObj.get("attributes")["globalId"] 83 | 84 | ackId = receivedMessage.get("ackId") 85 | utilities.ackMsg(projectName, subName, ackId) 86 | 87 | if globalId: 88 | epochTime = calendar.timegm(time.gmtime()) 89 | batchId = str(epochTime) 90 | print "... creating Speech API jobs for meeting: " + str(globalId) 91 | 92 | prodTranscript, orgIdentifier = lookupMeeting(globalId) 93 | logging.info(prodTranscript) 94 | print ".... production trascript is: " + prodTranscript 95 | 96 | jobCnt = runCycle(globalId, orgIdentifier, prodTranscript, batchId) 97 | print ".... created " + str(jobCnt) + " Speech API jobs" 98 | 99 | markTranscript(globalId, prodTranscript, batchId) 100 | print ".... updated meetingRegistry table" 101 | else: 102 | print "No Messages to Handle" 103 | 104 | 105 | if __name__ == '__main__': 106 | main() -------------------------------------------------------------------------------- /app-engine-utility-service/utilities.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import urllib 24 | import base64 25 | import MySQLdb 26 | from google.appengine.api import urlfetch 27 | 28 | from google.appengine.ext import vendor 29 | vendor.add("lib") 30 | from googleapiclient.discovery import build 31 | from oauth2client.client import GoogleCredentials 32 | 33 | MySQLdb.escape_string("'") 34 | 35 | 36 | def issueReq(reqUrl): 37 | responseStr = urlfetch.fetch(reqUrl) 38 | 39 | return responseStr.content 40 | 41 | 42 | def cleanInput(inputVal, inputLen): 43 | inputVal = str(inputVal) 44 | if len(inputVal) > inputLen: 45 | inputVal = inputVal[:inputLen] 46 | inputVal = inputVal.replace("/", "") 47 | inputVal = inputVal.replace("\\", "") 48 | inputVal = inputVal.replace(";", "") 49 | 50 | return inputVal 51 | 52 | 53 | def pubsubObj(): 54 | credentials = GoogleCredentials.get_application_default() 55 | serviceObj = build("pubsub", "v1", credentials = credentials) 56 | 57 | return serviceObj 58 | 59 | 60 | def publishMsg(projectName, gId, topicName, msgAction): 61 | serviceObj = pubsubObj() 62 | credentialsObj = GoogleCredentials.get_application_default() 63 | 64 | serviceObj = build("pubsub", "v1", credentials = credentialsObj) 65 | topicStr = "projects/%s/topics/%s" % (projectName, topicName) 66 | 67 | msgStr = base64.b64encode(msgAction) 68 | bodyObj = {"messages": 69 | [{ 70 | "attributes": { 71 | "globalId": str(gId) 72 | }, 73 | "data": msgStr 74 | }] 75 | } 76 | respObj = serviceObj.projects().topics().publish( 77 | topic = topicStr, 78 | body = bodyObj 79 | ).execute() 80 | 81 | return respObj 82 | 83 | 84 | def pullMsg(projectName, subName, returnImmediately): 85 | serviceObj = pubsubObj() 86 | subStr = "projects/%s/subscriptions/%s" % (projectName, subName) 87 | 88 | bodyObj = { 89 | "returnImmediately": returnImmediately, 90 | "maxMessages": 1 91 | } 92 | 93 | resp = serviceObj.projects().subscriptions().pull( 94 | subscription = subStr, 95 | body = bodyObj 96 | ).execute() 97 | 98 | return resp 99 | 100 | 101 | def ackMsg(projectName, subName, ackId): 102 | serviceObj = pubsubObj() 103 | subStr = "projects/%s/subscriptions/%s" % (projectName, subName) 104 | ackBody = {"ackIds": [ackId]} 105 | serviceObj.projects().subscriptions().acknowledge( 106 | subscription = subStr, 107 | body = ackBody 108 | ).execute() 109 | 110 | 111 | def dbExecution(sqlCmd, sqlData): 112 | 113 | sqlInstance = "__Cloud_SQL_Instance_Connection_Name__" 114 | 115 | if os.getenv("SERVER_SOFTWARE", "").startswith("Google App Engine/"): 116 | connection = MySQLdb.connect( unix_socket = "/cloudsql/" + sqlInstance, 117 | user = "__Cloud_SQL_Username__", 118 | passwd = "__Cloud_SQL_User_Password__", 119 | db = "prodDb") 120 | else: 121 | connection = MySQLdb.connect( host = "__Cloud_SQL_Public_IP_Address__", 122 | user = "__Cloud_SQL_Username__", 123 | passwd = "__Cloud_SQL_User_Password__", 124 | db = "prodDb") 125 | cursor = connection.cursor() 126 | if len(sqlData) > 0: 127 | try: 128 | cmdExecution = cursor.execute(sqlCmd, *[sqlData]) 129 | except: 130 | cmdExecution = cursor.execute(sqlCmd, [sqlData]) 131 | else: 132 | cmdExecution = cursor.execute(sqlCmd) 133 | connection.commit() 134 | numResults = cursor.rowcount 135 | resultRows = cursor.fetchall() 136 | cursor.close() 137 | connection.close() 138 | resultList = [cmdExecution, numResults, resultRows] 139 | 140 | return resultList 141 | 142 | 143 | if __name__ == "__main__": 144 | main() -------------------------------------------------------------------------------- /app-engine-utility-service/receiveResults.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import ujson 23 | import utilities 24 | from googleapiclient.discovery import build 25 | 26 | from google.appengine.ext import vendor 27 | vendor.add("lib") 28 | import httplib2 29 | from gcloud import storage 30 | from oauth2client.service_account import ServiceAccountCredentials 31 | 32 | bucketName = "__GCS_Storage_Bucket_Name__" 33 | 34 | 35 | def runCycle(jobId, apiName, gcsLoc, globalId, batchId): 36 | credentialsJson = "__Credential_JSON_File_Name__" 37 | 38 | scopesList = ["https://www.googleapis.com/auth/cloud-platform"] 39 | credentialsObj = ServiceAccountCredentials.from_json_keyfile_name( 40 | credentialsJson, 41 | scopes = scopesList 42 | ) 43 | 44 | payloadObj = { 45 | "key": "__Google_Speech_API_Key__" 46 | } 47 | 48 | httpObj = credentialsObj.authorize(httplib2.Http()) 49 | serviceObj = build( 50 | serviceName = "speech", 51 | version = "v1p1beta1", 52 | http = httpObj, 53 | developerKey= "__Google_Speech_API_Key__" 54 | ) 55 | 56 | reqObj = serviceObj.operations().get(name=apiName).execute() 57 | 58 | try: 59 | if reqObj["metadata"]["progressPercent"] == 100: 60 | clientObj = storage.Client() 61 | bucketObj = clientObj.get_bucket(bucketName) 62 | cloudPath = gcsLoc.replace("'", "") 63 | cloudPath = cloudPath.replace(".flac", ".json") 64 | bucketPrexif = "gs://" + bucketName + "/" 65 | cloudPath = cloudPath.replace(bucketPrexif, "") 66 | cloudPath = cloudPath.replace("transcodes/", "") 67 | 68 | globalDir = "/" + str(globalId) + "/" 69 | transDir = globalDir + "transcripts/" + str(batchId) + "-" 70 | newPath = cloudPath.replace(globalDir, transDir) 71 | 72 | blobObj = bucketObj.blob(newPath) 73 | blobObj.upload_from_string(ujson.dumps(reqObj)) 74 | 75 | sqlCmd = """update speechJobs set respExported = %s where jobId = %s""" 76 | sqlData = [1, jobId] 77 | queryResp = utilities.dbExecution(sqlCmd, sqlData) 78 | 79 | return "... job " + str(jobId) + " finished" 80 | else: 81 | return "... job " + str(reqObj["metadata"]["progressPercent"]) + "% complete" 82 | except Exception, e: 83 | return "... job queued" 84 | 85 | 86 | def nextAction(globalId): 87 | projectName = "__GCP_Project_ID__" 88 | topicName = "wordlistQueue" 89 | msgAction = "create-word-list" 90 | 91 | return utilities.publishMsg(projectName, globalId, topicName, msgAction) 92 | 93 | 94 | def main(): 95 | print "Content-type: text/plain; charset=UTF-8\n\n" 96 | 97 | sqlCmd = """select jobId, apiName, gcsLoc, globalId, batchId from speechJobs 98 | where beenProcessed = %s 99 | and respExported = %s 100 | and jobStatus is %s 101 | order by queueTimestamp asc limit 2""" 102 | sqlData = [1, 0, None] 103 | queryResp = utilities.dbExecution(sqlCmd, sqlData) 104 | 105 | for eachEntry in queryResp[2]: 106 | jobId = eachEntry[0] 107 | ##print jobId 108 | apiName = eachEntry[1] 109 | gcsLoc = eachEntry[2] 110 | globalId = eachEntry[3] 111 | batchId = eachEntry[4] 112 | 113 | if jobId: 114 | print "global Id: " + str(globalId) 115 | print "... job " + str(jobId) 116 | print runCycle(jobId, apiName, gcsLoc, globalId, batchId) 117 | 118 | sqlCmd = """select count(*) from speechJobs 119 | where respExported = %s 120 | and jobStatus is %s 121 | and globalId = %s""" 122 | sqlData = [0, None, globalId] 123 | queryResp = utilities.dbExecution(sqlCmd, sqlData) 124 | 125 | print "... " + str(queryResp[2][0][0]) + " jobs still in the queue" 126 | if queryResp[2][0][0] == 0: 127 | print nextAction(globalId) 128 | print "" 129 | 130 | 131 | if __name__ == "__main__": 132 | main() 133 | -------------------------------------------------------------------------------- /app-engine-front-end/ep-searchArchive.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import cgi 24 | import ujson 25 | import urllib 26 | import logging 27 | from datetime import datetime 28 | from collections import Counter 29 | from google.appengine.ext import vendor 30 | from google.appengine.api import urlfetch 31 | 32 | archive_video_search_service_url = "__Archive_Video_Search_Service_URL__" 33 | 34 | 35 | def mkDate(inputStr): 36 | datetimeObj = datetime.strptime(inputStr, "%Y%m%d") 37 | date_formatted_for_dateList = datetimeObj.strftime("%Y-%m-%d") 38 | dateDesc = datetimeObj.strftime("%b %-d, %Y") 39 | 40 | return [dateDesc, date_formatted_for_dateList] 41 | 42 | 43 | def main(): 44 | passedArgs = cgi.FieldStorage() 45 | queryString = passedArgs["q"].value 46 | orgIdentifier = passedArgs["orgId"].value 47 | 48 | logging.info("archive_search") 49 | 50 | payloadObj = { 51 | "orgId": orgIdentifier, 52 | "q": queryString 53 | } 54 | 55 | urlParams = urllib.urlencode(payloadObj, doseq=True) 56 | reqUrl = archive_video_search_service_url + "/?%s" % urlParams 57 | responseObj = urlfetch.fetch(reqUrl) 58 | responseStr = responseObj.content 59 | 60 | searchObj = ujson.loads(responseStr) 61 | 62 | dateList = [] 63 | cntList = [] 64 | descList = [] 65 | tooltipList = [] 66 | pieData = [] 67 | 68 | meeting_total = 0 69 | result_total = 0 70 | 71 | resultCnt = 0 72 | resultObj = [] 73 | for eachResp in searchObj["aggregations"]["group_by_meeting"]["buckets"]: 74 | meeting_total += 1 75 | dateIndex = eachResp["meeting_details"]["hits"]["hits"][0]["_source"]["meetingDate"] 76 | datetimeObj = datetime.strptime(dateIndex, "%Y%m%d") 77 | meetingDate = datetimeObj.strftime("%B %-d, %Y") 78 | 79 | tmpObj = {} 80 | tmpObj["urlIdentifier"] = eachResp["meeting_details"]["hits"]["hits"][0]["_source"]["urlIdentifier"] 81 | tmpObj["transcriptMatches"] = eachResp["doc_count"] 82 | tmpObj["meetingDate"] = meetingDate 83 | tmpObj["dateIndex"] = dateIndex 84 | tmpObj["meetingDesc"] = eachResp["meeting_details"]["hits"]["hits"][0]["_source"]["meetingDesc"] 85 | resultObj.append(tmpObj) 86 | resultCnt += 1 87 | 88 | meetingDate = eachResp["meeting_details"]["hits"]["hits"][0]["_source"]["meetingDate"] 89 | formatted_date_list = mkDate(meetingDate); 90 | dateDesc = formatted_date_list[0]; 91 | date_formatted_for_dateList = formatted_date_list[1]; 92 | 93 | returnCnt = eachResp["doc_count"] 94 | cntList.append(returnCnt) 95 | 96 | result_total += returnCnt 97 | 98 | dateList.append(date_formatted_for_dateList) 99 | 100 | meetingDesc = eachResp["meeting_details"]["hits"]["hits"][0]["_source"]["meetingDesc"] 101 | descList.append(meetingDesc) 102 | 103 | tooltipStr = "%s - %s - %s results" % (meetingDesc, dateDesc, returnCnt) 104 | tooltipList.append(tooltipStr) 105 | 106 | entryObj = {} 107 | entryObj["meetingDesc"] = meetingDesc 108 | entryObj["returnCnt"] = returnCnt 109 | pieData.append(entryObj) 110 | 111 | infoObj = {} 112 | infoObj["meeting_total"] = resultCnt 113 | infoObj["result_total"] = result_total 114 | 115 | respObj = {} 116 | respObj["searchResults"] = resultObj 117 | 118 | chartObj = {} 119 | chartObj["countList"] = cntList 120 | chartObj["dateList"] = dateList 121 | chartObj["tooltipList"] = tooltipList 122 | 123 | 124 | masterList = [] 125 | for eachEntry in pieData: 126 | for i in range(0, eachEntry["returnCnt"]): 127 | masterList.append(eachEntry["meetingDesc"]) 128 | tallyDict = dict(Counter(masterList)) 129 | pie_data_list = [] 130 | for eachEntry in tallyDict.keys(): 131 | pie_data_list.append([eachEntry, tallyDict[eachEntry]]) 132 | chartObj["pieData"] = pie_data_list 133 | 134 | respObj["chartData"] = chartObj 135 | respObj["resultsInfo"] = infoObj 136 | 137 | print "Expires: 0" 138 | print "Cache-Control: no-cache, no-store, must-revalidate, max-age=0" 139 | print "Content-Type: application/json\n" 140 | print ujson.dumps(respObj) 141 | 142 | 143 | if __name__ == "__main__": 144 | main() 145 | -------------------------------------------------------------------------------- /index-meeting/worker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import cgi 24 | import json 25 | import time 26 | import urllib 27 | import base64 28 | import requests 29 | from google.cloud import pubsub 30 | from google.cloud import storage 31 | from elasticsearch import helpers 32 | from elasticsearch import Elasticsearch 33 | from oauth2client.service_account import ServiceAccountCredentials 34 | 35 | service_account_json = "__Credential_JSON_File_Name__" 36 | dirPath = os.path.normpath(os.getcwd()) 37 | service_account_path = os.path.join(dirPath, service_account_json) 38 | 39 | projectId = "__GCP_Project_ID__" 40 | topicName = "indexQueue" 41 | subName = "index-meeting-subscription" 42 | 43 | bucketName = "__GCS_Storage_Bucket_Name__" 44 | utility_service_url = "__Utility_Service_URL__" 45 | 46 | psClient = pubsub.SubscriberClient() 47 | 48 | topicPath = psClient.topic_path( 49 | projectId, 50 | topicName 51 | ) 52 | 53 | subPath = psClient.subscription_path( 54 | projectId, 55 | subName 56 | ) 57 | 58 | subObj = psClient.subscribe( 59 | subPath 60 | ) 61 | 62 | cummTime = None 63 | 64 | searchClient = Elasticsearch( 65 | ["https://7b76c3d47a7445119d54a4089ae9e9a3.us-central1.gcp.cloud.es.io:9243/"], 66 | http_auth = ( 67 | "elastic", 68 | "EbJ3suYnJ5YdTIJRM9i38Swu" 69 | ) 70 | ) 71 | 72 | 73 | def psCall(reqUrl, postPayload): 74 | scopesList = ["https://www.googleapis.com/auth/cloud-platform"] 75 | credentialsObj = ServiceAccountCredentials.from_json_keyfile_name( 76 | service_account_path, 77 | scopes = scopesList 78 | ) 79 | 80 | accessToken = "Bearer %s" % credentialsObj.get_access_token().access_token 81 | headerObj = { 82 | "authorization": accessToken, 83 | } 84 | 85 | reqObj = requests.post( 86 | reqUrl, 87 | data = json.dumps(postPayload), 88 | headers = headerObj 89 | ) 90 | 91 | return reqObj.text 92 | 93 | 94 | def processFile(jsonObj, globalId, orgIdentifier, meetingDate, meetingDesc, urlIdentifier): 95 | maxTime = 0 96 | global cummTime 97 | indexList = [] 98 | if "response" in jsonObj: 99 | if "results" in jsonObj["response"]: 100 | for eachAlt in jsonObj["response"]["results"]: 101 | if "alternatives" in eachAlt: 102 | transcriptStr = eachAlt["alternatives"][0]["transcript"] 103 | videoTimestamp = eachAlt["alternatives"][0]["words"][0]["startTime"] 104 | 105 | videoTimestamp = str(videoTimestamp).replace("s", "") 106 | videoTimestamp = float(videoTimestamp) 107 | 108 | 109 | wordsLen = len(eachAlt["alternatives"][0]["words"]) - 1 110 | endTime = eachAlt["alternatives"][0]["words"][wordsLen]["endTime"] 111 | endTime = str(endTime).replace("s", "") 112 | endTime = float(endTime) 113 | 114 | segLen = round(endTime - videoTimestamp) 115 | 116 | displayTime = videoTimestamp + float(cummTime) 117 | maxTime = videoTimestamp 118 | 119 | transcriptStr = transcriptStr.replace("Lewisville", "Louisville") 120 | transcriptStr = transcriptStr.replace("Pro stack", "PROSTAC") 121 | transcriptStr = transcriptStr.replace("pro stack", "PROSTAC") 122 | transcriptStr = transcriptStr.replace("Pro Stacks", "PROSTAC") 123 | transcriptStr = transcriptStr.replace("pro Strat", "PROSTAC") 124 | transcriptStr = transcriptStr.replace("pro-sex", "PROSTAC") 125 | 126 | indexEntry = { 127 | "_index": orgIdentifier, 128 | "_type": "meeting-transcript", 129 | "_source": { 130 | "globalId": int(globalId), 131 | "mediaTimestamp": displayTime, 132 | "segmentLength": segLen, 133 | "meetingDesc": meetingDesc, 134 | "transcriptStr": transcriptStr, 135 | "meetingDate": meetingDate, 136 | "urlIdentifier": urlIdentifier 137 | } 138 | } 139 | indexList.append(indexEntry) 140 | 141 | return indexList 142 | 143 | 144 | def runIndexing(globalId, prodTrans, orgIdentifier, meetingDate, meetingDesc, urlIdentifier): 145 | cloudPath = "accounts/" + orgIdentifier + "/enrichments/" + str(globalId) + "/transcripts/" + str(prodTrans) + "/" 146 | 147 | clientObj = storage.Client.from_service_account_json(service_account_path) 148 | bucketObj = clientObj.get_bucket(bucketName) 149 | listObj = bucketObj.list_blobs(prefix = cloudPath) 150 | transcriptList = [] 151 | 152 | for eachEntry in listObj: 153 | if ".json" in eachEntry.name: 154 | transcriptList.append(eachEntry.name) 155 | 156 | global cummTime 157 | cummTime = 0 158 | maxTime = 0 159 | fileCnt = 0 160 | 161 | fileList = [] 162 | for eachFile in sorted(transcriptList, reverse=False): 163 | fileCnt += 1 164 | fileList.append(eachFile) 165 | 166 | blobObj = bucketObj.get_blob(eachFile) 167 | blobStr = blobObj.download_as_string() 168 | jsonObj = json.loads(blobStr) 169 | 170 | indexList = processFile( 171 | jsonObj, 172 | globalId, 173 | orgIdentifier, 174 | meetingDate, 175 | meetingDesc, 176 | urlIdentifier 177 | ) 178 | 179 | helpers.bulk(searchClient, indexList) 180 | 181 | cummTime = float(cummTime) + 10800 182 | 183 | return cloudPath, fileList, fileCnt, len(indexList) 184 | 185 | 186 | def dispatchWorker(ackId, globalId): 187 | try: 188 | reqUrl = utility_service_url + "/meetingDetails?gId=%s" % globalId 189 | responseObj = requests.get(reqUrl) 190 | respTxt = responseObj.text 191 | 192 | jsonObj = json.loads(respTxt) 193 | 194 | prodTrans = jsonObj["prodTranscript"] 195 | orgIdentifier = jsonObj["orgIdentifier"] 196 | meetingDate = jsonObj["meetingDate"] 197 | meetingDesc = jsonObj["meetingDesc"] 198 | beenIndexed = jsonObj["beenIndexed"] 199 | urlIdentifier = jsonObj["urlIdentifier"] 200 | 201 | if beenIndexed == 0: 202 | toggleResp = toggleIndex(globalId) 203 | print "... index marker has been updated " + str(toggleResp) 204 | cloudPath, fileList, fileCnt, indexLen = runIndexing( 205 | globalId, 206 | prodTrans, 207 | orgIdentifier, 208 | meetingDate, 209 | meetingDesc, 210 | urlIdentifier 211 | ) 212 | 213 | print globalId 214 | print "" 215 | print cloudPath 216 | print "" 217 | print fileList 218 | print "" 219 | 220 | print str(fileCnt) + " files processed" 221 | else: 222 | print "... entry has already been indexed" 223 | postPayload = { 224 | "ackIds": [ackId] 225 | } 226 | subStr = "projects/%s/subscriptions/%s" % (projectId, subName) 227 | reqUrl = "https://pubsub.googleapis.com/v1/%s:acknowledge" % subStr 228 | psMsg = psCall(reqUrl, postPayload) 229 | print "... Pubsub message acknowledged" 230 | except Exception as e: 231 | print "skip " + e.message 232 | 233 | 234 | def toggleIndex(globalId): 235 | reqUrl = utility_service_url + "/toggleIndex?gId=%s" % globalId 236 | responseObj = requests.get(reqUrl) 237 | respTxt = responseObj.text 238 | 239 | return respTxt 240 | 241 | 242 | def nextAction(globalId): 243 | reqUrl = utility_service_url + "/msgPublish" 244 | payloadObj = { 245 | "msgAction": "publish-transcript", 246 | "topicName": "publish-transcript-queue", 247 | "gId": globalId 248 | } 249 | responseObj = requests.get( 250 | reqUrl, 251 | params = payloadObj 252 | ) 253 | respTxt = responseObj.text 254 | 255 | return respTxt 256 | 257 | 258 | def main(): 259 | postPayload = { 260 | "returnImmediately": True, 261 | "maxMessages": 1 262 | } 263 | subStr = "projects/%s/subscriptions/%s" % (projectId, subName) 264 | reqUrl = "https://pubsub.googleapis.com/v1/%s:pull" % subStr 265 | 266 | while True: 267 | psMsg = psCall(reqUrl, postPayload) 268 | try: 269 | jsonObj = json.loads(psMsg) 270 | msgType = base64.b64decode(jsonObj["receivedMessages"][0]["message"]["data"]) 271 | print "Message Received. Type = '%s'" % str(msgType) 272 | ackId = jsonObj["receivedMessages"][0]["ackId"] 273 | globalId = jsonObj["receivedMessages"][0]["message"]["attributes"]["globalId"] 274 | print ackId 275 | print globalId 276 | dispatchWorker(ackId, globalId) 277 | print nextAction(globalId) 278 | except: 279 | pass 280 | time.sleep(4) 281 | 282 | 283 | if __name__ == "__main__": 284 | main() 285 | -------------------------------------------------------------------------------- /create-word-list/worker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import json 24 | import time 25 | import base64 26 | import requests 27 | import httplib2 28 | from google.cloud import pubsub 29 | from google.cloud import storage 30 | from oauth2client.service_account import ServiceAccountCredentials 31 | 32 | service_account_json = "__Credential_JSON_File_Name__" 33 | dirPath = os.path.normpath(os.getcwd()) 34 | service_account_path = os.path.join(dirPath, service_account_json) 35 | 36 | projectId = "__GCP_Project_ID__" 37 | topicName = "wordlistQueue" 38 | subName = "word-list-creation-subscription" 39 | 40 | bucketName = "__GCS_Storage_Bucket_Name__" 41 | utility_service_url = "__Utility_Service_URL__" 42 | 43 | psClient = pubsub.SubscriberClient() 44 | 45 | topicPath = psClient.topic_path( 46 | projectId, 47 | topicName 48 | ) 49 | 50 | subPath = psClient.subscription_path( 51 | projectId, 52 | subName 53 | ) 54 | 55 | subObj = psClient.subscribe( 56 | subPath 57 | ) 58 | 59 | 60 | def psCall(reqUrl, postPayload): 61 | scopesList = ["https://www.googleapis.com/auth/cloud-platform"] 62 | credentialsObj = ServiceAccountCredentials.from_json_keyfile_name( 63 | service_account_json, 64 | scopes = scopesList 65 | ) 66 | 67 | accessToken = "Bearer %s" % credentialsObj.get_access_token().access_token 68 | headerObj = { 69 | "authorization": accessToken, 70 | } 71 | 72 | reqObj = requests.post( 73 | reqUrl, 74 | data = json.dumps(postPayload), 75 | headers = headerObj 76 | ) 77 | 78 | return reqObj.text 79 | 80 | 81 | def acknowledgeMsg(ackId): 82 | postPayload = { 83 | "ackIds": [ackId] 84 | } 85 | subStr = "projects/%s/subscriptions/%s" % (projectId, subName) 86 | reqUrl = "https://pubsub.googleapis.com/v1/%s:acknowledge" % subStr 87 | psMsg = psCall(reqUrl, postPayload) 88 | 89 | return "... Pubsub message acknowledged" 90 | 91 | 92 | def get_api_results(globalId, orgIdentifier, prodTranscript): 93 | basePath = "accounts/" + orgIdentifier + "/enrichments/" + str(globalId) + "/transcripts/" 94 | cloudPath = basePath + str(prodTranscript) + "/" 95 | clientObj = storage.Client.from_service_account_json(service_account_path) 96 | bucketObj = clientObj.get_bucket(bucketName) 97 | listObj = bucketObj.list_blobs(prefix=cloudPath) 98 | transcriptList = [] 99 | for eachEntry in listObj: 100 | if ".json" in eachEntry.name: 101 | transcriptList.append(str(eachEntry.name)) 102 | #print eachEntry.size 103 | 104 | return transcriptList 105 | 106 | 107 | def results_files_to_string(transcriptList, exportType): 108 | clientObj = storage.Client.from_service_account_json(service_account_path) 109 | bucketObj = clientObj.get_bucket(bucketName) 110 | 111 | fileCnt = 0 112 | 113 | masterStr = "" 114 | for eachFile in sorted(transcriptList, reverse=False): 115 | fileCnt += 1 116 | 117 | blobObj = bucketObj.get_blob(eachFile) 118 | blobStr = blobObj.download_as_string() 119 | jsonObj = json.loads(blobStr) 120 | 121 | if "response" in jsonObj: 122 | if "results" in jsonObj["response"]: 123 | for eachAlt in jsonObj["response"]["results"]: 124 | tmpStr = "" 125 | if exportType is "list": 126 | if "alternatives" in eachAlt: 127 | for eachWord in eachAlt["alternatives"][0]["words"]: 128 | tmpStr = eachWord["word"] 129 | tmpStr = tmpStr.lower() 130 | tmpStr = tmpStr.replace(".", "") 131 | tmpStr = tmpStr.replace(",", "") 132 | tmpStr = tmpStr.replace("?", "") 133 | tmpStr = tmpStr.replace("!", "") 134 | tmpStr = tmpStr + "\n" 135 | masterStr = masterStr + tmpStr 136 | if exportType is "long": 137 | tmpStr = eachAlt["alternatives"][0]["transcript"] 138 | masterStr = masterStr + " " + tmpStr 139 | masterStr = masterStr.replace("lewisville", "louisville") 140 | masterStr = masterStr.replace("Pro stack", "PROSTAC") 141 | masterStr = masterStr.replace("pro stack", "PROSTAC") 142 | masterStr = masterStr.replace("Pro Stacks", "PROSTAC") 143 | masterStr = masterStr.replace("pro Strat", "PROSTAC") 144 | masterStr = masterStr.replace("pro-sex", "PROSTAC") 145 | 146 | return masterStr, fileCnt 147 | 148 | 149 | def write_string_to_gcs(globalId, orgIdentifier, prodTranscript, exportType, masterStr): 150 | basePath = "accounts/" + orgIdentifier + "/enrichments/" + str(globalId) + "/transcripts/" 151 | fileName = "rawTxt-" + str(globalId) + "-" + str(prodTranscript) + "-" + exportType + ".txt" 152 | newPath = basePath + fileName 153 | 154 | clientObj = storage.Client.from_service_account_json(service_account_path) 155 | bucketObj = clientObj.get_bucket(bucketName) 156 | blobObj = bucketObj.blob(newPath) 157 | blobObj.upload_from_string(masterStr.strip()) 158 | 159 | return "... file created in GCS" 160 | 161 | 162 | def lookupMeeting(globalId): 163 | reqUrl = utility_service_url + "/meetingDetails" 164 | payloadObj = { 165 | "gId": globalId 166 | } 167 | responseObj = requests.get(reqUrl, params=payloadObj) 168 | respTxt = responseObj.text 169 | jsonObj = json.loads(respTxt) 170 | 171 | return jsonObj["prodTranscript"], jsonObj["orgIdentifier"] 172 | 173 | 174 | def nextAction(globalId): 175 | reqUrl = utility_service_url + "/msgPublish" 176 | payloadObj = { 177 | "msgAction": "create-wordcloud", 178 | "topicName": "wordcloudQueue", 179 | "gId": globalId 180 | } 181 | responseObj = requests.get( 182 | reqUrl, 183 | params = payloadObj 184 | ) 185 | respTxt = responseObj.text 186 | respTxt = respTxt.replace("\r", "") 187 | respTxt = respTxt.replace("\n", "") 188 | 189 | return ".. next action initiated: " + str(respTxt) 190 | 191 | 192 | def issue_transcript_error(globalId): 193 | reqUrl = utility_service_url + "/toggleTranscriptErr" 194 | payloadObj = { 195 | "gId": globalId 196 | } 197 | responseObj = requests.get(reqUrl, params=payloadObj) 198 | respTxt = responseObj.text 199 | 200 | return respTxt 201 | 202 | 203 | def dispatchWorker(ackId, globalId): 204 | successFlag = None 205 | try: 206 | print ".. creating word list for meeting: " + str(globalId) 207 | prodTranscript, orgIdentifier = lookupMeeting(globalId) 208 | print "... word list will be created from transcript: " + str(prodTranscript) 209 | transcriptList = get_api_results(globalId, orgIdentifier, prodTranscript) 210 | print "... " + str(len(transcriptList)) + " files will be processed" 211 | masterStr, fileCnt = results_files_to_string(transcriptList, "list") 212 | 213 | print "... " + str(fileCnt) + " files processed" 214 | print ".... the masterStr is " + str(len(masterStr)) + " characters in length" 215 | if len(masterStr) > 0 and fileCnt > 0: 216 | print write_string_to_gcs(globalId, orgIdentifier, prodTranscript, "list", masterStr) 217 | successFlag = True 218 | else: 219 | print ".... something isn't right so issuing a transcriptErr" 220 | toggleResp = issue_transcript_error(globalId) 221 | print ".... " + str(toggleResp) 222 | successFlag = False 223 | 224 | print acknowledgeMsg(ackId) 225 | except Exception as e: 226 | print "something went wrong" 227 | print acknowledgeMsg(ackId) 228 | print "skip " + e.message 229 | successFlag = False 230 | 231 | return successFlag 232 | 233 | 234 | def main(): 235 | postPayload = { 236 | "returnImmediately": True, 237 | "maxMessages": 1 238 | } 239 | subStr = "projects/%s/subscriptions/%s" % (projectId, subName) 240 | reqUrl = "https://pubsub.googleapis.com/v1/%s:pull" % subStr 241 | 242 | while True: 243 | psMsg = psCall(reqUrl, postPayload) 244 | try: 245 | jsonObj = json.loads(psMsg) 246 | msgType = base64.b64decode(jsonObj["receivedMessages"][0]["message"]["data"]) 247 | ackId = jsonObj["receivedMessages"][0]["ackId"] 248 | globalId = jsonObj["receivedMessages"][0]["message"]["attributes"]["globalId"] 249 | successFlag = dispatchWorker(ackId, globalId) 250 | if successFlag == True: 251 | print nextAction(globalId) 252 | else: 253 | print ".. not initiating next action" 254 | print "" 255 | except: 256 | pass 257 | time.sleep(4) 258 | 259 | 260 | if __name__ == "__main__": 261 | main() 262 | -------------------------------------------------------------------------------- /transcode-video-to-audio/worker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import re 24 | import sys 25 | import json 26 | import time 27 | import base64 28 | import shutil 29 | import calendar 30 | import requests 31 | import subprocess 32 | from google.cloud import storage 33 | from time import gmtime, strftime 34 | from oauth2client.service_account import ServiceAccountCredentials 35 | 36 | dirPath = os.path.normpath(os.getcwd()) 37 | credentialsJson = "__Credential_JSON_File_Name__" 38 | 39 | projectId = "__GCP_Project_ID__" 40 | topicName = "transcodeQueue" 41 | subName = "media-transcode-subscription" 42 | 43 | bucketName = "__GCS_Storage_Bucket_Name__" 44 | utility_service_url = "__Utility_Service_URL__" 45 | 46 | segment_length_minutes = 180 47 | segment_length_seconds = segment_length_minutes * 60 48 | 49 | 50 | def psCall(reqUrl, postPayload): 51 | scopesList = ["https://www.googleapis.com/auth/cloud-platform"] 52 | credentialsObj = ServiceAccountCredentials.from_json_keyfile_name( 53 | credentialsJson, 54 | scopes = scopesList 55 | ) 56 | 57 | accessToken = "Bearer %s" % credentialsObj.get_access_token().access_token 58 | headerObj = { 59 | "authorization": accessToken, 60 | } 61 | 62 | reqObj = requests.post( 63 | reqUrl, 64 | data = json.dumps(postPayload), 65 | headers = headerObj 66 | ) 67 | 68 | return reqObj.text 69 | 70 | 71 | def modifyDeadline(ackId, timeWindow): 72 | postPayload = { 73 | "ackIds": [ackId], 74 | "ackDeadlineSeconds": timeWindow 75 | } 76 | subStr = "projects/%s/subscriptions/%s" % (projectId, subName) 77 | reqUrl = "https://pubsub.googleapis.com/v1/%s:modifyAckDeadline" % subStr 78 | print ".... extending acknowledgement deadline by " + str(timeWindow) + " seconds" 79 | 80 | return psCall(reqUrl, postPayload) 81 | 82 | 83 | def downloadFile(fileName, filePath, orgIdentifier): 84 | clientObj = storage.Client() 85 | bucketObj = clientObj.get_bucket(bucketName) 86 | cloudPath = "accounts/%s/video/" % orgIdentifier 87 | cloudPath = cloudPath + fileName 88 | print ".... defining cloud path for file" 89 | print "..... " + cloudPath 90 | blobObj = bucketObj.blob(cloudPath) 91 | print ".... downloading source file locally" 92 | print "..... " + filePath 93 | with open(filePath, "w") as fileObj: 94 | blobObj.download_to_file(fileObj) 95 | 96 | return 97 | 98 | 99 | def uploadFiles(globalId, segmentsFlac, ackId, orgIdentifier, tsDir): 100 | clientObj = storage.Client() 101 | bucketObj = clientObj.get_bucket("municeps") 102 | flacList = os.listdir(segmentsFlac) 103 | print "... preparing to upload %s files" % str(len(flacList)) 104 | for eachFile in os.listdir(segmentsFlac): 105 | print ".... " + eachFile 106 | cloudPath = "accounts/%s/enrichments/%s/transcodes/%s/%s" % (orgIdentifier, str(globalId), tsDir, eachFile) 107 | ##print ".... " + cloudPath 108 | blobObj = bucketObj.blob(cloudPath) 109 | localPath = segmentsFlac + "/" + eachFile 110 | ##print ".... " + localPath 111 | blobObj.upload_from_filename(localPath) 112 | modifyDeadline(ackId, 120) 113 | 114 | return 115 | 116 | 117 | def nameSegments(videoName, segmentsWav): 118 | timeStamp = strftime("%Y%m%d-%H%M%S", gmtime()) 119 | 120 | segmentName = videoName.lower() 121 | periodNum = len(segmentName.split(".")) 122 | segmentName = segmentName[::-1] 123 | segmentName = segmentName.split(".")[periodNum - 1] 124 | segmentName = segmentName[::-1] 125 | segmentName = re.sub(r'[^\w]', "", segmentName) 126 | segmentName = timeStamp + "-" + segmentName + "-" + "%04d.wav" 127 | segmentPath = os.path.join(segmentsWav, segmentName) 128 | 129 | return segmentPath 130 | 131 | 132 | def transcode(globalId, videoName, ackId, orgIdentifier, tsDir): 133 | workingDir = os.path.join(dirPath, str(globalId)) 134 | if not os.path.exists(workingDir): 135 | os.makedirs(workingDir) 136 | filePath = os.path.join(workingDir, videoName) 137 | 138 | print "... preparing to download media" 139 | print ".... " + videoName 140 | downloadFile(videoName, filePath, orgIdentifier) 141 | print ".... media downloaded" 142 | 143 | print "... creating WAV directory locally" 144 | segmentsWav = os.path.join(workingDir, "segmentsWav") 145 | print ".... " + segmentsWav 146 | if os.path.exists(segmentsWav): 147 | shutil.rmtree(segmentsWav) 148 | if not os.path.exists(segmentsWav): 149 | os.makedirs(segmentsWav) 150 | 151 | print "... creating FLAC directory locally" 152 | segmentsFlac = os.path.join(workingDir, "segmentsFlac") 153 | print ".... " + segmentsFlac 154 | if os.path.exists(segmentsFlac): 155 | shutil.rmtree(segmentsFlac) 156 | if not os.path.exists(segmentsFlac): 157 | os.makedirs(segmentsFlac) 158 | 159 | print "... defining segments path" 160 | segmentPath = nameSegments(videoName, segmentsWav) 161 | print ".... " + segmentPath 162 | 163 | modifyDeadline(ackId, 120) 164 | 165 | print "... converting to WAV format" 166 | 167 | print ".... source file is in " + filePath[-4:] + " format" 168 | 169 | segmentCmd = "ffmpeg -loglevel error -i %s -f segment -segment_time %s -reset_timestamps 1 -ac 1 -ar %s %s" % ( 170 | filePath, 171 | segment_length_seconds, 172 | "16000", 173 | segmentPath 174 | ) 175 | 176 | subprocess.call(segmentCmd, shell=True) 177 | 178 | modifyDeadline(ackId, 120) 179 | 180 | print ".... preparing to convert " + str(len(os.listdir(segmentsWav))) + " files" 181 | for eachFile in os.listdir(segmentsWav): 182 | wavPath = os.path.join(segmentsWav, eachFile) 183 | if os.path.isfile(wavPath): 184 | if wavPath[-4:] == ".wav": 185 | 186 | flacName = wavPath[:-4] + ".flac" 187 | flacName = flacName.replace(segmentsWav, "") 188 | flacName = flacName[1:] 189 | flacPath = os.path.join(segmentsFlac, flacName) 190 | 191 | print "..... running ffmpeg - converting to FLAC format" 192 | convertCmd = "ffmpeg -loglevel error -i %s %s" % ( 193 | wavPath, 194 | flacPath 195 | ) 196 | subprocess.call(convertCmd, shell=True) 197 | 198 | modifyDeadline(ackId, 120) 199 | 200 | uploadFiles(globalId, segmentsFlac, ackId, orgIdentifier, tsDir) 201 | 202 | #Clean up 203 | if os.path.exists(workingDir): 204 | shutil.rmtree(workingDir) 205 | print "... process complete" 206 | print acknowledgeMsg(ackId) 207 | 208 | return 209 | 210 | 211 | def acknowledgeMsg(ackId): 212 | postPayload = { 213 | "ackIds": [ackId] 214 | } 215 | subStr = "projects/%s/subscriptions/%s" % (projectId, subName) 216 | reqUrl = "https://pubsub.googleapis.com/v1/%s:acknowledge" % subStr 217 | psMsg = psCall(reqUrl, postPayload) 218 | 219 | return "... pubsub message acknowledged\n\n" 220 | 221 | 222 | def lookupName(globalId): 223 | reqUrl = utility_service_url + "/meetingDetails?gId=%s" % globalId 224 | responseObj = requests.get(reqUrl) 225 | respTxt = responseObj.text 226 | jsonObj = json.loads(respTxt) 227 | videoName = jsonObj["videoName"] 228 | beenTranscoded = jsonObj["beenTranscoded"] 229 | orgIdentifier = jsonObj["orgIdentifier"] 230 | 231 | return videoName, beenTranscoded, orgIdentifier 232 | 233 | 234 | def toggleTranscode(globalId): 235 | reqUrl = utility_service_url + "/toggleTranscode?gId=%s" % globalId 236 | responseObj = requests.get(reqUrl) 237 | respTxt = responseObj.text 238 | 239 | return respTxt 240 | 241 | 242 | def assignId(globalId, prodTranscode): 243 | reqUrl = utility_service_url + "/idTranscode" 244 | payloadObj = { 245 | "gId": globalId, 246 | "transcode": prodTranscode 247 | } 248 | responseObj = requests.get(reqUrl, params=payloadObj) 249 | respTxt = responseObj.text 250 | 251 | return respTxt 252 | 253 | 254 | def dispatchWorker(ackId, globalId): 255 | if 1==1: 256 | videoName, beenTranscoded, orgIdentifier = lookupName(globalId) 257 | print "... beenTranscoded: " + str(beenTranscoded) 258 | if beenTranscoded == 0: 259 | toggleResp = toggleTranscode(globalId) 260 | print "... transcode marker has been updated" + str(toggleResp) 261 | if videoName is not None: 262 | epochTime = calendar.timegm(time.gmtime()) 263 | tsDir = str(epochTime) + "-" + str(segment_length_minutes) + "-mins" 264 | transcode(globalId, videoName, ackId, orgIdentifier, tsDir) 265 | assignId(globalId, tsDir) 266 | else: 267 | print "... there is no video name" 268 | else: 269 | print "... entry has already been transcoded" 270 | print acknowledgeMsg(ackId) 271 | 272 | 273 | def nextAction(globalId): 274 | reqUrl = utility_service_url + "/msgPublish" 275 | payloadObj = { 276 | "msgAction": "speechJob", 277 | "topicName": "speechQueue", 278 | "gId": globalId 279 | } 280 | responseObj = requests.get( 281 | reqUrl, 282 | params = payloadObj 283 | ) 284 | respTxt = responseObj.text 285 | 286 | return respTxt 287 | 288 | 289 | def main(): 290 | postPayload = { 291 | "returnImmediately": True, 292 | "maxMessages": 1 293 | } 294 | subStr = "projects/%s/subscriptions/%s" % (projectId, subName) 295 | reqUrl = "https://pubsub.googleapis.com/v1/%s:pull" % subStr 296 | while True: 297 | psMsg = psCall(reqUrl, postPayload) 298 | try: 299 | jsonObj = json.loads(psMsg) 300 | msgType = base64.b64decode(jsonObj["receivedMessages"][0]["message"]["data"]) 301 | print "Message Received. Type = '%s'" % str(msgType) 302 | ackId = jsonObj["receivedMessages"][0]["ackId"] 303 | globalId = jsonObj["receivedMessages"][0]["message"]["attributes"]["globalId"] 304 | print ackId 305 | print globalId 306 | dispatchWorker(ackId, globalId) 307 | print nextAction(globalId) 308 | except: 309 | pass 310 | time.sleep(10) 311 | 312 | 313 | if __name__ == '__main__': 314 | main() 315 | 316 | 317 | -------------------------------------------------------------------------------- /generate-wordcloud/worker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import time 24 | import json 25 | import base64 26 | import random 27 | import requests 28 | import calendar 29 | from scipy.misc import imsave 30 | from scipy.misc import imresize 31 | from google.cloud import pubsub 32 | from collections import Counter 33 | from google.cloud import storage 34 | from nltk.corpus import stopwords 35 | from wordcloud import WordCloud, ImageColorGenerator 36 | from matplotlib.colors import LinearSegmentedColormap 37 | from oauth2client.service_account import ServiceAccountCredentials 38 | 39 | service_account_json = "__Credential_JSON_File_Name__" 40 | dirPath = os.path.normpath(os.getcwd()) 41 | service_account_path = os.path.join(dirPath, service_account_json) 42 | 43 | 44 | projectId = "__GCP_Project_ID__" 45 | topicName = "wordcloudQueue" 46 | subName = "wordcloud-creation-subscription" 47 | 48 | bucketName = "__GCS_Storage_Bucket_Name__" 49 | utility_service_url = "__Utility_Service_URL__" 50 | 51 | 52 | psClient = pubsub.SubscriberClient() 53 | 54 | topicPath = psClient.topic_path( 55 | projectId, 56 | topicName 57 | ) 58 | 59 | subPath = psClient.subscription_path( 60 | projectId, 61 | subName 62 | ) 63 | 64 | subObj = psClient.subscribe( 65 | subPath 66 | ) 67 | 68 | 69 | def psCall(reqUrl, postPayload): 70 | scopesList = ["https://www.googleapis.com/auth/cloud-platform"] 71 | credentialsObj = ServiceAccountCredentials.from_json_keyfile_name( 72 | service_account_json, 73 | scopes = scopesList 74 | ) 75 | 76 | accessToken = "Bearer %s" % credentialsObj.get_access_token().access_token 77 | headerObj = { 78 | "authorization": accessToken, 79 | } 80 | 81 | reqObj = requests.post( 82 | reqUrl, 83 | data = json.dumps(postPayload), 84 | headers = headerObj 85 | ) 86 | 87 | return reqObj.text 88 | 89 | 90 | def recolorBlue(**kwargs): 91 | return "hsl(%d, %d%%, %d%%)" % ( 92 | random.randint(210, 230), 93 | random.randint(60, 90), 94 | random.randint(35, 45)) 95 | 96 | 97 | def getStopwords(): 98 | stopwordsPath = os.path.join(dirPath, "stopwords-20180109-133115.json") 99 | with open(stopwordsPath) as stopwordFile: 100 | contentsStr = stopwordFile.read() 101 | jsonObj = json.loads(contentsStr) 102 | stopwordList = [] 103 | for eachEntry in jsonObj: 104 | stopwordList.append(eachEntry["word"]) 105 | 106 | return stopwordList 107 | 108 | 109 | def generateStr(globalId, orgIdentifier, prodTranscript): 110 | masterStr = "" 111 | stopWords = set(stopwords.words("english")) 112 | 113 | fileName = "rawTxt-" + str(globalId) + "-" + prodTranscript + "-list.txt" 114 | cloudPath = "accounts/" + orgIdentifier + "/enrichments/" + str(globalId) + "/transcripts/" + fileName 115 | clientObj = storage.Client.from_service_account_json(service_account_path) 116 | bucketObj = clientObj.get_bucket(bucketName) 117 | blobObj = bucketObj.blob(cloudPath) 118 | masterStr = blobObj.download_as_string() 119 | 120 | masterStr = masterStr.strip() 121 | masterStr = masterStr.replace("\n", " ") 122 | 123 | #masterStr = masterStr.replace("lewisville", "louisville") 124 | 125 | stopwordList = getStopwords() 126 | stopWords.update( 127 | stopwordList 128 | ) 129 | 130 | cleanList = [i for i in masterStr.lower().split() if i not in stopWords] 131 | 132 | c = Counter(cleanList) 133 | cleanStr = " ".join(cleanList) 134 | 135 | return [cleanStr, fileName, len(masterStr.lower().split()), len(cleanList), c.most_common(10)] 136 | 137 | 138 | def generateWordcloud(wordStr, outputFile): 139 | 140 | themeList = {} 141 | theme01 = {} 142 | theme01["bgColor"] = "#341c01" 143 | theme01["colorList"] = ["#fffff0", "#d0aa3a", "#cea92e", "#c1762e", "#aea764", "#d59733", "#e9e3cd"] 144 | themeList["theme01"] = theme01 145 | 146 | theme02 = {} 147 | theme02["bgColor"] = "#fff" 148 | theme02["colorList"] = ["#03318c", "#021f59", "#61a2ca", "#30588c", "#32628c"] 149 | themeList["theme02"] = theme02 150 | 151 | theme03 = {} 152 | theme03["bgColor"] = "#223564" 153 | theme03["colorList"] = ["#f7e4be", "#f0f4bc", "#9a80a4", "#848da6"] 154 | themeList["theme03"] = theme03 155 | 156 | theme04 = {} 157 | theme04["bgColor"] = "#091c2b" 158 | theme04["colorList"] = ["#edecf2", "#c1d4f2", "#6d98ba", "#3669a2", "#8793dd"] 159 | themeList["theme04"] = theme04 160 | 161 | theme05 = {} 162 | theme05["bgColor"] = "#000" 163 | theme05["colorList"] = ["#b95c28", "#638db2", "#f0f0f0", "#dbcc58", "#1b3c69", "#d5a753"] 164 | themeList["theme05"] = theme05 165 | 166 | theme06 = {} 167 | theme06["bgColor"] = "#262626" 168 | theme06["colorList"] = ["#468966", "#fff0a5", "#ffb03b", "#b64926", "#8e2800"] 169 | themeList["theme06"] = theme06 170 | 171 | 172 | theme07 = {} 173 | theme07["bgColor"] = "#fff" 174 | theme07["colorList"] = ["#438D9C", "#E8A664", "#9C6043", "#171717", "#c00000"] 175 | themeList["theme07"] = theme07 176 | #colorList = ["#d35400", "#c0392b", "#e74c3c", "#e67e22", "#f39c12"] 177 | #colorList = ["#f39c12", "#e67e22", "#e74c3c", "#c0392b", "#d35400"] 178 | 179 | liveTheme = "theme07" 180 | bgColor = themeList[liveTheme]["bgColor"] 181 | colorList = themeList[liveTheme]["colorList"] 182 | 183 | #colorList = ["#f1e3be", "#f1f3be", "#927fa1", "#858ca4"] 184 | 185 | colorMap = LinearSegmentedColormap.from_list("mycmap", colorList) 186 | 187 | fontPath = os.path.join(dirPath, "fonts/LilitaOne-Regular.ttf") 188 | 189 | wordcloudObj = WordCloud( 190 | font_path = fontPath, 191 | mode = "RGBA", 192 | width = 1200, 193 | height = 852, 194 | margin = 16, 195 | random_state = 0, 196 | background_color = bgColor, 197 | normalize_plurals = True, 198 | colormap = colorMap 199 | ).generate(wordStr) 200 | 201 | 202 | #wordcloudObj.recolor( 203 | # color_func = recolorBlue, 204 | # random_state = 5 205 | #) 206 | 207 | smallerImg = imresize(wordcloudObj, [382, 538]) 208 | imsave(outputFile, smallerImg) 209 | 210 | 211 | def gcsUpload(globalId, orgIdentifier, fileName, filePath): 212 | cloudPath = "accounts/" + orgIdentifier + "/enrichments/" + str(globalId) + "/wordclouds/" + fileName 213 | clientObj = storage.Client.from_service_account_json(service_account_path) 214 | bucketObj = clientObj.get_bucket(bucketName) 215 | blobObj = bucketObj.blob(cloudPath) 216 | 217 | metadataStr = "inline; filename='%s'" % fileName 218 | blobObj.content_disposition = metadataStr 219 | 220 | blobObj.upload_from_filename(filePath) 221 | blobObj.make_public() 222 | 223 | return blobObj.public_url 224 | 225 | 226 | def assignUrl(globalId, wcUrl): 227 | reqUrl = utility_service_url + "/idWordcloud" 228 | payloadObj = { 229 | "gId": globalId, 230 | "wcUrl": wcUrl 231 | } 232 | responseObj = requests.get(reqUrl, params=payloadObj) 233 | respTxt = responseObj.text 234 | 235 | return respTxt 236 | 237 | 238 | def lookupMeeting(globalId): 239 | reqUrl = utility_service_url + "/meetingDetails" 240 | payloadObj = { 241 | "gId": globalId 242 | } 243 | responseObj = requests.get(reqUrl, params=payloadObj) 244 | respTxt = responseObj.text 245 | jsonObj = json.loads(respTxt) 246 | 247 | return jsonObj["prodTranscript"], jsonObj["orgIdentifier"] 248 | 249 | 250 | def dispatchWorker(ackId, globalId): 251 | successFlag = None 252 | try: 253 | print ".. creating word cloud for meeting: " + str(globalId) 254 | prodTranscript, orgIdentifier = lookupMeeting(globalId) 255 | print "... word cloud will be made from transcript: " + str(prodTranscript) 256 | 257 | epochTime = calendar.timegm(time.gmtime()) 258 | outputFile = str(epochTime) + "-" + str(globalId) + "-wordcloud-538-by-382.png" 259 | filePath = os.path.join(dirPath, outputFile) 260 | 261 | outputList = generateStr(globalId, orgIdentifier, prodTranscript) 262 | print "... pulling words from file: " + outputList[1] 263 | print ".... " + str(outputList[2]) + " words identified" 264 | if outputList[2] > 0: 265 | print ".... " + str(outputList[3]) + " significant words identified" 266 | print ".... ten most common words: " 267 | for eachEntry in outputList[4]: 268 | print "..... " + str(eachEntry) 269 | wordStr = outputList[0] 270 | generateWordcloud(wordStr, filePath) 271 | 272 | wcUrl = gcsUpload(globalId, orgIdentifier, outputFile, filePath) 273 | os.remove(filePath) 274 | print "... word cloud URL: " + wcUrl 275 | print "... URL assigned in database: " + str(assignUrl(globalId, wcUrl)) 276 | successFlag = True 277 | else: 278 | print ".... stopping now because no words were identified" 279 | successFlag = False 280 | print acknowledgeMsg(ackId) 281 | except Exception as e: 282 | print acknowledgeMsg(ackId) 283 | print "skip " + e.message 284 | successFlag = False 285 | 286 | return successFlag 287 | 288 | def acknowledgeMsg(ackId): 289 | postPayload = { 290 | "ackIds": [ackId] 291 | } 292 | subStr = "projects/%s/subscriptions/%s" % (projectId, subName) 293 | reqUrl = "https://pubsub.googleapis.com/v1/%s:acknowledge" % subStr 294 | psMsg = psCall(reqUrl, postPayload) 295 | 296 | return "... Pubsub message acknowledged" 297 | 298 | 299 | def nextAction(globalId): 300 | reqUrl = utility_service_url + "/msgPublish" 301 | payloadObj = { 302 | "msgAction": "index", 303 | "topicName": "indexQueue", 304 | "gId": globalId 305 | } 306 | responseObj = requests.get( 307 | reqUrl, 308 | params = payloadObj 309 | ) 310 | respTxt = responseObj.text 311 | 312 | return respTxt 313 | 314 | 315 | def main(): 316 | #globalId = 2729 317 | #runWorkflow(globalId) 318 | 319 | postPayload = { 320 | "returnImmediately": True, 321 | "maxMessages": 1 322 | } 323 | subStr = "projects/%s/subscriptions/%s" % (projectId, subName) 324 | reqUrl = "https://pubsub.googleapis.com/v1/%s:pull" % subStr 325 | 326 | while True: 327 | psMsg = psCall(reqUrl, postPayload) 328 | try: 329 | jsonObj = json.loads(psMsg) 330 | msgType = base64.b64decode(jsonObj["receivedMessages"][0]["message"]["data"]) 331 | print "Message Received. Type = '%s'" % str(msgType) 332 | ackId = jsonObj["receivedMessages"][0]["ackId"] 333 | globalId = jsonObj["receivedMessages"][0]["message"]["attributes"]["globalId"] 334 | print ackId 335 | print globalId 336 | successFlag = dispatchWorker(ackId, globalId) 337 | if successFlag == True: 338 | pass 339 | print nextAction(globalId) 340 | else: 341 | print ".. not initiating next action" 342 | except: 343 | pass 344 | time.sleep(1) 345 | 346 | 347 | if __name__ == "__main__": 348 | main() 349 | -------------------------------------------------------------------------------- /publish-pdf-transcript/worker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is not an officially supported Google product, though support 4 | # will be provided on a best-effort basis. 5 | 6 | # Copyright 2018 Google LLC 7 | 8 | # Licensed under the Apache License, Version 2.0 (the "License"); you 9 | # may not use this file except in compliance with the License. 10 | 11 | # You may obtain a copy of the License at: 12 | 13 | # https://www.apache.org/licenses/LICENSE-2.0 14 | 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | 22 | import os 23 | import json 24 | import time 25 | import pdfkit 26 | import base64 27 | import requests 28 | from datetime import datetime 29 | from google.cloud import pubsub 30 | from google.cloud import storage 31 | from oauth2client.service_account import ServiceAccountCredentials 32 | 33 | service_account_json = "__Credential_JSON_File_Name__" 34 | dirPath = os.path.normpath(os.getcwd()) 35 | service_account_path = os.path.join(dirPath, service_account_json) 36 | 37 | projectId = "__GCP_Project_ID__" 38 | topicName = "publish-transcript-queue" 39 | subName = "publish-transcript-subscription" 40 | 41 | bucketName = "__GCS_Storage_Bucket_Name__" 42 | utility_service_url = "__Utility_Service_URL__" 43 | 44 | 45 | psClient = pubsub.SubscriberClient() 46 | 47 | topicPath = psClient.topic_path( 48 | projectId, 49 | topicName 50 | ) 51 | 52 | subPath = psClient.subscription_path( 53 | projectId, 54 | subName 55 | ) 56 | 57 | subObj = psClient.subscribe( 58 | subPath 59 | ) 60 | 61 | 62 | def mkPdf(inputFile): 63 | outputFile = inputFile.replace(".html", ".pdf") 64 | optionsObj = { 65 | "page-size": "Letter", 66 | "margin-top": "0.75in", 67 | "margin-right": "0.75in", 68 | "margin-bottom": "1.00in", 69 | "margin-left": "0.75in", 70 | "footer-center": "page [page]/[topage]", 71 | "footer-font-name": "Roboto", 72 | "footer-font-size": "8", 73 | "footer-spacing": "10" 74 | } 75 | pdfkit.from_file( 76 | inputFile, outputFile, 77 | options=optionsObj 78 | ) 79 | 80 | return outputFile 81 | 82 | 83 | def lookupMeeting(globalId): 84 | reqUrl = utility_service_url + "/meetingDetails" 85 | payloadObj = { 86 | "gId": globalId 87 | } 88 | responseObj = requests.get(reqUrl, params=payloadObj) 89 | respTxt = responseObj.text 90 | jsonObj = json.loads(respTxt) 91 | 92 | return jsonObj["prodTranscript"], jsonObj["meetingDate"], jsonObj["orgIdentifier"] 93 | 94 | 95 | def formatDate(meetingDate): 96 | datetimeObj = datetime.strptime(meetingDate, "%Y%m%d") 97 | formattedDate = datetimeObj.strftime("%B %d, %Y") 98 | weekDay = datetimeObj.strftime("%A") 99 | 100 | return formattedDate, weekDay 101 | 102 | 103 | def meetingDetails(globalId): 104 | reqUrl = utility_service_url + "/meetingDetails" 105 | payloadObj = { 106 | "gId": globalId 107 | } 108 | responseObj = requests.get(reqUrl, params=payloadObj) 109 | respTxt = responseObj.text 110 | jsonObj = json.loads(respTxt) 111 | 112 | formattedDate, weekDay = formatDate(jsonObj["meetingDate"]) 113 | 114 | return jsonObj["meetingDesc"], formattedDate, weekDay 115 | 116 | 117 | def gcsUpload(globalId, orgIdentifier, fileName, filePath): 118 | cloudPath = "accounts/" + orgIdentifier + "/enrichments/" + str(globalId) + "/transcripts/" + fileName 119 | clientObj = storage.Client.from_service_account_json(service_account_path) 120 | bucketObj = clientObj.get_bucket(bucketName) 121 | blobObj = bucketObj.blob(cloudPath) 122 | 123 | metadataStr = "inline; filename='%s'" % fileName 124 | blobObj.content_disposition = metadataStr 125 | 126 | blobObj.upload_from_filename(filePath) 127 | blobObj.make_public() 128 | 129 | return blobObj.public_url 130 | 131 | 132 | def assignUrl(globalId, transcriptUrl): 133 | reqUrl = utility_service_url + "/idTranscript" 134 | payloadObj = { 135 | "gId": globalId, 136 | "transcriptUrl": transcriptUrl 137 | } 138 | responseObj = requests.get(reqUrl, params=payloadObj) 139 | respTxt = responseObj.text 140 | 141 | return respTxt 142 | 143 | 144 | def runCycle(globalId, orgIdentifier, prodTranscript): 145 | basePath = "accounts/" + orgIdentifier + "/enrichments/" + str(globalId) + "/transcripts/" 146 | cloudPath = basePath + str(prodTranscript) + "/" 147 | clientObj = storage.Client.from_service_account_json(service_account_path) 148 | bucketObj = clientObj.get_bucket(bucketName) 149 | listObj = bucketObj.list_blobs(prefix=cloudPath) 150 | print "!!! length of listObj: " + str(listObj) 151 | transcriptList = [] 152 | for eachEntry in listObj: 153 | if ".json" in eachEntry.name: 154 | transcriptList.append(str(eachEntry.name)) 155 | 156 | fileCnt = 0 157 | 158 | htmlStr = "" 159 | for eachFile in sorted(transcriptList, reverse=False): 160 | fileCnt += 1 161 | blobObj = bucketObj.get_blob(eachFile) 162 | blobStr = blobObj.download_as_string() 163 | jsonObj = json.loads(blobStr) 164 | 165 | if "response" in jsonObj: 166 | if "results" in jsonObj["response"]: 167 | for eachAlt in jsonObj["response"]["results"]: 168 | tmpStr = "" 169 | timeVal = None 170 | if "alternatives" in eachAlt: 171 | timeVal = eachAlt["alternatives"][0]["words"][0]["startTime"] 172 | timeVal = timeVal.replace("s", "") 173 | timeVal = float(timeVal) 174 | if fileCnt > 1: 175 | timeVal = timeVal + ((fileCnt - 1) * 10800) 176 | displayTime = time.strftime("%H:%M:%S", time.gmtime(timeVal)) 177 | transcriptStr = eachAlt["alternatives"][0]["transcript"] 178 | transcriptStr = transcriptStr.strip() 179 | transcriptStr = transcriptStr.replace("Lewisville", "Louisville") 180 | transcriptStr = transcriptStr.replace("Pro stack", "PROSTAC") 181 | transcriptStr = transcriptStr.replace("pro stack", "PROSTAC") 182 | transcriptStr = transcriptStr.replace("Pro Stacks", "PROSTAC") 183 | transcriptStr = transcriptStr.replace("pro Strat", "PROSTAC") 184 | transcriptStr = transcriptStr.replace("pro-sex", "PROSTAC") 185 | htmlStr += "

%s

" % displayTime 186 | htmlStr += """

%s

""" % transcriptStr 187 | 188 | #newPath = basePath + "rawTxt-" + str(globalId) + "-" + exportType + "-" + str(prodTranscript) + ".txt" 189 | ##newPath = basePath + fileName 190 | ##blobObj = bucketObj.blob(newPath) 191 | ##blobObj.upload_from_string(masterStr.strip()) 192 | 193 | return htmlStr 194 | 195 | 196 | def mkTranscript(globalId, municipality_display_name, municipality_short_name): 197 | prodTranscript, rawDate, orgIdentifier = lookupMeeting(globalId) 198 | meetingDesc, meetingDate, weekDay = meetingDetails(globalId) 199 | 200 | 201 | htmlStr = """ 202 | 203 | 204 | 240 | 241 | 242 | """ 243 | 244 | htmlStr += "
" 245 | htmlStr += "

" + municipality_display_name + "

" 246 | htmlStr += "

%s

" % meetingDesc 247 | htmlStr += "

%s, %s

" % (weekDay, meetingDate) 248 | htmlStr += "
" 249 | htmlStr += "--------------------------------------------------------------------------------------------------------------------------------------------------------" 250 | htmlStr += "
" 251 | htmlStr += "This transcript was generated automatically using cutting edge speech-to-text technology, which isn't yet on par with human transcription. This transcript may not accurately reflect the contents of the meeting proceedings." 252 | htmlStr += "
" 253 | htmlStr += "--------------------------------------------------------------------------------------------------------------------------------------------------------" 254 | htmlStr += "
" 255 | htmlStr += "

" 256 | 257 | htmlStr += runCycle(globalId, orgIdentifier, prodTranscript) 258 | 259 | htmlStr += """ 260 | 261 | 262 | """ 263 | meetingDesc = meetingDesc.replace(" ", "-") 264 | meetingDesc = meetingDesc.replace("/", "-") 265 | meetingDesc = meetingDesc.replace(",", "") 266 | fileName = municipality_short_name + "-" + str(rawDate) + "-" + meetingDesc + "-Transcript.html" 267 | 268 | htmlPath = os.path.join(dirPath, fileName) 269 | htmlFile = open(htmlPath, "w") 270 | htmlFile.write(htmlStr.encode("ascii", "ignore")) 271 | htmlFile.close() 272 | 273 | pdfName = mkPdf(fileName) 274 | print "pdfName: " + pdfName 275 | pdfPath = os.path.join(dirPath, pdfName) 276 | print "pdfPath: " + pdfPath 277 | transcriptUrl = gcsUpload(globalId, orgIdentifier, pdfName, pdfPath) 278 | print assignUrl(globalId, transcriptUrl) 279 | os.remove(htmlPath) 280 | os.remove(pdfPath) 281 | 282 | 283 | def psCall(reqUrl, postPayload): 284 | scopesList = ["https://www.googleapis.com/auth/cloud-platform"] 285 | credentialsObj = ServiceAccountCredentials.from_json_keyfile_name( 286 | service_account_path, 287 | scopes = scopesList 288 | ) 289 | 290 | accessToken = "Bearer %s" % credentialsObj.get_access_token().access_token 291 | headerObj = { 292 | "authorization": accessToken, 293 | } 294 | 295 | reqObj = requests.post( 296 | reqUrl, 297 | data = json.dumps(postPayload), 298 | headers = headerObj 299 | ) 300 | 301 | return reqObj.text 302 | 303 | 304 | def acknowledgeMsg(ackId): 305 | postPayload = { 306 | "ackIds": [ackId] 307 | } 308 | subStr = "projects/%s/subscriptions/%s" % (projectId, subName) 309 | reqUrl = "https://pubsub.googleapis.com/v1/%s:acknowledge" % subStr 310 | psMsg = psCall(reqUrl, postPayload) 311 | 312 | return "... Pubsub message acknowledged" 313 | 314 | 315 | def dispatchWorker(ackId, globalId, municipality_display_name, municipality_short_name): 316 | try: 317 | mkTranscript(globalId, municipality_display_name, municipality_short_name) 318 | print acknowledgeMsg(ackId) 319 | except Exception as e: 320 | print "erroring out" 321 | print acknowledgeMsg(ackId) 322 | print "skip " + e.message 323 | 324 | 325 | def main(): 326 | postPayload = { 327 | "returnImmediately": True, 328 | "maxMessages": 1 329 | } 330 | subStr = "projects/%s/subscriptions/%s" % (projectId, subName) 331 | reqUrl = "https://pubsub.googleapis.com/v1/%s:pull" % subStr 332 | 333 | 334 | municipality_display_name = "" 335 | municipality_short_name = "" 336 | 337 | 338 | while True: 339 | psMsg = psCall(reqUrl, postPayload) 340 | try: 341 | jsonObj = json.loads(psMsg) 342 | msgType = base64.b64decode(jsonObj["receivedMessages"][0]["message"]["data"]) 343 | print "Message Received. Type = '%s'" % str(msgType) 344 | ackId = jsonObj["receivedMessages"][0]["ackId"] 345 | globalId = jsonObj["receivedMessages"][0]["message"]["attributes"]["globalId"] 346 | print ackId 347 | print globalId 348 | dispatchWorker(ackId, globalId, municipality_display_name, municipality_short_name) 349 | except: 350 | pass 351 | time.sleep(10) 352 | 353 | 354 | if __name__ == "__main__": 355 | main() -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /generate-wordcloud/stopwords-20180109-133115.json: -------------------------------------------------------------------------------- 1 | [{"word":"um","wordCnt":"0"}, 2 | {"word":"perhaps","wordCnt":"0"}, 3 | {"word":"uh","wordCnt":"0"}, 4 | {"word":"gonna","wordCnt":"0"}, 5 | {"word":"the","wordCnt":"367405"}, 6 | {"word":"to","wordCnt":"264006"}, 7 | {"word":"and","wordCnt":"213023"}, 8 | {"word":"that","wordCnt":"196829"}, 9 | {"word":"i","wordCnt":"162505"}, 10 | {"word":"of","wordCnt":"161428"}, 11 | {"word":"a","wordCnt":"149402"}, 12 | {"word":"we","wordCnt":"125275"}, 13 | {"word":"you","wordCnt":"115105"}, 14 | {"word":"in","wordCnt":"111799"}, 15 | {"word":"it","wordCnt":"101551"}, 16 | {"word":"is","wordCnt":"99624"}, 17 | {"word":"so","wordCnt":"88983"}, 18 | {"word":"for","wordCnt":"79291"}, 19 | {"word":"have","wordCnt":"72426"}, 20 | {"word":"this","wordCnt":"70247"}, 21 | {"word":"on","wordCnt":"69061"}, 22 | {"word":"be","wordCnt":"61657"}, 23 | {"word":"with","wordCnt":"49022"}, 24 | {"word":"but","wordCnt":"47697"}, 25 | {"word":"was","wordCnt":"44927"}, 26 | {"word":"if","wordCnt":"44165"}, 27 | {"word":"what","wordCnt":"42757"}, 28 | {"word":"just","wordCnt":"42427"}, 29 | {"word":"at","wordCnt":"42057"}, 30 | {"word":"it's","wordCnt":"41219"}, 31 | {"word":"know","wordCnt":"41177"}, 32 | {"word":"are","wordCnt":"40908"}, 33 | {"word":"there","wordCnt":"38731"}, 34 | {"word":"not","wordCnt":"38645"}, 35 | {"word":"think","wordCnt":"38467"}, 36 | {"word":"or","wordCnt":"36681"}, 37 | {"word":"would","wordCnt":"36201"}, 38 | {"word":"do","wordCnt":"35834"}, 39 | {"word":"as","wordCnt":"35536"}, 40 | {"word":"going","wordCnt":"35489"}, 41 | {"word":"they","wordCnt":"34610"}, 42 | {"word":"can","wordCnt":"34294"}, 43 | {"word":"like","wordCnt":"33358"}, 44 | {"word":"about","wordCnt":"31652"}, 45 | {"word":"that's","wordCnt":"30796"}, 46 | {"word":"don't","wordCnt":"27966"}, 47 | {"word":"some","wordCnt":"27036"}, 48 | {"word":"all","wordCnt":"26997"}, 49 | {"word":"then","wordCnt":"26995"}, 50 | {"word":"i'm","wordCnt":"26302"}, 51 | {"word":"one","wordCnt":"25955"}, 52 | {"word":"up","wordCnt":"25853"}, 53 | {"word":"out","wordCnt":"24804"}, 54 | {"word":"town","wordCnt":"24348"}, 55 | {"word":"from","wordCnt":"23993"}, 56 | {"word":"get","wordCnt":"23265"}, 57 | {"word":"here","wordCnt":"22785"}, 58 | {"word":"want","wordCnt":"22053"}, 59 | {"word":"will","wordCnt":"20141"}, 60 | {"word":"an","wordCnt":"19152"}, 61 | {"word":"go","wordCnt":"19058"}, 62 | {"word":"right","wordCnt":"18104"}, 63 | {"word":"more","wordCnt":"17476"}, 64 | {"word":"really","wordCnt":"17392"}, 65 | {"word":"when","wordCnt":"17171"}, 66 | {"word":"our","wordCnt":"16986"}, 67 | {"word":"we're","wordCnt":"16766"}, 68 | {"word":"your","wordCnt":"16636"}, 69 | {"word":"how","wordCnt":"16605"}, 70 | {"word":"see","wordCnt":"16381"}, 71 | {"word":"were","wordCnt":"15923"}, 72 | {"word":"because","wordCnt":"15764"}, 73 | {"word":"there's","wordCnt":"15583"}, 74 | {"word":"those","wordCnt":"15539"}, 75 | {"word":"time","wordCnt":"15449"}, 76 | {"word":"other","wordCnt":"15385"}, 77 | {"word":"me","wordCnt":"15144"}, 78 | {"word":"my","wordCnt":"14952"}, 79 | {"word":"now","wordCnt":"14751"}, 80 | {"word":"where","wordCnt":"14647"}, 81 | {"word":"had","wordCnt":"14395"}, 82 | {"word":"people","wordCnt":"14209"}, 83 | {"word":"by","wordCnt":"14030"}, 84 | {"word":"make","wordCnt":"13806"}, 85 | {"word":"been","wordCnt":"13494"}, 86 | {"word":"kind","wordCnt":"13399"}, 87 | {"word":"them","wordCnt":"13304"}, 88 | {"word":"board","wordCnt":"13177"}, 89 | {"word":"something","wordCnt":"13177"}, 90 | {"word":"could","wordCnt":"13059"}, 91 | {"word":"any","wordCnt":"13054"}, 92 | {"word":"lot","wordCnt":"12975"}, 93 | {"word":"back","wordCnt":"12689"}, 94 | {"word":"well","wordCnt":"12345"}, 95 | {"word":"need","wordCnt":"12147"}, 96 | {"word":"us","wordCnt":"12125"}, 97 | {"word":"very","wordCnt":"12065"}, 98 | {"word":"their","wordCnt":"11768"}, 99 | {"word":"which","wordCnt":"11697"}, 100 | {"word":"come","wordCnt":"11604"}, 101 | {"word":"way","wordCnt":"11314"}, 102 | {"word":"over","wordCnt":"11277"}, 103 | {"word":"has","wordCnt":"11243"}, 104 | {"word":"no","wordCnt":"11136"}, 105 | {"word":"things","wordCnt":"10915"}, 106 | {"word":"you're","wordCnt":"10772"}, 107 | {"word":"work","wordCnt":"10757"}, 108 | {"word":"also","wordCnt":"10678"}, 109 | {"word":"down","wordCnt":"10661"}, 110 | {"word":"year","wordCnt":"10609"}, 111 | {"word":"got","wordCnt":"10392"}, 112 | {"word":"these","wordCnt":"10348"}, 113 | {"word":"plan","wordCnt":"10077"}, 114 | {"word":"meeting","wordCnt":"10016"}, 115 | {"word":"say","wordCnt":"9968"}, 116 | {"word":"into","wordCnt":"9946"}, 117 | {"word":"put","wordCnt":"9794"}, 118 | {"word":"through","wordCnt":"9779"}, 119 | {"word":"little","wordCnt":"9736"}, 120 | {"word":"actually","wordCnt":"9654"}, 121 | {"word":"two","wordCnt":"9247"}, 122 | {"word":"okay","wordCnt":"9076"}, 123 | {"word":"look","wordCnt":"9057"}, 124 | {"word":"superior","wordCnt":"9018"}, 125 | {"word":"good","wordCnt":"8953"}, 126 | {"word":"did","wordCnt":"8950"}, 127 | {"word":"they're","wordCnt":"8944"}, 128 | {"word":"mean","wordCnt":"8935"}, 129 | {"word":"maybe","wordCnt":"8640"}, 130 | {"word":"next","wordCnt":"8556"}, 131 | {"word":"last","wordCnt":"8550"}, 132 | {"word":"space","wordCnt":"8489"}, 133 | {"word":"should","wordCnt":"8314"}, 134 | {"word":"much","wordCnt":"8263"}, 135 | {"word":"public","wordCnt":"8162"}, 136 | {"word":"open","wordCnt":"8014"}, 137 | {"word":"sure","wordCnt":"7880"}, 138 | {"word":"area","wordCnt":"7880"}, 139 | {"word":"park","wordCnt":"7860"}, 140 | {"word":"said","wordCnt":"7829"}, 141 | {"word":"than","wordCnt":"7803"}, 142 | {"word":"yeah","wordCnt":"7718"}, 143 | {"word":"thing","wordCnt":"7607"}, 144 | {"word":"years","wordCnt":"7587"}, 145 | {"word":"community","wordCnt":"7342"}, 146 | {"word":"take","wordCnt":"7328"}, 147 | {"word":"first","wordCnt":"7292"}, 148 | {"word":"number","wordCnt":"7292"}, 149 | {"word":"thank","wordCnt":"7095"}, 150 | {"word":"part","wordCnt":"7087"}, 151 | {"word":"property","wordCnt":"6846"}, 152 | {"word":"center","wordCnt":"6792"}, 153 | {"word":"th","wordCnt":"6683"}, 154 | {"word":"great","wordCnt":"6662"}, 155 | {"word":"why","wordCnt":"6637"}, 156 | {"word":"street","wordCnt":"6593"}, 157 | {"word":"before","wordCnt":"6556"}, 158 | {"word":"off","wordCnt":"6424"}, 159 | {"word":"probably","wordCnt":"6410"}, 160 | {"word":"again","wordCnt":"6369"}, 161 | {"word":"different","wordCnt":"6334"}, 162 | {"word":"around","wordCnt":"6306"}, 163 | {"word":"point","wordCnt":"6299"}, 164 | {"word":"we've","wordCnt":"6235"}, 165 | {"word":"done","wordCnt":"6228"}, 166 | {"word":"doing","wordCnt":"6172"}, 167 | {"word":"use","wordCnt":"6138"}, 168 | {"word":"parking","wordCnt":"6055"}, 169 | {"word":"only","wordCnt":"6040"}, 170 | {"word":"question","wordCnt":"6022"}, 171 | {"word":"building","wordCnt":"5948"}, 172 | {"word":"might","wordCnt":"5876"}, 173 | {"word":"cuz","wordCnt":"5853"}, 174 | {"word":"coming","wordCnt":"5806"}, 175 | {"word":"he","wordCnt":"5765"}, 176 | {"word":"still","wordCnt":"5713"}, 177 | {"word":"being","wordCnt":"5712"}, 178 | {"word":"new","wordCnt":"5694"}, 179 | {"word":"who","wordCnt":"5684"}, 180 | {"word":"looking","wordCnt":"5648"}, 181 | {"word":"project","wordCnt":"5642"}, 182 | {"word":"does","wordCnt":"5575"}, 183 | {"word":"bit","wordCnt":"5534"}, 184 | {"word":"anything","wordCnt":"5519"}, 185 | {"word":"creek","wordCnt":"5453"}, 186 | {"word":"even","wordCnt":"5420"}, 187 | {"word":"development","wordCnt":"5397"}, 188 | {"word":"didn't","wordCnt":"5308"}, 189 | {"word":"discussion","wordCnt":"5209"}, 190 | {"word":"same","wordCnt":"5184"}, 191 | {"word":"staff","wordCnt":"5175"}, 192 | {"word":"water","wordCnt":"5102"}, 193 | {"word":"process","wordCnt":"5098"}, 194 | {"word":"traffic","wordCnt":"5074"}, 195 | {"word":"wanted","wordCnt":"5005"}, 196 | {"word":"questions","wordCnt":"4988"}, 197 | {"word":"another","wordCnt":"4929"}, 198 | {"word":"what's","wordCnt":"4907"}, 199 | {"word":"i'll","wordCnt":"4888"}, 200 | {"word":"side","wordCnt":"4820"}, 201 | {"word":"trying","wordCnt":"4789"}, 202 | {"word":"talk","wordCnt":"4760"}, 203 | {"word":"having","wordCnt":"4749"}, 204 | {"word":"start","wordCnt":"4684"}, 205 | {"word":"day","wordCnt":"4636"}, 206 | {"word":"may","wordCnt":"4593"}, 207 | {"word":"yes","wordCnt":"4588"}, 208 | {"word":"i've","wordCnt":"4561"}, 209 | {"word":"give","wordCnt":"4509"}, 210 | {"word":"three","wordCnt":"4503"}, 211 | {"word":"talking","wordCnt":"4481"}, 212 | {"word":"can't","wordCnt":"4477"}, 213 | {"word":"road","wordCnt":"4423"}, 214 | {"word":"her","wordCnt":"4393"}, 215 | {"word":"couple","wordCnt":"4392"}, 216 | {"word":"doesn't","wordCnt":"4367"}, 217 | {"word":"guess","wordCnt":"4331"}, 218 | {"word":"cost","wordCnt":"4308"}, 219 | {"word":"thought","wordCnt":"4292"}, 220 | {"word":"end","wordCnt":"4274"}, 221 | {"word":"able","wordCnt":"4265"}, 222 | {"word":"getting","wordCnt":"4253"}, 223 | {"word":"whatever","wordCnt":"4230"}, 224 | {"word":"too","wordCnt":"4215"}, 225 | {"word":"many","wordCnt":"4207"}, 226 | {"word":"issue","wordCnt":"4201"}, 227 | {"word":"boulder","wordCnt":"4200"}, 228 | {"word":"money","wordCnt":"4178"}, 229 | {"word":"planning","wordCnt":"4177"}, 230 | {"word":"try","wordCnt":"4058"}, 231 | {"word":"trail","wordCnt":"4053"}, 232 | {"word":"committee","wordCnt":"4047"}, 233 | {"word":"long","wordCnt":"4046"}, 234 | {"word":"big","wordCnt":"4014"}, 235 | {"word":"change","wordCnt":"3973"}, 236 | {"word":"design","wordCnt":"3966"}, 237 | {"word":"feel","wordCnt":"3927"}, 238 | {"word":"his","wordCnt":"3918"}, 239 | {"word":"else","wordCnt":"3876"}, 240 | {"word":"she","wordCnt":"3857"}, 241 | {"word":"place","wordCnt":"3853"}, 242 | {"word":"pretty","wordCnt":"3847"}, 243 | {"word":"after","wordCnt":"3846"}, 244 | {"word":"already","wordCnt":"3829"}, 245 | {"word":"forward","wordCnt":"3823"}, 246 | {"word":"most","wordCnt":"3815"}, 247 | {"word":"guys","wordCnt":"3771"}, 248 | {"word":"him","wordCnt":"3770"}, 249 | {"word":"county","wordCnt":"3751"}, 250 | {"word":"saying","wordCnt":"3688"}, 251 | {"word":"tonight","wordCnt":"3667"}, 252 | {"word":"idea","wordCnt":"3598"}, 253 | {"word":"budget","wordCnt":"3588"}, 254 | {"word":"whole","wordCnt":"3547"}, 255 | {"word":"working","wordCnt":"3526"}, 256 | {"word":"between","wordCnt":"3494"}, 257 | {"word":"ask","wordCnt":"3484"}, 258 | {"word":"commission","wordCnt":"3478"}, 259 | {"word":"better","wordCnt":"3465"}, 260 | {"word":"every","wordCnt":"3442"}, 261 | {"word":"feet","wordCnt":"3376"}, 262 | {"word":"move","wordCnt":"3349"}, 263 | {"word":"issues","wordCnt":"3342"}, 264 | {"word":"went","wordCnt":"3318"}, 265 | {"word":"find","wordCnt":"3314"}, 266 | {"word":"understand","wordCnt":"3293"}, 267 | {"word":"within","wordCnt":"3284"}, 268 | {"word":"bring","wordCnt":"3260"}, 269 | {"word":"call","wordCnt":"3237"}, 270 | {"word":"second","wordCnt":"3218"}, 271 | {"word":"far","wordCnt":"3215"}, 272 | {"word":"always","wordCnt":"3198"}, 273 | {"word":"stuff","wordCnt":"3198"}, 274 | {"word":"school","wordCnt":"3178"}, 275 | {"word":"keep","wordCnt":"3177"}, 276 | {"word":"made","wordCnt":"3174"}, 277 | {"word":"help","wordCnt":"3156"}, 278 | {"word":"comment","wordCnt":"3143"}, 279 | {"word":"i'd","wordCnt":"3141"}, 280 | {"word":"agenda","wordCnt":"3128"}, 281 | {"word":"both","wordCnt":"3082"}, 282 | {"word":"residential","wordCnt":"3061"}, 283 | {"word":"let","wordCnt":"3047"}, 284 | {"word":"information","wordCnt":"3016"}, 285 | {"word":"additional","wordCnt":"3011"}, 286 | {"word":"week","wordCnt":"2975"}, 287 | {"word":"everything","wordCnt":"2944"}, 288 | {"word":"came","wordCnt":"2914"}, 289 | {"word":"believe","wordCnt":"2897"}, 290 | {"word":"line","wordCnt":"2888"}, 291 | {"word":"drive","wordCnt":"2876"}, 292 | {"word":"group","wordCnt":"2876"}, 293 | {"word":"comments","wordCnt":"2866"}, 294 | {"word":"agreement","wordCnt":"2851"}, 295 | {"word":"whether","wordCnt":"2848"}, 296 | {"word":"let's","wordCnt":"2847"}, 297 | {"word":"talked","wordCnt":"2843"}, 298 | {"word":"home","wordCnt":"2831"}, 299 | {"word":"south","wordCnt":"2802"}, 300 | {"word":"together","wordCnt":"2775"}, 301 | {"word":"original","wordCnt":"2771"}, 302 | {"word":"everybody","wordCnt":"2748"}, 303 | {"word":"along","wordCnt":"2746"}, 304 | {"word":"under","wordCnt":"2736"}, 305 | {"word":"million","wordCnt":"2736"}, 306 | {"word":"am","wordCnt":"2727"}, 307 | {"word":"sign","wordCnt":"2713"}, 308 | {"word":"least","wordCnt":"2708"}, 309 | {"word":"house","wordCnt":"2697"}, 310 | {"word":"tell","wordCnt":"2696"}, 311 | {"word":"service","wordCnt":"2696"}, 312 | {"word":"never","wordCnt":"2692"}, 313 | {"word":"office","wordCnt":"2683"}, 314 | {"word":"important","wordCnt":"2680"}, 315 | {"word":"district","wordCnt":"2678"}, 316 | {"word":"hear","wordCnt":"2656"}, 317 | {"word":"program","wordCnt":"2652"}, 318 | {"word":"goes","wordCnt":"2649"}, 319 | {"word":"away","wordCnt":"2640"}, 320 | {"word":"since","wordCnt":"2627"}, 321 | {"word":"residents","wordCnt":"2607"}, 322 | {"word":"trails","wordCnt":"2606"}, 323 | {"word":"show","wordCnt":"2603"}, 324 | {"word":"kids","wordCnt":"2578"}, 325 | {"word":"site","wordCnt":"2575"}, 326 | {"word":"problem","wordCnt":"2564"}, 327 | {"word":"square","wordCnt":"2556"}, 328 | {"word":"build","wordCnt":"2549"}, 329 | {"word":"today","wordCnt":"2547"}, 330 | {"word":"opportunity","wordCnt":"2546"}, 331 | {"word":"needs","wordCnt":"2540"}, 332 | {"word":"access","wordCnt":"2540"}, 333 | {"word":"construction","wordCnt":"2537"}, 334 | {"word":"item","wordCnt":"2535"}, 335 | {"word":"best","wordCnt":"2533"}, 336 | {"word":"each","wordCnt":"2526"}, 337 | {"word":"front","wordCnt":"2518"}, 338 | {"word":"either","wordCnt":"2511"}, 339 | {"word":"happen","wordCnt":"2499"}, 340 | {"word":"rock","wordCnt":"2496"}, 341 | {"word":"changes","wordCnt":"2492"}, 342 | {"word":"comes","wordCnt":"2486"}, 343 | {"word":"business","wordCnt":"2464"}, 344 | {"word":"we'll","wordCnt":"2457"}, 345 | {"word":"developer","wordCnt":"2453"}, 346 | {"word":"heard","wordCnt":"2449"}, 347 | {"word":"used","wordCnt":"2437"}, 348 | {"word":"level","wordCnt":"2426"}, 349 | {"word":"pay","wordCnt":"2414"}, 350 | {"word":"sense","wordCnt":"2410"}, 351 | {"word":"future","wordCnt":"2403"}, 352 | {"word":"agree","wordCnt":"2382"}, 353 | {"word":"left","wordCnt":"2378"}, 354 | {"word":"tax","wordCnt":"2374"}, 355 | {"word":"items","wordCnt":"2347"}, 356 | {"word":"says","wordCnt":"2344"}, 357 | {"word":"ago","wordCnt":"2333"}, 358 | {"word":"nice","wordCnt":"2332"}, 359 | {"word":"basically","wordCnt":"2331"}, 360 | {"word":"code","wordCnt":"2330"}, 361 | {"word":"set","wordCnt":"2313"}, 362 | {"word":"somebody","wordCnt":"2307"}, 363 | {"word":"month","wordCnt":"2305"}, 364 | {"word":"turn","wordCnt":"2302"}, 365 | {"word":"enough","wordCnt":"2292"}, 366 | {"word":"areas","wordCnt":"2286"}, 367 | {"word":"night","wordCnt":"2283"}, 368 | {"word":"meetings","wordCnt":"2282"}, 369 | {"word":"high","wordCnt":"2282"}, 370 | {"word":"provide","wordCnt":"2280"}, 371 | {"word":"terms","wordCnt":"2249"}, 372 | {"word":"members","wordCnt":"2245"}, 373 | {"word":"sort","wordCnt":"2242"}, 374 | {"word":"north","wordCnt":"2230"}, 375 | {"word":"few","wordCnt":"2225"}, 376 | {"word":"resolution","wordCnt":"2208"}, 377 | {"word":"interested","wordCnt":"2200"}, 378 | {"word":"once","wordCnt":"2199"}, 379 | {"word":"state","wordCnt":"2199"}, 380 | {"word":"making","wordCnt":"2192"}, 381 | {"word":"approved","wordCnt":"2192"}, 382 | {"word":"foot","wordCnt":"2188"}, 383 | {"word":"mccaslin","wordCnt":"2168"}, 384 | {"word":"half","wordCnt":"2163"}, 385 | {"word":"type","wordCnt":"2161"}, 386 | {"word":"support","wordCnt":"2158"}, 387 | {"word":"particular","wordCnt":"2155"}, 388 | {"word":"parks","wordCnt":"2152"}, 389 | {"word":"haven't","wordCnt":"2150"}, 390 | {"word":"list","wordCnt":"2137"}, 391 | {"word":"top","wordCnt":"2128"}, 392 | {"word":"name","wordCnt":"2099"}, 393 | {"word":"thinking","wordCnt":"2090"}, 394 | {"word":"recommendation","wordCnt":"2089"}, 395 | {"word":"reason","wordCnt":"2080"}, 396 | {"word":"sorry","wordCnt":"2078"}, 397 | {"word":"without","wordCnt":"2075"}, 398 | {"word":"land","wordCnt":"2070"}, 399 | {"word":"love","wordCnt":"2066"}, 400 | {"word":"wasn't","wordCnt":"2063"}, 401 | {"word":"review","wordCnt":"2062"}, 402 | {"word":"answer","wordCnt":"2054"}, 403 | {"word":"its","wordCnt":"2052"}, 404 | {"word":"fact","wordCnt":"2039"}, 405 | {"word":"main","wordCnt":"2036"}, 406 | {"word":"until","wordCnt":"2025"}, 407 | {"word":"commercial","wordCnt":"2022"}, 408 | {"word":"yet","wordCnt":"2022"}, 409 | {"word":"hard","wordCnt":"2016"}, 410 | {"word":"across","wordCnt":"2004"}, 411 | {"word":"close","wordCnt":"1997"}, 412 | {"word":"continue","wordCnt":"1996"}, 413 | {"word":"anybody","wordCnt":"1995"}, 414 | {"word":"during","wordCnt":"1992"}, 415 | {"word":"seems","wordCnt":"1988"}, 416 | {"word":"homes","wordCnt":"1987"}, 417 | {"word":"based","wordCnt":"1983"}, 418 | {"word":"general","wordCnt":"1974"}, 419 | {"word":"existing","wordCnt":"1967"}, 420 | {"word":"past","wordCnt":"1963"}, 421 | {"word":"final","wordCnt":"1958"}, 422 | {"word":"presentation","wordCnt":"1949"}, 423 | {"word":"he's","wordCnt":"1945"}, 424 | {"word":"real","wordCnt":"1935"}, 425 | {"word":"months","wordCnt":"1924"}, 426 | {"word":"play","wordCnt":"1919"}, 427 | {"word":"add","wordCnt":"1915"}, 428 | {"word":"small","wordCnt":"1914"}, 429 | {"word":"field","wordCnt":"1908"}, 430 | {"word":"send","wordCnt":"1895"}, 431 | {"word":"hearing","wordCnt":"1889"}, 432 | {"word":"wouldn't","wordCnt":"1889"}, 433 | {"word":"option","wordCnt":"1887"}, 434 | {"word":"remember","wordCnt":"1884"}, 435 | {"word":"own","wordCnt":"1881"}, 436 | {"word":"old","wordCnt":"1859"}, 437 | {"word":"plans","wordCnt":"1855"}, 438 | {"word":"minutes","wordCnt":"1852"}, 439 | {"word":"four","wordCnt":"1850"}, 440 | {"word":"looks","wordCnt":"1846"}, 441 | {"word":"amount","wordCnt":"1831"}, 442 | {"word":"live","wordCnt":"1825"}, 443 | {"word":"seen","wordCnt":"1821"}, 444 | {"word":"putting","wordCnt":"1813"}, 445 | {"word":"motion","wordCnt":"1813"}, 446 | {"word":"though","wordCnt":"1811"}, 447 | {"word":"proposed","wordCnt":"1811"}, 448 | {"word":"order","wordCnt":"1810"}, 449 | {"word":"specific","wordCnt":"1810"}, 450 | {"word":"possible","wordCnt":"1807"}, 451 | {"word":"facility","wordCnt":"1806"}, 452 | {"word":"ever","wordCnt":"1801"}, 453 | {"word":"nothing","wordCnt":"1800"}, 454 | {"word":"times","wordCnt":"1799"}, 455 | {"word":"alright","wordCnt":"1793"}, 456 | {"word":"isn't","wordCnt":"1789"}, 457 | {"word":"event","wordCnt":"1782"}, 458 | {"word":"interest","wordCnt":"1782"}, 459 | {"word":"person","wordCnt":"1778"}, 460 | {"word":"full","wordCnt":"1774"}, 461 | {"word":"bridge","wordCnt":"1768"}, 462 | {"word":"run","wordCnt":"1765"}, 463 | {"word":"read","wordCnt":"1764"}, 464 | {"word":"improvements","wordCnt":"1762"}, 465 | {"word":"consider","wordCnt":"1758"}, 466 | {"word":"hundred","wordCnt":"1755"}, 467 | {"word":"makes","wordCnt":"1753"}, 468 | {"word":"discuss","wordCnt":"1748"}, 469 | {"word":"city","wordCnt":"1741"}, 470 | {"word":"taking","wordCnt":"1736"}, 471 | {"word":"less","wordCnt":"1733"}, 472 | {"word":"clear","wordCnt":"1730"}, 473 | {"word":"current","wordCnt":"1730"}, 474 | {"word":"west","wordCnt":"1729"}, 475 | {"word":"certainly","wordCnt":"1724"}, 476 | {"word":"happy","wordCnt":"1724"}, 477 | {"word":"fire","wordCnt":"1718"}, 478 | {"word":"days","wordCnt":"1710"}, 479 | {"word":"built","wordCnt":"1707"}, 480 | {"word":"decision","wordCnt":"1705"}, 481 | {"word":"options","wordCnt":"1702"}, 482 | {"word":"figure","wordCnt":"1702"}, 483 | {"word":"system","wordCnt":"1698"}, 484 | {"word":"approval","wordCnt":"1690"}, 485 | {"word":"moving","wordCnt":"1689"}, 486 | {"word":"case","wordCnt":"1689"}, 487 | {"word":"started","wordCnt":"1688"}, 488 | {"word":"asked","wordCnt":"1686"}, 489 | {"word":"thanks","wordCnt":"1681"}, 490 | {"word":"room","wordCnt":"1678"}, 491 | {"word":"appreciate","wordCnt":"1674"}, 492 | {"word":"green","wordCnt":"1668"}, 493 | {"word":"section","wordCnt":"1665"}, 494 | {"word":"trustees","wordCnt":"1664"}, 495 | {"word":"later","wordCnt":"1664"}, 496 | {"word":"course","wordCnt":"1663"}, 497 | {"word":"rather","wordCnt":"1663"}, 498 | {"word":"location","wordCnt":"1659"}, 499 | {"word":"certain","wordCnt":"1658"}, 500 | {"word":"quite","wordCnt":"1658"}, 501 | {"word":"available","wordCnt":"1657"}, 502 | {"word":"dollars","wordCnt":"1644"}, 503 | {"word":"called","wordCnt":"1644"}, 504 | {"word":"further","wordCnt":"1636"}, 505 | {"word":"address","wordCnt":"1631"}, 506 | {"word":"colorado","wordCnt":"1625"}, 507 | {"word":"stop","wordCnt":"1617"}, 508 | {"word":"concern","wordCnt":"1614"}, 509 | {"word":"approve","wordCnt":"1613"}, 510 | {"word":"family","wordCnt":"1613"}, 511 | {"word":"asking","wordCnt":"1613"}, 512 | {"word":"five","wordCnt":"1607"}, 513 | {"word":"impact","wordCnt":"1599"}, 514 | {"word":"exactly","wordCnt":"1580"}, 515 | {"word":"concept","wordCnt":"1572"}, 516 | {"word":"bike","wordCnt":"1571"}, 517 | {"word":"car","wordCnt":"1565"}, 518 | {"word":"dog","wordCnt":"1562"}, 519 | {"word":"contract","wordCnt":"1551"}, 520 | {"word":"leave","wordCnt":"1551"}, 521 | {"word":"retail","wordCnt":"1550"}, 522 | {"word":"fine","wordCnt":"1545"}, 523 | {"word":"regarding","wordCnt":"1544"}, 524 | {"word":"spend","wordCnt":"1543"}, 525 | {"word":"correct","wordCnt":"1541"}, 526 | {"word":"works","wordCnt":"1539"}, 527 | {"word":"coal","wordCnt":"1539"}, 528 | {"word":"job","wordCnt":"1531"}, 529 | {"word":"gets","wordCnt":"1531"}, 530 | {"word":"using","wordCnt":"1529"}, 531 | {"word":"block","wordCnt":"1527"}, 532 | {"word":"single","wordCnt":"1526"}, 533 | {"word":"hope","wordCnt":"1519"}, 534 | {"word":"meet","wordCnt":"1519"}, 535 | {"word":"buildings","wordCnt":"1518"}, 536 | {"word":"potential","wordCnt":"1507"}, 537 | {"word":"several","wordCnt":"1502"}, 538 | {"word":"deal","wordCnt":"1501"}, 539 | {"word":"walk","wordCnt":"1493"}, 540 | {"word":"mind","wordCnt":"1488"}, 541 | {"word":"term","wordCnt":"1487"}, 542 | {"word":"obviously","wordCnt":"1486"}, 543 | {"word":"light","wordCnt":"1484"}, 544 | {"word":"size","wordCnt":"1482"}, 545 | {"word":"while","wordCnt":"1478"}, 546 | {"word":"hours","wordCnt":"1470"}, 547 | {"word":"ftp","wordCnt":"1469"}, 548 | {"word":"currently","wordCnt":"1467"}, 549 | {"word":"parkway","wordCnt":"1465"}, 550 | {"word":"residence","wordCnt":"1464"}, 551 | {"word":"numbers","wordCnt":"1460"}, 552 | {"word":"someone","wordCnt":"1460"}, 553 | {"word":"concerned","wordCnt":"1459"}, 554 | {"word":"looked","wordCnt":"1458"}, 555 | {"word":"marshall","wordCnt":"1453"}, 556 | {"word":"services","wordCnt":"1451"}, 557 | {"word":"east","wordCnt":"1446"}, 558 | {"word":"application","wordCnt":"1440"}, 559 | {"word":"supposed","wordCnt":"1435"}, 560 | {"word":"request","wordCnt":"1432"}, 561 | {"word":"almost","wordCnt":"1424"}, 562 | {"word":"outside","wordCnt":"1424"}, 563 | {"word":"buy","wordCnt":"1422"}, 564 | {"word":"structure","wordCnt":"1422"}, 565 | {"word":"library","wordCnt":"1418"}, 566 | {"word":"transportation","wordCnt":"1417"}, 567 | {"word":"session","wordCnt":"1417"}, 568 | {"word":"oh","wordCnt":"1410"}, 569 | {"word":"date","wordCnt":"1406"}, 570 | {"word":"brought","wordCnt":"1405"}, 571 | {"word":"streets","wordCnt":"1405"}, 572 | {"word":"discussed","wordCnt":"1404"}, 573 | {"word":"involved","wordCnt":"1396"}, 574 | {"word":"lane","wordCnt":"1396"}, 575 | {"word":"direction","wordCnt":"1394"}, 576 | {"word":"care","wordCnt":"1393"}, 577 | {"word":"ice","wordCnt":"1390"}, 578 | {"word":"hour","wordCnt":"1388"}, 579 | {"word":"saw","wordCnt":"1381"}, 580 | {"word":"include","wordCnt":"1381"}, 581 | {"word":"piece","wordCnt":"1380"}, 582 | {"word":"projects","wordCnt":"1376"}, 583 | {"word":"word","wordCnt":"1375"}, 584 | {"word":"spaces","wordCnt":"1374"}, 585 | {"word":"concrete","wordCnt":"1372"}, 586 | {"word":"hotel","wordCnt":"1367"}, 587 | {"word":"neighborhood","wordCnt":"1366"}, 588 | {"word":"pd","wordCnt":"1359"}, 589 | {"word":"example","wordCnt":"1357"}, 590 | {"word":"definitely","wordCnt":"1356"}, 591 | {"word":"won't","wordCnt":"1354"}, 592 | {"word":"connection","wordCnt":"1354"}, 593 | {"word":"took","wordCnt":"1354"}, 594 | {"word":"wants","wordCnt":"1354"}, 595 | {"word":"mentioned","wordCnt":"1347"}, 596 | {"word":"rest","wordCnt":"1331"}, 597 | {"word":"anyway","wordCnt":"1331"}, 598 | {"word":"create","wordCnt":"1320"}, 599 | {"word":"trees","wordCnt":"1310"}, 600 | {"word":"such","wordCnt":"1309"}, 601 | {"word":"favor","wordCnt":"1304"}, 602 | {"word":"included","wordCnt":"1300"}, 603 | {"word":"perspective","wordCnt":"1299"}, 604 | {"word":"pool","wordCnt":"1294"}, 605 | {"word":"please","wordCnt":"1290"}, 606 | {"word":"ones","wordCnt":"1290"}, 607 | {"word":"free","wordCnt":"1288"}, 608 | {"word":"increase","wordCnt":"1285"}, 609 | {"word":"maintenance","wordCnt":"1281"}, 610 | {"word":"soon","wordCnt":"1279"}, 611 | {"word":"lots","wordCnt":"1279"}, 612 | {"word":"page","wordCnt":"1279"}, 613 | {"word":"large","wordCnt":"1275"}, 614 | {"word":"phase","wordCnt":"1274"}, 615 | {"word":"ideas","wordCnt":"1268"}, 616 | {"word":"advisory","wordCnt":"1267"}, 617 | {"word":"roundabout","wordCnt":"1262"}, 618 | {"word":"speed","wordCnt":"1260"}, 619 | {"word":"report","wordCnt":"1260"}, 620 | {"word":"fun","wordCnt":"1259"}, 621 | {"word":"you've","wordCnt":"1257"}, 622 | {"word":"manager","wordCnt":"1257"}, 623 | {"word":"everyone","wordCnt":"1252"}, 624 | {"word":"businesses","wordCnt":"1249"}, 625 | {"word":"hopefully","wordCnt":"1249"}, 626 | {"word":"concerns","wordCnt":"1244"}, 627 | {"word":"stay","wordCnt":"1243"}, 628 | {"word":"allow","wordCnt":"1242"}, 629 | {"word":"earlier","wordCnt":"1236"}, 630 | {"word":"vote","wordCnt":"1235"}, 631 | {"word":"entire","wordCnt":"1233"}, 632 | {"word":"energy","wordCnt":"1233"}, 633 | {"word":"sports","wordCnt":"1227"}, 634 | {"word":"you'll","wordCnt":"1224"}, 635 | {"word":"update","wordCnt":"1223"}, 636 | {"word":"schedule","wordCnt":"1213"}, 637 | {"word":"weather","wordCnt":"1213"}, 638 | {"word":"court","wordCnt":"1213"}, 639 | {"word":"language","wordCnt":"1212"}, 640 | {"word":"cars","wordCnt":"1208"}, 641 | {"word":"interesting","wordCnt":"1208"}, 642 | {"word":"market","wordCnt":"1204"}, 643 | {"word":"signs","wordCnt":"1200"}, 644 | {"word":"properties","wordCnt":"1199"}, 645 | {"word":"local","wordCnt":"1198"}, 646 | {"word":"evening","wordCnt":"1197"}, 647 | {"word":"drop","wordCnt":"1196"}, 648 | {"word":"starting","wordCnt":"1194"}, 649 | {"word":"weeks","wordCnt":"1191"}, 650 | {"word":"study","wordCnt":"1191"}, 651 | {"word":"behind","wordCnt":"1190"}, 652 | {"word":"happened","wordCnt":"1183"}, 653 | {"word":"wait","wordCnt":"1181"}, 654 | {"word":"email","wordCnt":"1176"}, 655 | {"word":"communities","wordCnt":"1174"}, 656 | {"word":"sales","wordCnt":"1172"}, 657 | {"word":"landscaping","wordCnt":"1170"}, 658 | {"word":"safety","wordCnt":"1169"}, 659 | {"word":"happens","wordCnt":"1168"}, 660 | {"word":"document","wordCnt":"1167"}, 661 | {"word":"quick","wordCnt":"1165"}, 662 | {"word":"face","wordCnt":"1163"}, 663 | {"word":"conversation","wordCnt":"1158"}, 664 | {"word":"somewhere","wordCnt":"1155"}, 665 | {"word":"avenue","wordCnt":"1150"}, 666 | {"word":"pick","wordCnt":"1149"}, 667 | {"word":"recreation","wordCnt":"1143"}, 668 | {"word":"speak","wordCnt":"1137"}, 669 | {"word":"rocky","wordCnt":"1130"}, 670 | {"word":"ready","wordCnt":"1127"}, 671 | {"word":"early","wordCnt":"1124"}, 672 | {"word":"needed","wordCnt":"1120"}, 673 | {"word":"corner","wordCnt":"1119"}, 674 | {"word":"buffer","wordCnt":"1117"}, 675 | {"word":"ways","wordCnt":"1117"}, 676 | {"word":"middle","wordCnt":"1113"}, 677 | {"word":"july","wordCnt":"1110"}, 678 | {"word":"folks","wordCnt":"1110"}, 679 | {"word":"ft","wordCnt":"1109"}, 680 | {"word":"step","wordCnt":"1108"}, 681 | {"word":"per","wordCnt":"1106"}, 682 | {"word":"bad","wordCnt":"1105"}, 683 | {"word":"private","wordCnt":"1105"}, 684 | {"word":"necessarily","wordCnt":"1100"}, 685 | {"word":"forth","wordCnt":"1095"}, 686 | {"word":"significant","wordCnt":"1089"}, 687 | {"word":"separate","wordCnt":"1084"}, 688 | {"word":"revenue","wordCnt":"1084"}, 689 | {"word":"told","wordCnt":"1083"}, 690 | {"word":"events","wordCnt":"1080"}, 691 | {"word":"position","wordCnt":"1077"}, 692 | {"word":"means","wordCnt":"1075"}, 693 | {"word":"survey","wordCnt":"1075"}, 694 | {"word":"plant","wordCnt":"1073"}, 695 | {"word":"parts","wordCnt":"1068"}, 696 | {"word":"given","wordCnt":"1067"}, 697 | {"word":"resident","wordCnt":"1060"}, 698 | {"word":"summer","wordCnt":"1057"}, 699 | {"word":"pedestrian","wordCnt":"1057"}, 700 | {"word":"understanding","wordCnt":"1056"}, 701 | {"word":"table","wordCnt":"1053"}, 702 | {"word":"portion","wordCnt":"1053"}, 703 | {"word":"follow","wordCnt":"1049"}, 704 | {"word":"price","wordCnt":"1048"}, 705 | {"word":"units","wordCnt":"1047"}, 706 | {"word":"potentially","wordCnt":"1047"}, 707 | {"word":"acres","wordCnt":"1046"}, 708 | {"word":"special","wordCnt":"1045"}, 709 | {"word":"towards","wordCnt":"1044"}, 710 | {"word":"waste","wordCnt":"1039"}, 711 | {"word":"seeing","wordCnt":"1038"}, 712 | {"word":"major","wordCnt":"1036"}, 713 | {"word":"ahead","wordCnt":"1032"}, 714 | {"word":"department","wordCnt":"1030"}, 715 | {"word":"gone","wordCnt":"1028"}, 716 | {"word":"decide","wordCnt":"1028"}, 717 | {"word":"august","wordCnt":"1026"}, 718 | {"word":"williams","wordCnt":"1025"}, 719 | {"word":"track","wordCnt":"1022"}, 720 | {"word":"specifically","wordCnt":"1022"}, 721 | {"word":"morning","wordCnt":"1020"}, 722 | {"word":"changed","wordCnt":"1018"}, 723 | {"word":"phone","wordCnt":"1018"}, 724 | {"word":"instead","wordCnt":"1018"}, 725 | {"word":"short","wordCnt":"1015"}, 726 | {"word":"sometimes","wordCnt":"1013"}, 727 | {"word":"third","wordCnt":"1013"}, 728 | {"word":"recommendations","wordCnt":"1012"}, 729 | {"word":"truck","wordCnt":"1001"}, 730 | {"word":"they've","wordCnt":"1001"}, 731 | {"word":"low","wordCnt":"1001"}, 732 | {"word":"policy","wordCnt":"998"}, 733 | {"word":"pull","wordCnt":"996"}, 734 | {"word":"allowed","wordCnt":"996"}, 735 | {"word":"door","wordCnt":"994"}, 736 | {"word":"appropriate","wordCnt":"992"}, 737 | {"word":"feedback","wordCnt":"991"}, 738 | {"word":"share","wordCnt":"991"}, 739 | {"word":"member","wordCnt":"989"}, 740 | {"word":"station","wordCnt":"988"}, 741 | {"word":"bill","wordCnt":"987"}, 742 | {"word":"sit","wordCnt":"987"}, 743 | {"word":"rec","wordCnt":"987"}, 744 | {"word":"higher","wordCnt":"984"}, 745 | {"word":"housing","wordCnt":"984"}, 746 | {"word":"council","wordCnt":"983"}, 747 | {"word":"develop","wordCnt":"982"}, 748 | {"word":"drainage","wordCnt":"981"}, 749 | {"word":"cut","wordCnt":"979"}, 750 | {"word":"places","wordCnt":"979"}, 751 | {"word":"lights","wordCnt":"979"}, 752 | {"word":"data","wordCnt":"977"}, 753 | {"word":"actual","wordCnt":"976"}, 754 | {"word":"committees","wordCnt":"973"}, 755 | {"word":"running","wordCnt":"972"}, 756 | {"word":"opinion","wordCnt":"970"}, 757 | {"word":"anyone","wordCnt":"970"}, 758 | {"word":"condition","wordCnt":"966"}, 759 | {"word":"amendment","wordCnt":"966"}, 760 | {"word":"proposal","wordCnt":"965"}, 761 | {"word":"total","wordCnt":"965"}, 762 | {"word":"similar","wordCnt":"962"}, 763 | {"word":"life","wordCnt":"961"}, 764 | {"word":"found","wordCnt":"960"}, 765 | {"word":"six","wordCnt":"955"}, 766 | {"word":"capital","wordCnt":"954"}, 767 | {"word":"matter","wordCnt":"952"}, 768 | {"word":"intersection","wordCnt":"951"}, 769 | {"word":"amenities","wordCnt":"951"}, 770 | {"word":"control","wordCnt":"949"}, 771 | {"word":"st","wordCnt":"949"}, 772 | {"word":"happening","wordCnt":"947"}, 773 | {"word":"check","wordCnt":"946"}, 774 | {"word":"required","wordCnt":"945"}, 775 | {"word":"sent","wordCnt":"945"}, 776 | {"word":"website","wordCnt":"944"}, 777 | {"word":"absolutely","wordCnt":"941"}, 778 | {"word":"sounds","wordCnt":"940"}, 779 | {"word":"difference","wordCnt":"940"}, 780 | {"word":"points","wordCnt":"939"}, 781 | {"word":"marketplace","wordCnt":"937"}, 782 | {"word":"picture","wordCnt":"937"}, 783 | {"word":"story","wordCnt":"936"}, 784 | {"word":"recycling","wordCnt":"933"}, 785 | {"word":"man","wordCnt":"933"}, 786 | {"word":"landscape","wordCnt":"929"}, 787 | {"word":"sidewalk","wordCnt":"928"}, 788 | {"word":"red","wordCnt":"922"}, 789 | {"word":"experience","wordCnt":"922"}, 790 | {"word":"it'll","wordCnt":"922"}, 791 | {"word":"june","wordCnt":"921"}, 792 | {"word":"sitting","wordCnt":"920"}, 793 | {"word":"seem","wordCnt":"918"}, 794 | {"word":"received","wordCnt":"915"}, 795 | {"word":"chance","wordCnt":"915"}, 796 | {"word":"wrong","wordCnt":"914"}, 797 | {"word":"near","wordCnt":"913"}, 798 | {"word":"hey","wordCnt":"913"}, 799 | {"word":"parcel","wordCnt":"912"}, 800 | {"word":"priority","wordCnt":"912"}, 801 | {"word":"valley","wordCnt":"911"}, 802 | {"word":"april","wordCnt":"910"}, 803 | {"word":"crossing","wordCnt":"908"}, 804 | {"word":"related","wordCnt":"906"}, 805 | {"word":"quarter","wordCnt":"904"}, 806 | {"word":"lanes","wordCnt":"903"}, 807 | {"word":"funding","wordCnt":"902"}, 808 | {"word":"ground","wordCnt":"901"}, 809 | {"word":"natural","wordCnt":"901"}, 810 | {"word":"situation","wordCnt":"900"}, 811 | {"word":"possibly","wordCnt":"899"}, 812 | {"word":"discussions","wordCnt":"898"}, 813 | {"word":"denver","wordCnt":"897"}, 814 | {"word":"added","wordCnt":"895"}, 815 | {"word":"versus","wordCnt":"893"}, 816 | {"word":"head","wordCnt":"890"}, 817 | {"word":"pass","wordCnt":"890"}, 818 | {"word":"against","wordCnt":"887"}, 819 | {"word":"she's","wordCnt":"885"}, 820 | {"word":"hall","wordCnt":"885"}, 821 | {"word":"including","wordCnt":"882"}, 822 | {"word":"team","wordCnt":"880"}, 823 | {"word":"uses","wordCnt":"876"}, 824 | {"word":"thousand","wordCnt":"875"}, 825 | {"word":"huge","wordCnt":"873"}, 826 | {"word":"funds","wordCnt":"873"}, 827 | {"word":"taken","wordCnt":"871"}, 828 | {"word":"cool","wordCnt":"871"}, 829 | {"word":"programs","wordCnt":"866"}, 830 | {"word":"bigger","wordCnt":"866"}, 831 | {"word":"requirements","wordCnt":"863"}, 832 | {"word":"focus","wordCnt":"863"}, 833 | {"word":"alex","wordCnt":"862"}, 834 | {"word":"monday","wordCnt":"860"}, 835 | {"word":"quickly","wordCnt":"858"}, 836 | {"word":"especially","wordCnt":"857"}, 837 | {"word":"fit","wordCnt":"856"}, 838 | {"word":"ability","wordCnt":"855"}, 839 | {"word":"cross","wordCnt":"854"}, 840 | {"word":"indiana","wordCnt":"854"}, 841 | {"word":"vision","wordCnt":"853"}, 842 | {"word":"snow","wordCnt":"852"}, 843 | {"word":"consent","wordCnt":"851"}, 844 | {"word":"couldn't","wordCnt":"850"}, 845 | {"word":"who's","wordCnt":"849"}, 846 | {"word":"presented","wordCnt":"849"}, 847 | {"word":"trustee","wordCnt":"847"}, 848 | {"word":"friday","wordCnt":"847"}, 849 | {"word":"september","wordCnt":"846"}, 850 | {"word":"nd","wordCnt":"846"}, 851 | {"word":"cold","wordCnt":"844"}, 852 | {"word":"cover","wordCnt":"844"}, 853 | {"word":"paid","wordCnt":"843"}] --------------------------------------------------------------------------------