├── .env ├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── azuredeploy.json ├── azuredeploy.parameters.json ├── docker-compose.yml ├── project-fortis-backup ├── .dockerignore ├── backup-cassandra-keyspace.sh ├── docker │ ├── Dockerfile │ ├── run-backup.sh │ └── run-cqlsh.sh └── travis │ ├── ci.sh │ └── publish.sh ├── project-fortis-interfaces ├── .dockerignore ├── .eslintrc ├── .gitignore ├── docker │ ├── Dockerfile │ └── run-react.sh ├── package-lock.json ├── package.json ├── public │ ├── images │ │ ├── OCHA_Logo.png │ │ ├── intl_alert.png │ │ ├── stroer_logo.png │ │ └── umea_white.svg │ └── index.html ├── src │ ├── actions │ │ ├── Admin │ │ │ └── index.js │ │ ├── Dashboard │ │ │ └── index.js │ │ ├── Facts │ │ │ └── index.js │ │ ├── constants.js │ │ └── shared.js │ ├── components │ │ ├── Admin │ │ │ ├── Admin.js │ │ │ ├── AdminSettings.js │ │ │ ├── AdminWatchlist.js │ │ │ ├── BlacklistEditor.js │ │ │ ├── CustomEventsEditor.js │ │ │ ├── DataGrid.js │ │ │ ├── SiteExportButton.js │ │ │ ├── Streams │ │ │ │ ├── CreateStream.js │ │ │ │ ├── StreamConstants.js │ │ │ │ ├── StreamEditor.js │ │ │ │ ├── StreamParamsButtonFormatter.js │ │ │ │ └── StreamStatusButtonFormatter.js │ │ │ ├── TrustedSources.js │ │ │ ├── UserRoles.js │ │ │ └── shared.js │ │ ├── Facts │ │ │ ├── FactsList.js │ │ │ └── ListView.js │ │ ├── Footer.js │ │ ├── Graphics │ │ │ ├── DoughnutChart.js │ │ │ ├── GraphCard.js │ │ │ ├── NoData.js │ │ │ ├── Sentiment.js │ │ │ ├── Timeline.js │ │ │ └── WordCloud.js │ │ ├── Header │ │ │ ├── Header.js │ │ │ └── index.js │ │ ├── Insights │ │ │ ├── ActiveFiltersView.js │ │ │ ├── ActivityFeed.js │ │ │ ├── CategoryPicker.js │ │ │ ├── CsvExporter.js │ │ │ ├── Dashboard.js │ │ │ ├── DataSelector.js │ │ │ ├── DrawerActionsIconButton.js │ │ │ ├── FortisEvent.js │ │ │ ├── HeatmapToggle.js │ │ │ ├── LanguagePicker.js │ │ │ ├── Layouts │ │ │ │ └── index.js │ │ │ ├── MapBoundingReset.js │ │ │ ├── Maps │ │ │ │ ├── HeatMap.js │ │ │ │ ├── MarkerCluster.js │ │ │ │ ├── MarkerClusterGroup.js │ │ │ │ ├── TileLayer.js │ │ │ │ └── style.scss │ │ │ ├── PopularLocationsChart.js │ │ │ ├── PopularSourcesChart.js │ │ │ ├── PopularTermsChart.js │ │ │ ├── SentimentTreeview.js │ │ │ ├── ShareButton.js │ │ │ ├── Subheader.js │ │ │ ├── TermFilter.js │ │ │ ├── TimeSeriesGraph.js │ │ │ ├── TopicCloud.js │ │ │ ├── TranslateButton.js │ │ │ ├── TreeFilter.js │ │ │ ├── TypeaheadSearch.js │ │ │ └── shared.js │ │ └── dialogs │ │ │ ├── DialogBox.js │ │ │ ├── EventDetails.js │ │ │ ├── Highlighter.js │ │ │ └── MapViewPort.js │ ├── config.js │ ├── images │ │ ├── MSFT_logo_png.png │ │ ├── OCHA_Logo.png │ │ ├── layers-2x.png │ │ ├── layers.png │ │ ├── marker-icon-2x.png │ │ ├── marker-icon-red.png │ │ ├── marker-icon.png │ │ ├── marker-shadow.png │ │ ├── nav_bg2.svg │ │ ├── partner_catalyst_icon.ico │ │ ├── partner_catalyst_icon.png │ │ ├── partner_catalyst_logo.png │ │ ├── select.png │ │ └── umea_white.svg │ ├── index.js │ ├── routes │ │ ├── AdminPage.js │ │ ├── AppPage.js │ │ ├── DashboardPage.js │ │ ├── FactsPage.js │ │ ├── NotFoundPage.js │ │ ├── UnsupportedBrowserPage.js │ │ └── routes.js │ ├── services │ │ ├── Admin │ │ │ └── index.js │ │ ├── Dashboard │ │ │ └── index.js │ │ ├── Facts │ │ │ └── index.js │ │ ├── featureService.js │ │ ├── graphql │ │ │ ├── fragments │ │ │ │ ├── Admin │ │ │ │ │ └── index.js │ │ │ │ ├── Dashboard │ │ │ │ │ └── index.js │ │ │ │ └── Facts │ │ │ │ │ └── index.js │ │ │ ├── mutations │ │ │ │ └── Admin │ │ │ │ │ └── index.js │ │ │ └── queries │ │ │ │ ├── Admin │ │ │ │ └── index.js │ │ │ │ ├── Dashboard │ │ │ │ └── index.js │ │ │ │ └── Facts │ │ │ │ └── index.js │ │ └── shared.js │ ├── stores │ │ ├── AdminStore.js │ │ └── DataStore.js │ ├── styles │ │ ├── Admin │ │ │ └── Admin.css │ │ ├── Facts │ │ │ └── Facts.css │ │ ├── Footer.css │ │ ├── Global.css │ │ ├── Graphics │ │ │ ├── GraphCard.css │ │ │ ├── colors.js │ │ │ └── styles.js │ │ ├── Header.css │ │ └── Insights │ │ │ ├── ActiveFiltersView.css │ │ │ ├── ActivityFeed.css │ │ │ ├── ActivityFeed.js │ │ │ ├── Dashboard.css │ │ │ ├── DataSelector.css │ │ │ ├── DialogBox.css │ │ │ ├── HeatMap.css │ │ │ ├── SentimentTreeView.css │ │ │ ├── SentimentTreeview.js │ │ │ └── TypeaheadSearch.css │ └── utils │ │ ├── Fact.js │ │ ├── HtmlSanitizer.js │ │ └── Utils.js └── travis │ └── ci.sh ├── project-fortis-pipeline ├── .gitignore ├── docs │ ├── aad-setup.md │ ├── admin-settings.md │ ├── azure-deploy-parameters.md │ ├── background.md │ ├── development-faq.md │ ├── development-setup.md │ └── production-setup.md ├── fortis-deploy.sh ├── localdeploy │ ├── parameters.json │ ├── parse-output.py │ ├── seed-data │ │ ├── seed-data-no-events.tar.gz │ │ ├── seed-data-twitter-clewolff.tar.gz │ │ ├── seed-data-twitter-steph.tar.gz │ │ ├── seed-data-twitter.tar.gz │ │ └── seed-data.tar.gz │ ├── setup-development-azure-resources.sh │ └── template.json ├── ops │ ├── charts │ │ ├── cassandra │ │ │ ├── .helmignore │ │ │ ├── Chart.yaml │ │ │ ├── templates │ │ │ │ ├── _helpers.tpl │ │ │ │ ├── statefulset.yaml │ │ │ │ └── svc.yaml │ │ │ └── values.yaml │ │ └── spark │ │ │ ├── .helmignore │ │ │ ├── Chart.yaml │ │ │ ├── templates │ │ │ ├── NOTES.txt │ │ │ ├── _helpers.tpl │ │ │ ├── secret.yaml │ │ │ ├── spark-master-deployment.yaml │ │ │ ├── spark-master-service.yaml │ │ │ ├── spark-worker-statefulset.yaml │ │ │ └── spark-zeppelin-deployment.yaml │ │ │ └── values.yaml │ ├── config │ │ └── defaultTopics │ │ │ ├── climate.json │ │ │ ├── health.json │ │ │ └── humanitarian.json │ ├── create-cluster.sh │ ├── create-tags.sh │ ├── install-cassandra.sh │ ├── install-featureservice.sh │ ├── install-fortis-backup.sh │ ├── install-fortis-interfaces.sh │ ├── install-fortis-services.sh │ ├── install-spark.sh │ └── storage-ddls │ │ ├── cassandra-setup.cql │ │ └── settings-setup.cql └── travis │ └── ci.sh ├── project-fortis-services ├── .dockerignore ├── .eslintignore ├── .eslintrc ├── .gitattributes ├── .gitignore ├── config.js ├── docker │ ├── Dockerfile │ ├── run-cqlsh.sh │ └── run-node.sh ├── package-lock.json ├── package.json ├── server.js ├── src │ ├── auth.js │ ├── clients │ │ ├── appinsights │ │ │ ├── AppInsightsClient.js │ │ │ ├── AppInsightsConstants.js │ │ │ └── LoggingClient.js │ │ ├── cassandra │ │ │ └── CassandraConnector.js │ │ ├── eventhub │ │ │ └── EventHubSender.js │ │ ├── locations │ │ │ └── FeatureServiceClient.js │ │ ├── storage │ │ │ └── BlobStorageClient.js │ │ ├── streaming │ │ │ ├── ServiceBusClient.js │ │ │ └── StreamingController.js │ │ └── translator │ │ │ └── MsftTranslator.js │ ├── resolvers │ │ ├── Edges │ │ │ ├── index.js │ │ │ └── queries.js │ │ ├── Messages │ │ │ ├── index.js │ │ │ ├── mutations.js │ │ │ └── queries.js │ │ ├── Settings │ │ │ ├── index.js │ │ │ ├── mutations.js │ │ │ ├── queries.js │ │ │ └── shared.js │ │ ├── Tiles │ │ │ ├── index.js │ │ │ └── queries.js │ │ └── shared.js │ ├── routes │ │ └── healthcheck.js │ ├── schemas │ │ ├── EdgesSchema.js │ │ ├── MessageSchema.js │ │ ├── SettingsSchema.js │ │ └── TilesSchema.js │ ├── scripts │ │ ├── addusers.js │ │ ├── createsite.js │ │ └── ingestsetting.js │ └── utils │ │ ├── collections.js │ │ └── request.js └── travis │ ├── ci.sh │ └── publish.sh └── project-fortis-spark ├── .dockerignore ├── .gitignore ├── build.sbt ├── docker ├── Dockerfile ├── run-cqlsh.sh └── run-spark.sh ├── lib ├── spark-streaming-twitter_2.11-2.2.0-SNAPSHOT.jar ├── tritonus_remaining-0.3.6.jar └── tritonus_share-0.3.6.jar ├── project ├── build.properties └── plugins.sbt ├── src ├── main │ ├── resources │ │ └── ApplicationInsights.xml │ └── scala │ │ └── com │ │ └── microsoft │ │ └── partnercatalyst │ │ └── fortis │ │ └── spark │ │ ├── CassandraTest.scala │ │ ├── Constants.scala │ │ ├── FortisSettings.scala │ │ ├── Pipeline.scala │ │ ├── ProjectFortis.scala │ │ ├── StreamsChangeListener.scala │ │ ├── analyzer │ │ ├── AnalysisDefaults.scala │ │ ├── Analyzer.scala │ │ ├── BingAnalyzer.scala │ │ ├── CustomEventAnalyzer.scala │ │ ├── ExtendedFortisEvent.scala │ │ ├── FacebookCommentAnalyzer.scala │ │ ├── FacebookPostAnalyzer.scala │ │ ├── HTMLAnalyzer.scala │ │ ├── InstagramAnalyzer.scala │ │ ├── RSSAnalyzer.scala │ │ ├── RadioAnalyzer.scala │ │ ├── RedditAnalyzer.scala │ │ ├── TadawebAnalyzer.scala │ │ └── TwitterAnalyzer.scala │ │ ├── dba │ │ ├── CassandraConfigurationManager.scala │ │ ├── CassandraSchema.scala │ │ └── ConfigurationManager.scala │ │ ├── dto │ │ ├── BlacklistedItem.scala │ │ ├── ComputedTile.scala │ │ ├── ComputedTrend.scala │ │ ├── FortisEvent.scala │ │ ├── Geofence.scala │ │ └── SiteSettings.scala │ │ ├── logging │ │ ├── AppInsightsTelemetry.scala │ │ ├── FortisTelemetry.scala │ │ └── Timer.scala │ │ ├── serialization │ │ └── KryoRegistrator.scala │ │ ├── sinks │ │ └── cassandra │ │ │ ├── CassandraConfig.scala │ │ │ ├── CassandraEventSchema.scala │ │ │ ├── CassandraEventsSink.scala │ │ │ ├── CassandraExtensions.scala │ │ │ ├── FortisConnectionFactory.scala │ │ │ ├── Period.scala │ │ │ ├── aggregators │ │ │ ├── ConjunctiveTopicsOffineAggregator.scala │ │ │ └── TileAggregator.scala │ │ │ └── dto │ │ │ ├── AggregationRecord.scala │ │ │ ├── FortisRecords.scala │ │ │ └── UserDefinedTypes.scala │ │ ├── sources │ │ ├── StreamProviderFactory.scala │ │ ├── streamfactories │ │ │ ├── BingPageStreamFactory.scala │ │ │ ├── EventHubStreamFactory.scala │ │ │ ├── FacebookCommentStreamFactory.scala │ │ │ ├── FacebookPageStreamFactory.scala │ │ │ ├── HTMLStreamFactory.scala │ │ │ ├── InstagramLocationStreamFactory.scala │ │ │ ├── InstagramTagStreamFactory.scala │ │ │ ├── ParameterExtensions.scala │ │ │ ├── RSSStreamFactory.scala │ │ │ ├── RadioStreamFactory.scala │ │ │ ├── RedditStreamFactory.scala │ │ │ ├── StreamFactoryBase.scala │ │ │ └── TwitterStreamFactory.scala │ │ ├── streamprovider │ │ │ ├── ConnectorConfig.scala │ │ │ ├── StreamFactory.scala │ │ │ ├── StreamProvider.scala │ │ │ └── StreamProviderException.scala │ │ └── streamwrappers │ │ │ ├── customevents │ │ │ ├── CustomEvent.scala │ │ │ └── CustomEventsAdapter.scala │ │ │ ├── radio │ │ │ ├── RadioInputDStream.scala │ │ │ ├── RadioStreamUtils.scala │ │ │ ├── RadioTranscription.scala │ │ │ └── TranscriptionReceiver.scala │ │ │ └── tadaweb │ │ │ ├── TadawebAdapter.scala │ │ │ └── TadawebEvent.scala │ │ ├── transformcontext │ │ ├── Delta.scala │ │ ├── TransformContext.scala │ │ ├── TransformContextMessages.scala │ │ └── TransformContextProvider.scala │ │ └── transforms │ │ ├── ZipModelsProvider.scala │ │ ├── entities │ │ └── EntityRecognizer.scala │ │ ├── gender │ │ └── GenderDetector.scala │ │ ├── image │ │ ├── ImageAnalyzer.scala │ │ └── dto │ │ │ └── Json.scala │ │ ├── language │ │ ├── CognitiveServicesLanguageDetector.scala │ │ ├── LanguageDetector.scala │ │ ├── LocalLanguageDetector.scala │ │ ├── TextNormalizer.scala │ │ └── dto │ │ │ └── Json.scala │ │ ├── locations │ │ ├── LocationsExtractor.scala │ │ ├── LocationsExtractorFactory.scala │ │ ├── LuceneLocationsExtractor.scala │ │ ├── PlaceRecognizer.scala │ │ ├── StringUtils.scala │ │ ├── Tile.scala │ │ ├── client │ │ │ └── FeatureServiceClient.scala │ │ └── dto │ │ │ └── Json.scala │ │ ├── nlp │ │ ├── OpeNER.scala │ │ └── Tokenizer.scala │ │ ├── people │ │ └── PeopleRecognizer.scala │ │ ├── sentiment │ │ ├── CognitiveServicesSentimentDetector.scala │ │ ├── SentimentDetector.scala │ │ ├── WordListSentimentDetector.scala │ │ └── dto │ │ │ └── Json.scala │ │ ├── summary │ │ └── KeywordSummarizer.scala │ │ └── topic │ │ ├── Blacklist.scala │ │ ├── KeyphraseExtractor.scala │ │ ├── KeywordExtractor.scala │ │ ├── LuceneKeyphraseExtractor.scala │ │ └── LuceneKeywordExtractor.scala └── test │ └── scala │ └── com │ └── microsoft │ └── partnercatalyst │ └── fortis │ └── spark │ ├── SparkSpec.scala │ ├── StreamsChangeListenerTestSpec.scala │ ├── dto │ └── FortisEventSpec.scala │ ├── sinks │ └── cassandra │ │ ├── CassandraConjunctiveTopicsTestSpec.scala │ │ ├── CassandraConjunctiveTopicsTestSpecRdd.scala │ │ ├── CassandraExtensionsTests.scala │ │ ├── CassandraIntegrationTestSpec.scala │ │ └── PeriodSpec.scala │ ├── sources │ └── streamfactories │ │ └── TwitterStreamFactorySpec.scala │ └── transforms │ ├── image │ └── ImageAnalyzerSpec.scala │ ├── language │ ├── CognitiveServicesLanguageDetectorSpec.scala │ └── LocalLanguageDetectorSpec.scala │ ├── locations │ ├── LocationsExtractorSpec.scala │ ├── PlaceRecognizerIntegrationSpec.scala │ ├── StringUtilsSpec.scala │ ├── TileUtilsSpec.scala │ └── client │ │ └── FeatureServiceClientSpec.scala │ ├── nlp │ └── TokenizerSpec.scala │ ├── sentiment │ ├── CognitiveServicesSentimentDetectorSpec.scala │ ├── SentimentDetectorSpec.scala │ ├── WordListSentimentDetectorIntegrationSpec.scala │ └── WordListSentimentDetectorSpec.scala │ ├── summary │ └── KeywordSummarizerSpec.scala │ └── topic │ ├── BlacklistSpec.scala │ ├── KeywordExtractorSpec.scala │ └── LuceneKeyphraseExtractorSpec.scala ├── travis ├── ci.sh └── publish.sh └── version.sbt /.env: -------------------------------------------------------------------------------- 1 | BUILD_TAG=latest 2 | PROJECT_FORTIS_SPARK_CONTEXT_UI_PORT=7777 3 | PROJECT_FORTIS_SPARK_MASTER_UI_PORT=7778 4 | PROJECT_FORTIS_SPARK_WORKER_UI_PORT=7779 5 | PROJECT_FORTIS_INTERFACES_PORT=8888 6 | PROJECT_FORTIS_SERVICES_PORT=8889 7 | FEATURE_SERVICE_PORT=9090 8 | CASSANDRA_SEED_DATA_URL=https://raw.githubusercontent.com/CatalystCode/project-fortis/master/project-fortis-pipeline/localdeploy/seed-data/seed-data-twitter.tar.gz 9 | AD_CLIENT_ID=bf1ceec7-cfc7-49c5-9ed3-c6390b87dda5 10 | USERS=scicoria@microsoft.com,erisch@microsoft.com 11 | ADMINS=clewolff@microsoft.com,stmarker@microsoft.com,naros@microsoft.com,keha@microsoft.com 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env-secrets 2 | 3 | .idea/ 4 | .DS_Store 5 | *.orig 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | matrix: 2 | include: 3 | 4 | - language: bash 5 | script: 6 | - project-fortis-backup/travis/ci.sh 7 | services: 8 | - docker 9 | deploy: 10 | - provider: script 11 | script: project-fortis-backup/travis/publish.sh 12 | on: 13 | repo: CatalystCode/project-fortis 14 | tags: true 15 | env: 16 | NAME=project-fortis-backup 17 | 18 | - language: node_js 19 | node_js: 20 | - "6" 21 | cache: 22 | directories: 23 | project-fortis-interfaces/node_modules 24 | script: 25 | project-fortis-interfaces/travis/ci.sh 26 | env: 27 | NAME=project-fortis-interfaces 28 | 29 | - language: bash 30 | script: 31 | - project-fortis-pipeline/travis/ci.sh 32 | env: 33 | NAME=project-fortis-pipeline 34 | 35 | - language: node_js 36 | sudo: required 37 | node_js: 38 | - "9" 39 | cache: 40 | directories: 41 | project-fortis-services/node_modules 42 | script: 43 | project-fortis-services/travis/ci.sh 44 | services: 45 | - docker 46 | deploy: 47 | - provider: script 48 | script: project-fortis-services/travis/publish.sh 49 | on: 50 | repo: CatalystCode/project-fortis 51 | tags: true 52 | env: 53 | NAME=project-fortis-services 54 | 55 | - language: scala 56 | jdk: oraclejdk8 57 | scala: 2.11.7 58 | dist: trusty 59 | sudo: required 60 | script: 61 | - project-fortis-spark/travis/ci.sh 62 | deploy: 63 | - provider: script 64 | script: project-fortis-spark/travis/publish.sh 65 | on: 66 | repo: CatalystCode/project-fortis 67 | tags: true 68 | env: 69 | NAME=project-fortis-spark 70 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Catalyst Code 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # project-fortis 2 | 3 | Project Fortis is a data ingestion, analysis and visualization pipeline. The 4 | Fortis pipeline collects social media conversations and postings from the public 5 | web and darknet data sources. 6 | 7 | - [Find out more about the project](project-fortis-pipeline/docs/background.md) 8 | - [Learn how to set up Fortis in Azure](project-fortis-pipeline/docs/production-setup.md) 9 | - [Onboarding guide for developers](project-fortis-pipeline/docs/development-setup.md) 10 | -------------------------------------------------------------------------------- /azuredeploy.parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "githubProjectParent": { 6 | "value": "CatalystCode" 7 | }, 8 | "githubProjectRelease": { 9 | "value": "master" 10 | }, 11 | "siteName": { 12 | "value": "enter-your-fortis-site-name" 13 | }, 14 | "siteLocation": { 15 | "value": "East US" 16 | }, 17 | "acsMasterCount": { 18 | "value": 1 19 | }, 20 | "acsAgentCount": { 21 | "value": 6 22 | }, 23 | "agentVMSize": { 24 | "value": "Standard_L4s" 25 | }, 26 | "sparkWorkers": { 27 | "value": 6 28 | }, 29 | "cassandraNodes": { 30 | "value": 5 31 | }, 32 | "siteType": { 33 | "value": "none" 34 | }, 35 | "sshPublicKey": { 36 | "value": "CHANGE ME" 37 | }, 38 | "mapboxAccessToken": { 39 | "value": "CHANGE ME" 40 | }, 41 | "fortisSiteCloneUrl": { 42 | "value": "" 43 | }, 44 | "servicePrincipalAppId": { 45 | "value": "CHANGE ME" 46 | }, 47 | "servicePrincipalAppKey": { 48 | "value": "CHANGE ME" 49 | }, 50 | "activeDirectoryClientId": { 51 | "value": "" 52 | }, 53 | "fortisAdmins": { 54 | "value": "" 55 | }, 56 | "fortisUsers": { 57 | "value": "" 58 | }, 59 | "letsEncryptEmail": { 60 | "value": "" 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /project-fortis-backup/.dockerignore: -------------------------------------------------------------------------------- 1 | travis/ 2 | -------------------------------------------------------------------------------- /project-fortis-backup/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:xenial 2 | 3 | # install cassandra 4 | ENV CASSANDRA_HOME="/opt/cassandra" 5 | ARG CASSANDRA_VERSION="3.11.0" 6 | ARG CASSANDRA_ARTIFACT="apache-cassandra-${CASSANDRA_VERSION}" 7 | ARG CASSANDRA_URL="http://archive.apache.org/dist/cassandra/${CASSANDRA_VERSION}/${CASSANDRA_ARTIFACT}-bin.tar.gz" 8 | RUN apt-get update && \ 9 | apt-get -qq install -y --no-install-recommends wget ca-certificates python && \ 10 | wget -qO - ${CASSANDRA_URL} | tar -xzC /opt && \ 11 | ln -s /opt/${CASSANDRA_ARTIFACT} ${CASSANDRA_HOME} 12 | 13 | RUN apt-get -qq update && \ 14 | apt-get -qq install -y libssl-dev libffi-dev python-dev curl apt-transport-https && \ 15 | echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ xenial main" | tee /etc/apt/sources.list.d/azure-cli.list && \ 16 | curl -L https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \ 17 | apt-get -qq update && \ 18 | apt-get -qq install -y azure-cli 19 | 20 | # install app dependencies 21 | RUN apt-get -qq install -y cron gzip 22 | 23 | # install backup scripts 24 | ADD backup-cassandra-keyspace.sh /app/backup-cassandra-keyspace.sh 25 | ADD docker/run-cqlsh.sh /app/cqlsh 26 | ADD docker/run-backup.sh /app/backup 27 | 28 | CMD /app/backup 29 | 30 | # configuration for azure blob account where backups are stored 31 | ENV USER_FILES_BLOB_ACCOUNT_NAME="" 32 | ENV USER_FILES_BLOB_ACCOUNT_KEY="" 33 | ENV BACKUP_CONTAINER_NAME="backups" 34 | ENV BACKUP_DELETE_LOOKBACK="2 weeks ago" 35 | ENV BACKUP_INTERVAL="2h" 36 | 37 | # a one-node local cassandra is set up via docker-compose, if you wish to use a 38 | # larger cluster (e.g. hosted in Azure), just override this variable with the 39 | # hostname of your cluster 40 | ENV FORTIS_CASSANDRA_HOST="cassandra" 41 | ENV FORTIS_CASSANDRA_PORT="9042" 42 | ENV FORTIS_CASSANDRA_USERNAME="cassandra" 43 | ENV FORTIS_CASSANDRA_PASSWORD="cassandra" 44 | -------------------------------------------------------------------------------- /project-fortis-backup/docker/run-backup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | while :; do 4 | sleep "$BACKUP_INTERVAL" 5 | /app/backup-cassandra-keyspace.sh settings 6 | done 7 | -------------------------------------------------------------------------------- /project-fortis-backup/docker/run-cqlsh.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | "$CASSANDRA_HOME/bin/cqlsh" \ 4 | --request-timeout=3600 \ 5 | --username="$FORTIS_CASSANDRA_USERNAME" \ 6 | --password="$FORTIS_CASSANDRA_PASSWORD" \ 7 | "$FORTIS_CASSANDRA_HOST" \ 8 | "$FORTIS_CASSANDRA_PORT" 9 | -------------------------------------------------------------------------------- /project-fortis-backup/travis/ci.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | pushd "$(dirname "$0")/.." 6 | 7 | # shellcheck disable=SC2046 8 | shellcheck $(find . -name '*.sh') 9 | 10 | popd 11 | -------------------------------------------------------------------------------- /project-fortis-backup/travis/publish.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | log() { 6 | echo "$@" >&2 7 | } 8 | 9 | check_preconditions() { 10 | if [ -z "${TRAVIS_TAG}" ]; then 11 | log "Build is not a tag, skipping publish" 12 | exit 0 13 | fi 14 | if [ -z "${DOCKER_USERNAME}" ] || [ -z "${DOCKER_PASSWORD}" ]; then 15 | log "Docker credentials not provided, unable to publish builds" 16 | exit 1 17 | fi 18 | } 19 | 20 | create_image() { 21 | touch .env-secrets 22 | BUILD_TAG="${TRAVIS_TAG}" docker-compose build project_fortis_backup 23 | } 24 | 25 | publish_image() { 26 | docker login --username="${DOCKER_USERNAME}" --password="${DOCKER_PASSWORD}" 27 | BUILD_TAG="${TRAVIS_TAG}" docker-compose push project_fortis_backup 28 | } 29 | 30 | pushd "$(dirname "$0")/../.." 31 | 32 | check_preconditions 33 | create_image 34 | publish_image 35 | 36 | popd 37 | -------------------------------------------------------------------------------- /project-fortis-interfaces/.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | build/ 3 | travis/ 4 | README.md 5 | .gitignore 6 | -------------------------------------------------------------------------------- /project-fortis-interfaces/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "react-app", 3 | "rules": { 4 | "jsx-a11y/alt-text": "off" 5 | } 6 | } -------------------------------------------------------------------------------- /project-fortis-interfaces/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # node-waf configuration 20 | .lock-wscript 21 | 22 | # Compiled binary addons (http://nodejs.org/api/addons.html) 23 | build/Release 24 | 25 | # Dependency directory 26 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 27 | node_modules 28 | node_modules_old 29 | bower_components 30 | dist 31 | build 32 | tmp 33 | *.pem 34 | *.config.json 35 | 36 | # Localhost config file 37 | .env 38 | 39 | # Editors 40 | .vscode/* 41 | -------------------------------------------------------------------------------- /project-fortis-interfaces/docker/run-react.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | npm run devserver 4 | -------------------------------------------------------------------------------- /project-fortis-interfaces/public/images/OCHA_Logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-interfaces/public/images/OCHA_Logo.png -------------------------------------------------------------------------------- /project-fortis-interfaces/public/images/intl_alert.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-interfaces/public/images/intl_alert.png -------------------------------------------------------------------------------- /project-fortis-interfaces/public/images/stroer_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-interfaces/public/images/stroer_logo.png -------------------------------------------------------------------------------- /project-fortis-interfaces/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Fortis Dashboard 6 | 7 | 8 | 9 | 10 | 21 | 22 | 23 |
24 |
25 | 26 | 27 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/actions/Facts/index.js: -------------------------------------------------------------------------------- 1 | import { SERVICES } from '../../services/Facts'; 2 | import { ResponseHandler } from '../shared'; 3 | 4 | const _methods = { 5 | loadFacts(pipelinekeys, mainTerm, fromDate, toDate, pageState, callback) { 6 | SERVICES.loadFacts( 7 | pipelinekeys, mainTerm, fromDate, toDate, pageState, 8 | (error, response, body) => ResponseHandler(error, response, body, callback)); 9 | }, 10 | }; 11 | 12 | const methods = { FACTS: _methods }; 13 | 14 | module.exports = { 15 | methods 16 | }; 17 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/actions/shared.js: -------------------------------------------------------------------------------- 1 | function ResponseHandler (error, response, body, callback) { 2 | if (!error && response.statusCode === 200 && body.data && !body.errors) { 3 | callback(undefined, body.data); 4 | } else { 5 | const code = response ? response.statusCode : 500; 6 | const message = `GraphQL call failed: ${formatGraphQlErrorDetails(body, error)}`; 7 | callback({ code, message }, undefined); 8 | } 9 | } 10 | 11 | function formatGraphQlErrorDetails(body, error) { 12 | if (!body) { 13 | return error; 14 | } 15 | 16 | if (!body.errors) { 17 | return body; 18 | } 19 | 20 | const errors = Array.from(new Set(body.errors.map(error => error.message))); 21 | errors.sort(); 22 | return errors.join("; "); 23 | } 24 | 25 | const DataSources = (source, enabledStreams) => enabledStreams.has(source) ? enabledStreams.get(source).sourceValues : undefined; 26 | 27 | module.exports = { 28 | ResponseHandler, 29 | DataSources 30 | }; -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Admin/shared.js: -------------------------------------------------------------------------------- 1 | const DEFAULT_COLUMN = { 2 | editable: false, 3 | filterable: false, 4 | resizable: true 5 | }; 6 | 7 | function getColumns(columnValues) { 8 | return columnValues.map(value => Object.assign({}, DEFAULT_COLUMN, value)); 9 | } 10 | 11 | module.exports = { 12 | getColumns 13 | }; -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Footer.js: -------------------------------------------------------------------------------- 1 | import React, { Component } from 'react'; 2 | import logo from '../images/MSFT_logo_png.png'; 3 | import '../styles/Footer.css'; 4 | 5 | class Footer extends Component { 6 | render() { 7 | return ( 8 | 18 | ); 19 | } 20 | 21 | } 22 | 23 | export default Footer; 24 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Graphics/GraphCard.js: -------------------------------------------------------------------------------- 1 | import React, { Component } from 'react'; 2 | import {Card, CardHeader, CardTitle, CardActions, CardMedia} from 'material-ui/Card'; 3 | 4 | import styles from '../../styles/Graphics/styles'; 5 | import '../../styles/Graphics/GraphCard.css'; 6 | 7 | export default class GraphCard extends Component { 8 | render() { 9 | return ( 10 | 11 | { 12 | this.props.cardHeader ? 13 | : undefined 14 | } 15 | 16 | {this.props.children} 17 | 18 | { 19 | this.props.cardTitle ? : undefined 20 | } 21 | { 22 | this.props.cardActions ? 23 | 24 | {this.props.cardActions} 25 | : undefined 26 | } 27 | 28 | ); 29 | } 30 | } -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Graphics/NoData.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import IconButton from 'material-ui/IconButton'; 3 | import ActionHighlightOff from 'material-ui/svg-icons/action/highlight-off'; 4 | import { fullWhite } from 'material-ui/styles/colors'; 5 | 6 | export default class NoData extends React.Component { 7 | render() { 8 | return ( 9 | 10 | 11 | 12 | ); 13 | } 14 | }; 15 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Graphics/Sentiment.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { getSentimentAttributes } from '../Insights/shared'; 3 | 4 | export default class Sentiment extends React.Component { 5 | render() { 6 | const { value, showGraph } = this.props; 7 | const sentiment = getSentimentAttributes(value); 8 | const className = `material-icons sentimentIcon ${sentiment.style}`; 9 | const sentimentIcon = {sentiment.icon}; 10 | const displayValue = parseFloat(value * 10).toFixed(0); 11 | const graphbarClassname = `sentimentGraphBar ${sentiment.style}`; 12 | 13 | if (!showGraph) { 14 | return
{sentimentIcon}
; 15 | } 16 | 17 | return ( 18 |
19 |
20 |
21 | { displayValue } 22 |
23 | {sentimentIcon} 24 |
25 |
26 | ); 27 | } 28 | }; 29 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Graphics/Timeline.js: -------------------------------------------------------------------------------- 1 | import React, { Component } from 'react'; 2 | import numeralLibs from 'numeral'; 3 | import { AreaChart, XAxis, CartesianGrid, Brush, YAxis, Tooltip, Legend, ResponsiveContainer } from 'recharts'; 4 | 5 | const toNumericDisplay = (decimal, fixed = 0) => { 6 | return numeralLibs(decimal).format(decimal > 1000 ? '+0.0a' : '0a'); 7 | }; 8 | 9 | function highestCountFirst(item1, item2) { 10 | if (!item1 || !item1.value) return -1; 11 | if (!item2 || !item2.value) return 1; 12 | return item1.value < item2.value; 13 | } 14 | 15 | export default class Timeline extends Component { 16 | render() { 17 | return ( 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | {this.props.children} 28 | 29 | 30 | 31 | 32 | {this.props.children} 33 | 34 | 35 | ); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Graphics/WordCloud.js: -------------------------------------------------------------------------------- 1 | import React, { Component } from 'react'; 2 | import { ResponsiveContainer } from 'recharts'; 3 | import { TagCloud } from "react-tagcloud"; 4 | 5 | const style = { 6 | display: 'table' 7 | }; 8 | 9 | /** 10 | * Render the cloud using D3. Not stateless, because async rendering of d3-cloud 11 | */ 12 | export default class WordCloud extends Component { 13 | constructor(props) { 14 | super(props); 15 | this.state = { 16 | cloudDimensions: [] 17 | }; 18 | } 19 | 20 | render() { 21 | const { words, minSize, maxSize, customRenderer, onClick } = this.props; 22 | 23 | return ( 24 | 25 | 31 | 32 | ); 33 | } 34 | } -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Header/index.js: -------------------------------------------------------------------------------- 1 | import Header from './Header'; 2 | 3 | export default Header; -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Insights/CategoryPicker.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ContentFilterList from 'material-ui/svg-icons/content/filter-list'; 3 | import { fullWhite } from 'material-ui/styles/colors'; 4 | import DrawerActionsIconButton from './DrawerActionsIconButton'; 5 | 6 | const CATEGORY_ALL = ''; 7 | 8 | export default class CategoryPicker extends React.Component { 9 | formatCategory = (category) => category || 'all'; 10 | formatText = (category) => `Reload with category '${this.formatCategory(category)}'`; 11 | formatLabel = (category) => `Category: ${this.formatCategory(category)}`; 12 | formatTooltip = (category) => `Current category: '${this.formatCategory(category)}'. Click to change category`; 13 | 14 | render() { 15 | const { category, allCategories, tooltipPosition, onChangeCategory } = this.props; 16 | 17 | return ( 18 | } 26 | tooltipPosition={tooltipPosition} 27 | /> 28 | ); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Insights/DrawerActionsIconButton.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Drawer from 'material-ui/Drawer'; 3 | import IconButton from 'material-ui/IconButton'; 4 | import MenuItem from 'material-ui/MenuItem'; 5 | 6 | export default class DrawerActionsIconButton extends React.Component { 7 | constructor(props) { 8 | super(props); 9 | this.state = { 10 | open: false 11 | }; 12 | } 13 | 14 | onClick = (value) => { 15 | this.props.onClick(value); 16 | this.setState({ open: false }); 17 | } 18 | 19 | handleDrawerToggle = () => { 20 | this.setState({ open: !this.state.open }); 21 | } 22 | 23 | handleDrawerChange = (open) => { 24 | this.setState({ open }); 25 | } 26 | 27 | renderMenuItems() { 28 | return this.props.items.map((item) => 29 | this.onClick(item)} 35 | /> 36 | ); 37 | } 38 | 39 | render() { 40 | const { item, tooltipPosition, icon } = this.props; 41 | 42 | return ( 43 |
44 | 45 | {icon} 46 | 47 | 51 | {this.renderMenuItems()} 52 | 53 |
54 | ); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Insights/HeatmapToggle.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import IconButton from 'material-ui/IconButton/IconButton'; 3 | import NavigationFullscreen from 'material-ui/svg-icons/navigation/fullscreen'; 4 | import NavigationFullscreenExit from 'material-ui/svg-icons/navigation/fullscreen-exit'; 5 | import { fullWhite } from 'material-ui/styles/colors'; 6 | 7 | export default class HeatmapToggle extends React.Component { 8 | constructor(props) { 9 | super(props); 10 | 11 | this.state = { 12 | expanded: false 13 | }; 14 | } 15 | 16 | onClick = () => { 17 | this.props.onClick(); 18 | this.setState({ 19 | expanded: !this.state.expanded 20 | }); 21 | } 22 | 23 | render() { 24 | const { tooltipOn, tooltipOff, tooltipPosition } = this.props; 25 | const { expanded } = this.state; 26 | 27 | return ( 28 |
29 | 30 | {expanded ? : } 31 | 32 |
33 | ); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Insights/LanguagePicker.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ActionLanguage from 'material-ui/svg-icons/action/language'; 3 | import { fullWhite } from 'material-ui/styles/colors'; 4 | import DrawerActionsIconButton from './DrawerActionsIconButton'; 5 | 6 | export default class LanguagePicker extends React.Component { 7 | formatText = (language) => `Set language to '${language}'`; 8 | formatLabel = (language) => `Language: ${language}`; 9 | formatTooltip = (language) => `Current language: '${language}'. Click to change language`; 10 | 11 | render() { 12 | const { language, supportedLanguages, tooltipPosition, onChangeLanguage } = this.props; 13 | 14 | return ( 15 | } 23 | tooltipPosition={tooltipPosition} 24 | /> 25 | ); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Insights/Layouts/index.js: -------------------------------------------------------------------------------- 1 | const layout = { 2 | "lg": [ 3 | { "i": "topics", "x": 0, "y": 0, "w": 4, "h": 8}, 4 | { "i": "locations", "x": 4, "y": 0, "w": 4, "h": 8}, 5 | { "i": "sources", "x": 8, "y": 0, "w": 4, "h": 8}, 6 | { "i": "timeline", "x": 12, "y": 0, "w": 12, "h": 8 }, 7 | { "i": "watchlist", "x": 0, "y": 9, "w": 5, "h": 16}, 8 | { "i": "heatmap", "x": 5, "y": 6, "w": 14, "h": 16 }, 9 | { "i": "newsfeed", "x": 19, "y": 6, "w": 5, "h": 16 } 10 | ] 11 | }; 12 | 13 | const layoutCollapsed = { 14 | "lg": [ 15 | { "i": "watchlist", "x": 0, "y": 0, "w": 4, "h": 22, static: true }, 16 | { "i": "heatmap", "x": 4, "y": 0, "w": 14, "h": 22, static: true }, 17 | { "i": "newsfeed", "x": 18, "y": 0, "w": 6, "h": 22, static: true } 18 | ] 19 | }; 20 | 21 | const defaultLayout = { layout, layoutCollapsed }; 22 | 23 | module.exports = { 24 | defaultLayout 25 | }; 26 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Insights/MapBoundingReset.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import IconButton from 'material-ui/IconButton/IconButton'; 3 | import Map from 'material-ui/svg-icons/maps/map'; 4 | import { fullWhite } from 'material-ui/styles/colors'; 5 | 6 | export default class MapBoundingReset extends React.Component { 7 | render() { 8 | const { tooltipPosition } = this.props; 9 | const tooltip = `Click to reset map boundaries.`; 10 | 11 | return ( 12 |
13 | 14 | 15 | 16 |
17 | ); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Insights/Maps/TileLayer.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { TileLayer as LeafletTileLayer } from 'react-leaflet'; 3 | import { reactAppMapboxTileLayerUrl } from '../../../config'; 4 | 5 | export class TileLayer extends React.Component { 6 | render() { 7 | return ( 8 | 14 | ); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Insights/Maps/style.scss: -------------------------------------------------------------------------------- 1 | .marker-cluster { 2 | &-styled { 3 | @import './node_modules/leaflet.markercluster/dist/MarkerCluster.Default'; 4 | } 5 | 6 | &-animated { 7 | @import './node_modules/leaflet.markercluster/dist/MarkerCluster'; 8 | } 9 | 10 | &-group { 11 | display: none; 12 | } 13 | } -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Insights/ShareButton.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import IconButton from 'material-ui/IconButton/IconButton'; 3 | import Snackbar from 'material-ui/Snackbar'; 4 | import SocialShare from 'material-ui/svg-icons/social/share'; 5 | import { fullWhite } from 'material-ui/styles/colors'; 6 | import { CopyToClipboard } from 'react-copy-to-clipboard'; 7 | 8 | export default class ShareButton extends React.Component { 9 | constructor(props) { 10 | super(props); 11 | 12 | this.state = { 13 | notification: '', 14 | copied: false 15 | }; 16 | } 17 | 18 | onCopy = () => { 19 | this.setState({ copied: true, notification: this.props.notification }); 20 | } 21 | 22 | onRequestClose = () => { 23 | this.setState({ notification: '' }); 24 | } 25 | 26 | render() { 27 | const { tooltipPosition, tooltip, link } = this.props; 28 | const { notification } = this.state; 29 | 30 | return ( 31 |
32 | 33 | 34 | 35 | 36 | 37 | 43 |
44 | ); 45 | } 46 | }; 47 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Insights/Subheader.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | const Subheader = (props, context) => { 4 | const { 5 | children, 6 | inset, 7 | style, 8 | ...other, 9 | } = props; 10 | 11 | const styles = { 12 | root: { 13 | boxSizing: 'border-box', 14 | fontSize: 14, 15 | lineHeight: '48px', 16 | paddingLeft: inset ? 72 : 16, 17 | width: '100%', 18 | }, 19 | }; 20 | 21 | return ( 22 |
23 | {children} 24 |
25 | ); 26 | }; 27 | 28 | export default Subheader; -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/Insights/TermFilter.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import createReactClass from 'create-react-class'; 3 | import Fluxxor from 'fluxxor'; 4 | import Multiselect from 'react-widgets/lib/Multiselect'; 5 | 6 | const FluxMixin = Fluxxor.FluxMixin(React), 7 | StoreWatchMixin = Fluxxor.StoreWatchMixin("DataStore"); 8 | 9 | export const TermFilter = createReactClass({ 10 | mixins: [FluxMixin, StoreWatchMixin], 11 | 12 | getInitialState(){ 13 | return {}; 14 | }, 15 | 16 | onFilterChange(filters){ 17 | this.getFlux().actions.DASHBOARD.changeTermsFilter(filters); 18 | }, 19 | 20 | getStateFromFlux() { 21 | return this.getFlux().store("DataStore").getState(); 22 | }, 23 | 24 | FilterEnabledTerms(){ 25 | let filteredTerms = []; 26 | 27 | for (var [term, value] of this.state.associatedKeywords.entries()) { 28 | if(value.enabled){ 29 | filteredTerms.push(term); 30 | } 31 | } 32 | 33 | return filteredTerms; 34 | }, 35 | 36 | render(){ 37 | return ( 38 |
39 | { 40 | this.props.data && this.props.data.length > 0 ? 41 | 43 | : undefined 44 | } 45 |
46 | ); 47 | } 48 | }); 49 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/dialogs/DialogBox.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Dialog from 'material-ui/Dialog'; 3 | import FlatButton from 'material-ui/FlatButton'; 4 | import { SERVICES } from '../../services/Dashboard'; 5 | import EventDetails from './EventDetails'; 6 | import '../../styles/Insights/DialogBox.css'; 7 | 8 | const dialogWideStyle = { 9 | width: '80%', 10 | height: '80%', 11 | maxWidth: 'none' 12 | }; 13 | 14 | export default class DialogBox extends React.Component { 15 | constructor(props) { 16 | super(props); 17 | 18 | this.state = { 19 | open: false, 20 | enabledStreams: props.enabledStreams 21 | }; 22 | } 23 | 24 | open = (id) => { 25 | this._loadDetail(id); 26 | } 27 | 28 | close = () => { 29 | this.setState({ open: false }); 30 | } 31 | 32 | _loadDetail(id){ 33 | SERVICES.FetchMessageDetail(id, (error, response, body) => { 34 | if (error || response.statusCode !== 200 || !body.data || !body.data.event ) { 35 | console.error("Failed to fetch details for id:", id, error); 36 | return; 37 | } 38 | 39 | const payload = Object.assign({}, body.data.event, {open: true}); 40 | this.setState({...payload}); 41 | }); 42 | } 43 | 44 | render() { 45 | const { open } = this.state; 46 | const { language, settings } = this.props; 47 | 48 | const actions = [ 49 | , 54 | ]; 55 | 56 | return ( 57 | 65 |
66 | 71 |
72 | 73 | ); 74 | } 75 | }; 76 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/dialogs/Highlighter.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { sanitize } from '../../utils/HtmlSanitizer'; 3 | 4 | export class Highlighter extends React.Component { 5 | sanitizeHtml(html) { 6 | const { extraClasses } = this.props; 7 | 8 | html = sanitize(html, node => { 9 | const classesToAdd = extraClasses[node.nodeName.toLowerCase()] || []; 10 | classesToAdd.forEach(className => { 11 | node.classList += className; 12 | }); 13 | return node; 14 | }); 15 | 16 | return html; 17 | } 18 | 19 | highlightHtml(html) { 20 | const { highlightWords } = this.props; 21 | 22 | const toHighlight = highlightWords.slice().sort((a, b) => a.word.length < b.word.length); 23 | 24 | toHighlight.forEach(({ word, className }) => { 25 | html = html.replace(new RegExp(word, 'ig'), `${word}`); 26 | }); 27 | 28 | return html; 29 | } 30 | 31 | renderHtml() { 32 | let html = this.props.textToHighlight; 33 | html = this.sanitizeHtml(html); 34 | html = this.highlightHtml(html); 35 | return html; 36 | } 37 | 38 | render() { 39 | return ( 40 | 41 | ); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/components/dialogs/MapViewPort.js: -------------------------------------------------------------------------------- 1 | import geoViewport from '@mapbox/geo-viewport'; 2 | import PropTypes from 'prop-types'; 3 | import bbox from '@turf/bbox'; 4 | import React from 'react'; 5 | import { reactAppMapboxTileServerUrl } from '../../config'; 6 | 7 | const DEFAULT_ZOOM = 8; 8 | const PIN_COLOR = '0000FF'; 9 | const PIN_SIZE = 'l'; 10 | 11 | export default class MapViewPort extends React.Component { 12 | render() { 13 | const geoJsonFeatures = this.props.coordinates.map(coordinatePair => Object.assign({}, { 14 | "type": "Feature", 15 | "properties": {}, 16 | "geometry": { 17 | "type": "Point", 18 | "coordinates": coordinatePair 19 | } 20 | } 21 | )); 22 | 23 | const geoJson = Object.assign({}, {"type": "FeatureCollection", "features": geoJsonFeatures}); 24 | const bounds = bbox(geoJson); 25 | const vp = geoViewport.viewport(bounds, this.props.mapSize); 26 | const pins = this.props.coordinates.map(coordinatePair => `pin-${PIN_SIZE}-cross+${PIN_COLOR}(${coordinatePair.join(",")})`); 27 | const mapImageSrc = `${reactAppMapboxTileServerUrl}/${pins.join(',')}/${vp.center.join(',')},${pins.length > 1 ? vp.zoom : DEFAULT_ZOOM}@2x/${this.props.mapSize.join('x')}.png?access_token=${this.props.accessToken}`; 28 | 29 | return ( 30 | 31 | ); 32 | } 33 | } 34 | 35 | MapViewPort.propTypes = { 36 | coordinates: PropTypes.array.isRequired, 37 | mapSize: PropTypes.array.isRequired 38 | } 39 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/config.js: -------------------------------------------------------------------------------- 1 | const reactAppMapboxTileLayerUrl = process.env.REACT_APP_MAPBOX_TILE_LAYER_URL || 'https://api.mapbox.com/styles/v1/mapbox/satellite-streets-v10/tiles/256/{z}/{x}/{y}'; 2 | 3 | const reactAppMapboxTileServerUrl = process.env.REACT_APP_MAPBOX_TILE_SERVER_URL || 'https://api.mapbox.com/v4/mapbox.streets'; 4 | 5 | const reactAppAdTokenStoreKey = process.env.REACT_APP_AD_TOKEN_STORE_KEY || 'Fortis.AD.Token'; 6 | 7 | const reactAppAdClientId = process.env.REACT_APP_AD_CLIENT_ID || ''; 8 | 9 | const reactAppServiceHost = process.env.REACT_APP_SERVICE_HOST; 10 | if (!reactAppServiceHost) console.error('Service host is not defined!'); 11 | 12 | module.exports = { 13 | reactAppMapboxTileLayerUrl, 14 | reactAppMapboxTileServerUrl, 15 | reactAppAdClientId, 16 | reactAppAdTokenStoreKey, 17 | reactAppServiceHost 18 | }; 19 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/images/MSFT_logo_png.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-interfaces/src/images/MSFT_logo_png.png -------------------------------------------------------------------------------- /project-fortis-interfaces/src/images/OCHA_Logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-interfaces/src/images/OCHA_Logo.png -------------------------------------------------------------------------------- /project-fortis-interfaces/src/images/layers-2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-interfaces/src/images/layers-2x.png -------------------------------------------------------------------------------- /project-fortis-interfaces/src/images/layers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-interfaces/src/images/layers.png -------------------------------------------------------------------------------- /project-fortis-interfaces/src/images/marker-icon-2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-interfaces/src/images/marker-icon-2x.png -------------------------------------------------------------------------------- /project-fortis-interfaces/src/images/marker-icon-red.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-interfaces/src/images/marker-icon-red.png -------------------------------------------------------------------------------- /project-fortis-interfaces/src/images/marker-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-interfaces/src/images/marker-icon.png -------------------------------------------------------------------------------- /project-fortis-interfaces/src/images/marker-shadow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-interfaces/src/images/marker-shadow.png -------------------------------------------------------------------------------- /project-fortis-interfaces/src/images/nav_bg2.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/images/partner_catalyst_icon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-interfaces/src/images/partner_catalyst_icon.ico -------------------------------------------------------------------------------- /project-fortis-interfaces/src/images/partner_catalyst_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-interfaces/src/images/partner_catalyst_icon.png -------------------------------------------------------------------------------- /project-fortis-interfaces/src/images/partner_catalyst_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-interfaces/src/images/partner_catalyst_logo.png -------------------------------------------------------------------------------- /project-fortis-interfaces/src/images/select.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-interfaces/src/images/select.png -------------------------------------------------------------------------------- /project-fortis-interfaces/src/index.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Fluxxor from 'fluxxor'; 3 | import BrowserDetection from 'react-browser-detection'; 4 | import { default as ReactDOM } from 'react-dom'; 5 | import { Router, hashHistory } from 'react-router'; 6 | import { DataStore } from './stores/DataStore'; 7 | import { AdminStore } from './stores/AdminStore'; 8 | import { methods as DashboardActions } from './actions/Dashboard'; 9 | import { methods as AdminActions } from './actions/Admin'; 10 | import { methods as FactsActions } from './actions/Facts'; 11 | import { routes } from './routes/routes'; 12 | import UnsupportedBrowserPage from './routes/UnsupportedBrowserPage'; 13 | import constants from './actions/constants'; 14 | import 'bootstrap/dist/css/bootstrap.css'; 15 | 16 | const userProfile = constants.USER_PROFILE; 17 | 18 | const stores = { 19 | DataStore: new DataStore(userProfile), 20 | AdminStore: new AdminStore(), 21 | }; 22 | 23 | const flux = new Fluxxor.Flux(stores, Object.assign({}, DashboardActions, AdminActions, FactsActions)); 24 | 25 | function createElement(Component, props) { 26 | props.flux = flux; 27 | return 28 | }; 29 | 30 | function renderApp() { 31 | return ; 32 | } 33 | 34 | function renderUnsupported() { 35 | return ; 36 | } 37 | 38 | const browserConfig = { 39 | chrome: renderApp, 40 | firefox: renderApp, 41 | default: renderUnsupported 42 | }; 43 | 44 | ReactDOM.render( 45 | { browserConfig }, 46 | document.getElementById('app') 47 | ); 48 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/routes/DashboardPage.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import createReactClass from 'create-react-class'; 3 | import Fluxxor from 'fluxxor'; 4 | import Dashboard from '../components/Insights/Dashboard'; 5 | 6 | const FluxMixin = Fluxxor.FluxMixin(React); 7 | const StoreWatchMixin = Fluxxor.StoreWatchMixin("DataStore"); 8 | 9 | export const DashboardPage = createReactClass({ 10 | mixins: [FluxMixin, StoreWatchMixin], 11 | 12 | getStateFromFlux() { 13 | return this.getFlux().store("DataStore").getState(); 14 | }, 15 | 16 | propertyLiterals() { 17 | const { dataSource, bbox, termFilters, maintopic, externalsourceid, datetimeSelection, 18 | fromDate, toDate, language, zoomLevel, settings, timespanType, enabledStreams, 19 | conjunctivetopics, heatmapTileIds, timeSeriesGraphData, popularLocations, popularTerms, 20 | timeSeriesCsv, popularLocationsCsv, popularTermsCsv, topSourcesCsv, category, 21 | allCategories, 22 | topSources, trustedSources, fullTermList, selectedplace, dashboardIsLoadedFromShareLink } = this.getStateFromFlux(); 23 | 24 | return { dataSource, maintopic, termFilters, bbox, enabledStreams, 25 | externalsourceid, datetimeSelection, fromDate, toDate, language, 26 | zoomLevel, settings, timespanType, heatmapTileIds, category, 27 | conjunctivetopics, timeSeriesGraphData, popularLocations, popularTerms, 28 | timeSeriesCsv, popularLocationsCsv, popularTermsCsv, topSourcesCsv, 29 | allCategories, 30 | topSources, trustedSources, fullTermList, selectedplace, dashboardIsLoadedFromShareLink }; 31 | }, 32 | 33 | render() { 34 | if (!this.state.bbox.length) { 35 | return
; 36 | } 37 | 38 | return ( 39 |
40 | 41 |
42 | )} 43 | }); -------------------------------------------------------------------------------- /project-fortis-interfaces/src/routes/FactsPage.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import createReactClass from 'create-react-class'; 3 | import Fluxxor from 'fluxxor'; 4 | import { FactsList } from '../components/Facts/FactsList'; 5 | import '../styles/Facts/Facts.css'; 6 | 7 | const FluxMixin = Fluxxor.FluxMixin(React); 8 | const StoreWatchMixin = Fluxxor.StoreWatchMixin("DataStore"); 9 | 10 | export const FactsPage = createReactClass({ 11 | mixins: [FluxMixin, StoreWatchMixin], 12 | 13 | getStateFromFlux() { 14 | return this.getFlux().store("DataStore").getState(); 15 | }, 16 | 17 | render() { 18 | return ( 19 |
20 | 21 |
22 | ); 23 | } 24 | }); -------------------------------------------------------------------------------- /project-fortis-interfaces/src/routes/NotFoundPage.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | export class NotFoundPage extends React.Component { 4 | render() { 5 | return ( 6 |
7 |

This page does not exist.

8 |
9 | ); 10 | } 11 | }; 12 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/routes/UnsupportedBrowserPage.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | export default class UnsupportedBrowserPage extends React.Component { 4 | render() { 5 | return ( 6 |
7 |

Your browser is not supported. Please try using Chrome or Firefox

8 |
9 | ); 10 | } 11 | }; 12 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/routes/routes.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Route } from 'react-router' 3 | import { AppPage } from './AppPage'; 4 | import { DashboardPage } from './DashboardPage'; 5 | import { FactsPage } from './FactsPage'; 6 | import { AdminPage } from './AdminPage'; 7 | import { NotFoundPage } from './NotFoundPage'; 8 | 9 | export const routes = ( 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | ); 18 | 19 | export function changeCategory(category) { 20 | window.location = category ? `#/dashboard/${category}` : '#/dashboard'; 21 | window.location.reload(); 22 | } 23 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/services/Facts/index.js: -------------------------------------------------------------------------------- 1 | import * as FactsFragments from '../graphql/fragments/Facts'; 2 | import * as FactsQueries from '../graphql/queries/Facts'; 3 | import { fetchGqlData, MESSAGES_ENDPOINT } from '../shared'; 4 | 5 | export const SERVICES = { 6 | loadFacts(pipelinekeys, mainTerm, fromDate, toDate, pageState, callback) { 7 | const selectionFragments = ` 8 | ${FactsFragments.factsFragment} 9 | `; 10 | 11 | const query = ` 12 | ${selectionFragments} 13 | ${FactsQueries.FactsQuery} 14 | `; 15 | 16 | const variables = { 17 | mainTerm, 18 | fromDate, 19 | toDate, 20 | pageState, 21 | pipelinekeys 22 | }; 23 | 24 | fetchGqlData(MESSAGES_ENDPOINT, { variables, query }, callback); 25 | }, 26 | } -------------------------------------------------------------------------------- /project-fortis-interfaces/src/services/featureService.js: -------------------------------------------------------------------------------- 1 | import request from 'request'; 2 | import { reactAppServiceHost } from '../config'; 3 | import { auth } from './shared'; 4 | 5 | export function fetchLocationsFromFeatureService(bbox, matchName, namespace, callback) { 6 | if (!matchName || !bbox || bbox.length !== 4) { 7 | return callback(null, []); 8 | } 9 | 10 | const url = `${reactAppServiceHost}/api/featureservice/features/bbox/${bbox.join('/')}?include=bbox,centroid&filter_namespace=${namespace}&filter_name=${matchName}`; 11 | 12 | request({ 13 | url, 14 | headers: { 'Authorization': `Bearer ${auth.token}` }, 15 | json: true 16 | }, (err, response) => callback(err, (response && response.body && response.body.features) || [])); 17 | } 18 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/services/graphql/fragments/Admin/index.js: -------------------------------------------------------------------------------- 1 | export const users = `fragment UsersView on UserCollection { 2 | users { 3 | identifier, 4 | role 5 | } 6 | }` 7 | 8 | export const siteSettingsFragment = `fragment FortisSiteDefinitionView on SiteCollection { 9 | site { 10 | name 11 | properties { 12 | targetBbox 13 | defaultZoomLevel 14 | logo 15 | title 16 | fbToken 17 | mapSvcToken 18 | defaultLocation 19 | defaultLanguage 20 | featureservicenamespace 21 | storageConnectionString 22 | featuresConnectionString 23 | supportedLanguages 24 | } 25 | } 26 | accessLevels 27 | }`; 28 | 29 | export const site = `fragment SiteView on SiteCollection { 30 | site { 31 | name 32 | properties { 33 | targetBbox 34 | defaultZoomLevel 35 | logo 36 | title 37 | defaultLocation 38 | defaultLanguage 39 | supportedLanguages 40 | featureservicenamespace 41 | mapSvcToken 42 | translationSvcToken 43 | cogSpeechSvcToken 44 | cogVisionSvcToken 45 | cogTextSvcToken 46 | } 47 | } 48 | }`; 49 | 50 | export const topics = `fragment TopicsView on SiteTerms { 51 | edges { 52 | topicid 53 | name 54 | translatedname 55 | namelang 56 | translatednamelang 57 | category 58 | translations { 59 | key 60 | value 61 | } 62 | } 63 | }`; 64 | 65 | export const blacklist = `fragment BlacklistView on BlacklistCollection { 66 | filters { 67 | id 68 | filteredTerms 69 | isLocation 70 | } 71 | }`; 72 | 73 | export const trustedsources = `fragment TrustedSourcesView on SourceCollection { 74 | sources { 75 | rowKey 76 | displayname 77 | externalsourceid 78 | pipelinekey 79 | } 80 | }`; 81 | 82 | export const streams = `fragment StreamsView on StreamCollection { 83 | streams { 84 | streamId 85 | pipelineKey 86 | pipelineLabel 87 | pipelineIcon 88 | streamFactory 89 | params { 90 | key 91 | value 92 | } 93 | enabled 94 | } 95 | }`; 96 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/services/graphql/fragments/Facts/index.js: -------------------------------------------------------------------------------- 1 | export const factsFragment = ` 2 | fragment FortisFactsView on FeatureCollection { 3 | features { 4 | properties { 5 | messageid, 6 | summary, 7 | edges, 8 | eventtime, 9 | sourceeventid, 10 | externalsourceid, 11 | sentiment, 12 | language, 13 | pipelinekey, 14 | link, 15 | title, 16 | link 17 | } 18 | coordinates 19 | } 20 | pageState 21 | } 22 | `.trim(); -------------------------------------------------------------------------------- /project-fortis-interfaces/src/services/graphql/mutations/Admin/index.js: -------------------------------------------------------------------------------- 1 | export const restartPipeline = `mutation RestartPipeline { 2 | restartPipeline 3 | }`; 4 | 5 | export const addUsers = `mutation AddUsers($input: UserListInput!) { 6 | addUsers(input: $input) { 7 | ...UsersView 8 | } 9 | }`; 10 | 11 | export const removeUsers = `mutation RemoveUsers($input: UserListInput!) { 12 | removeUsers(input: $input) { 13 | ...UsersView 14 | } 15 | }`; 16 | 17 | export const editSite = `mutation EditSite($input: EditableSiteSettings!) { 18 | editSite(input: $input) { 19 | name 20 | } 21 | }`; 22 | 23 | export const saveTopics = `mutation SaveTopics($input: MutatedTerms) { 24 | addKeywords(input: $input) { 25 | ...TopicsView 26 | } 27 | }`; 28 | 29 | export const removeTopics = `mutation RemoveTopics($input: MutatedTerms) { 30 | removeKeywords(input: $input) { 31 | ...TopicsView 32 | } 33 | }`; 34 | 35 | export const saveTrustedSources = `mutation SaveTrustedSources($input: SourceListInput!) { 36 | addTrustedSources(input: $input) { 37 | ...TrustedSourcesView 38 | } 39 | }`; 40 | 41 | export const removeTrustedSources = `mutation RemoveTrustedSources($input: SourceListInput!) { 42 | removeTrustedSources(input: $input) { 43 | ...TrustedSourcesView 44 | } 45 | }`; 46 | 47 | export const saveBlacklists = `mutation SaveBlacklists($input: BlacklistTermDefintion!) { 48 | modifyBlacklist(input: $input) { 49 | ...BlacklistView 50 | } 51 | }`; 52 | 53 | export const removeBlacklists = `mutation RemoveBlacklists($input: BlacklistTermDefintion!) { 54 | removeBlacklist(input: $input) { 55 | ...BlacklistView 56 | } 57 | }`; 58 | 59 | export const saveStreams = `mutation SaveStreams($input: StreamListInput!) { 60 | modifyStreams(input: $input) { 61 | ...StreamsView 62 | } 63 | }`; 64 | 65 | export const removeStreams = `mutation RemoveStreams($input: StreamListInput!) { 66 | removeStreams(input: $input) { 67 | ...StreamsView 68 | } 69 | }`; -------------------------------------------------------------------------------- /project-fortis-interfaces/src/services/graphql/queries/Admin/index.js: -------------------------------------------------------------------------------- 1 | export const getUsers = `query users { 2 | users { 3 | ...UsersView 4 | } 5 | }`; 6 | 7 | export const getAdminSite = `sites { 8 | ...FortisSiteDefinitionView 9 | }`; 10 | 11 | export const getAdminSiteDefinition = `query sites { 12 | site { 13 | name 14 | properties { 15 | targetBbox 16 | defaultZoomLevel 17 | logo 18 | title 19 | mapSvcToken 20 | fbToken 21 | defaultLocation 22 | defaultLanguage 23 | storageConnectionString 24 | featuresConnectionString 25 | supportedLanguages 26 | } 27 | } 28 | }`; 29 | 30 | export const getPipelineTerms = `siteTerms(translationLanguage:$translationLanguage, category: $category){ 31 | edges { 32 | name 33 | translatedname 34 | } 35 | categories { 36 | name 37 | } 38 | }`; 39 | 40 | export const getPipelineStreams = `streams { 41 | streams { 42 | pipelineKey 43 | pipelineIcon 44 | pipelineLabel 45 | enabled 46 | } 47 | }`; 48 | 49 | export const getPipelineWatchlist = `query PipelineDefintion($translationLanguage: String!, $category: String) { 50 | terms: ${getPipelineTerms} 51 | }`; 52 | 53 | export const getPipelineDefinition = `query PipelineDefintion($translationLanguage: String, $category: String) { 54 | terms: ${getPipelineTerms} 55 | streams: ${getPipelineStreams} 56 | configuration: ${getAdminSite} 57 | }`; 58 | 59 | export const getSite = `query Sites { 60 | sites { 61 | ...SiteView 62 | } 63 | }`; 64 | 65 | export const getTopics = `query SiteTerms($translationLanguage: String, $category: String) { 66 | siteTerms(translationLanguage: $translationLanguage, category: $category) { 67 | ...TopicsView 68 | } 69 | }`; 70 | 71 | export const getTrustedSources = `query TrustedSources { 72 | trustedSources { 73 | ...TrustedSourcesView 74 | } 75 | }`; 76 | 77 | export const getBlacklists = `query Blacklist { 78 | termBlacklist { 79 | ...BlacklistView 80 | } 81 | }`; 82 | 83 | export const getStreams = `query Streams { 84 | streams { 85 | ...StreamsView 86 | } 87 | }`; -------------------------------------------------------------------------------- /project-fortis-interfaces/src/services/graphql/queries/Facts/index.js: -------------------------------------------------------------------------------- 1 | export const FactsQuery = ` 2 | query FetchFacts($pipelinekeys: [String]!, $mainTerm: String!, $fromDate: String!, $toDate: String!) { 3 | facts: byPipeline(pipelinekeys: $pipelinekeys, mainTerm: $mainTerm, fromDate: $fromDate, toDate: $toDate) { 4 | ...FortisFactsView 5 | } 6 | } 7 | `.trim(); -------------------------------------------------------------------------------- /project-fortis-interfaces/src/services/shared.js: -------------------------------------------------------------------------------- 1 | import request from 'request'; 2 | import { reactAppServiceHost, reactAppAdTokenStoreKey } from '../config'; 3 | 4 | const auth = { token: null }; // token will get set as soon as it's available 5 | 6 | function fetchGqlData(endpoint, { query, variables }, callback) { 7 | request({ 8 | url: `${reactAppServiceHost}/api/${endpoint}`, 9 | method: 'POST', 10 | json: true, 11 | withCredentials: false, 12 | headers: { 'Authorization': `Bearer ${auth.token}` }, 13 | body: { query, variables } 14 | }, (error, response, body) => { 15 | if (response && response.statusCode === 401) { 16 | auth.token = null; 17 | localStorage.removeItem(reactAppAdTokenStoreKey); 18 | } 19 | callback(error, response, body); 20 | }); 21 | } 22 | 23 | const MESSAGES_ENDPOINT = 'messages'; 24 | const TILES_ENDPOINT = 'tiles'; 25 | const EDGES_ENDPOINT = 'edges'; 26 | const SETTINGS_ENDPOINT = 'settings'; 27 | 28 | module.exports = { 29 | MESSAGES_ENDPOINT, 30 | TILES_ENDPOINT, 31 | EDGES_ENDPOINT, 32 | SETTINGS_ENDPOINT, 33 | auth, 34 | fetchGqlData 35 | }; 36 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/styles/Footer.css: -------------------------------------------------------------------------------- 1 | .Footer { 2 | background: #3f3f4f; 3 | color: #fff; 4 | height: 38px; 5 | bottom: 0; 6 | position: fixed; 7 | } 8 | 9 | .Footer-container { 10 | margin: 0 auto; 11 | padding: 3px 15px; 12 | text-align: center; 13 | } 14 | 15 | .Footer-text { 16 | color: rgba(255, 255, 255, .5); 17 | } 18 | 19 | .Footer-text--muted { 20 | color: rgba(255, 255, 255, .3); 21 | } 22 | 23 | .Footer-spacer { 24 | color: rgba(255, 255, 255, .3); 25 | } 26 | 27 | .Footer-text, 28 | .Footer-link { 29 | padding: 2px 5px; 30 | font-size: 1em; 31 | } 32 | 33 | .Footer-link, 34 | .Footer-link:active, 35 | .Footer-link:visited { 36 | color: rgba(255, 255, 255, .6); 37 | text-decoration: none; 38 | } 39 | 40 | .Footer-link:hover { 41 | color: rgba(255, 255, 255, 1); 42 | } 43 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/styles/Graphics/colors.js: -------------------------------------------------------------------------------- 1 | import * as colors from 'material-ui/styles/colors'; 2 | 3 | var ThemeColors = [ 4 | colors.pink800, 5 | colors.purple800, 6 | colors.cyan800, 7 | colors.red800, 8 | colors.blue800, 9 | colors.lightBlue800, 10 | colors.deepPurple800, 11 | colors.lime800, 12 | colors.teal800 13 | ]; 14 | 15 | export default { 16 | ThemeColors, 17 | 18 | DangerColor: colors.red500, 19 | PersonColor: colors.teal700, 20 | IntentsColor: colors.tealA700, 21 | 22 | GoodColor: colors.lightBlue700, 23 | BadColor: colors.red700, 24 | 25 | PositiveColor: colors.lightBlue700, 26 | NeutralColor: colors.grey500, 27 | 28 | getColor: (idx) => { 29 | return ThemeColors[idx]; 30 | } 31 | } -------------------------------------------------------------------------------- /project-fortis-interfaces/src/styles/Graphics/styles.js: -------------------------------------------------------------------------------- 1 | export default { 2 | cardMediaStyle: { 3 | height: 240 4 | }, 5 | cardHeaderStyle: { 6 | height: 'auto' 7 | } 8 | } -------------------------------------------------------------------------------- /project-fortis-interfaces/src/styles/Insights/ActiveFiltersView.css: -------------------------------------------------------------------------------- 1 | .active-filters-view > div { 2 | position: relative; 3 | } 4 | .active-filters-view > div > svg { 5 | position: absolute; 6 | right: 0; 7 | } 8 | 9 | .active-filters-view--chip { 10 | margin-bottom: 0.5em !important; 11 | } 12 | 13 | .active-filters-view--chip--label-container { 14 | width: 140px; 15 | overflow: hidden; 16 | white-space: nowrap; 17 | text-overflow: ellipsis; 18 | } -------------------------------------------------------------------------------- /project-fortis-interfaces/src/styles/Insights/Dashboard.css: -------------------------------------------------------------------------------- 1 | .app-container{ 2 | background:#30303d; 3 | } 4 | 5 | .graphContainer{ 6 | background-color: #30303d; 7 | color: #fff; 8 | } 9 | 10 | .summaryPieContainer{ 11 | padding-right: 5px!important; 12 | padding-left: 5px!important; 13 | } 14 | 15 | .timeSeriesContainer{ 16 | padding-left: 0px; 17 | } 18 | 19 | #graphdiv{ 20 | width:100%; 21 | margin-bottom: 40px; 22 | } 23 | 24 | .news-feed-title{ 25 | color: #a3a3b3; 26 | text-transform: uppercase; 27 | padding-left: 6px; 28 | font-size: 12px; 29 | font-weight: 700; 30 | } 31 | 32 | .termBrowserContainer{ 33 | padding-right: 0px; 34 | margin-right: 0px; 35 | padding-left: 8px!important; 36 | } 37 | 38 | .heatmapContainer{ 39 | padding-left: 0px!important; 40 | } 41 | 42 | .dashboard-grid > div > div > div { 43 | width: 100%; 44 | } 45 | 46 | .dashboard-footer { 47 | border-top-width: 1px !important; 48 | } 49 | 50 | .dashboard-footer .dashboard-actions { 51 | display: flex; 52 | justify-content: space-around; 53 | } 54 | -------------------------------------------------------------------------------- /project-fortis-interfaces/src/styles/Insights/DataSelector.css: -------------------------------------------------------------------------------- 1 | .dateRow{ 2 | display: flex; 3 | } 4 | 5 | .dateFilterColumn{ 6 | display: flex; 7 | justify-content: space-between; 8 | margin-bottom: -20px; 9 | } 10 | 11 | .dateFilter{ 12 | width: 226px; 13 | } 14 | 15 | #save-button{ 16 | margin-left:8px; 17 | margin-top: 8px; 18 | } 19 | 20 | #cancel-button{ 21 | margin-top: 10px; 22 | margin-left: 4px; 23 | } 24 | 25 | .fill { 26 | min-height: 100%; 27 | height: 100%; 28 | } 29 | 30 | body{ 31 | background-color: #30303d; 32 | height: 100%; 33 | min-height: 100%; 34 | } -------------------------------------------------------------------------------- /project-fortis-interfaces/src/styles/Insights/DialogBox.css: -------------------------------------------------------------------------------- 1 | .date { 2 | color: grey; 3 | font-size: 1em; 4 | } 5 | 6 | .title { 7 | font-size: 1.5em; 8 | } 9 | 10 | .text { 11 | font-size: 1em; 12 | } 13 | 14 | .sentiment { 15 | font-size: 1em; 16 | } -------------------------------------------------------------------------------- /project-fortis-interfaces/src/styles/Insights/HeatMap.css: -------------------------------------------------------------------------------- 1 | .react-progress-bar-percent-override{ 2 | height: 15px; 3 | } 4 | 5 | .marker-cluster-base { 6 | border: 3px solid #ececec; 7 | border-radius: 50%; 8 | height: 40px; 9 | line-height: 37px; 10 | font-weight: 700; 11 | text-align: center; 12 | width: 40px; 13 | } 14 | 15 | #leafletMap{ 16 | margin-top: 0px; 17 | position: absolute; 18 | bottom: 0; 19 | top: 0; 20 | width: 100%; 21 | height: 100%; 22 | background-color: #333; 23 | } -------------------------------------------------------------------------------- /project-fortis-interfaces/src/styles/Insights/SentimentTreeView.css: -------------------------------------------------------------------------------- 1 | .tagFilterRow{ 2 | padding: 0 35px 10px; 3 | } 4 | 5 | .panel-selector{ 6 | padding-bottom: 0px!important; 7 | background-color: rgb(63, 63, 79)!important; 8 | border: 0px; 9 | } 10 | 11 | .form-control { 12 | background-color: transparent; 13 | border-width: 0; 14 | border-bottom-width: 1px; 15 | border-color: #ccc; 16 | border-color: rgba(240, 240, 240, 0.2); 17 | border-radius: 0; 18 | color: #fefefe; 19 | outline: none; 20 | -webkit-box-shadow: none !important; 21 | -moz-box-shadow: none !important; 22 | box-shadow: none !important; 23 | } 24 | 25 | .input-group .form-control.edgeFilterInput { 26 | text-overflow: ellipsis; 27 | border-radius: 0; 28 | height: 30px; 29 | width: 250px; 30 | } 31 | 32 | .input-group .form-control.edgeFilterInput.small { 33 | width: 200px; 34 | } 35 | 36 | .badge{ 37 | font-size: 10px; 38 | background-color: #337ab7!important; 39 | padding: 3px 4px; 40 | border: 1px solid #a3a3b3; 41 | margin-left: 5px; 42 | } 43 | 44 | .relevantTerm:hover{ 45 | text-decoration: underline; 46 | } 47 | 48 | .badge-disabled{ 49 | color: #a3a3b3!important; 50 | background-color: #333!important; 51 | padding: 3px 4px; 52 | margin-left: 5px; 53 | } 54 | 55 | #edge-filter-icon { 56 | width: 24px; 57 | } -------------------------------------------------------------------------------- /project-fortis-interfaces/src/utils/Fact.js: -------------------------------------------------------------------------------- 1 | import { contains } from './Utils.js'; 2 | 3 | // Fact component utils 4 | 5 | // get prev and next fact using fact message id 6 | export function getAdjacentArticles(id, facts) { 7 | let result = { 8 | "prev": null, 9 | "next": null, 10 | }; 11 | 12 | if (!facts.length > 0) { 13 | return result; 14 | } 15 | 16 | let fact = facts.find(x => x.properties.messageid === id); 17 | let index = facts.indexOf(fact); 18 | let l = facts.length; 19 | 20 | if (index - 1 < l) { 21 | result.prev = facts[index - 1]; 22 | } 23 | if (index + 1 < l) { 24 | result.next = facts[index + 1]; 25 | } 26 | return result; 27 | } 28 | 29 | // returns a copy of filtered facts 30 | export function getFilteredResults(facts, tag) { 31 | if (tag.length === 0) { 32 | return facts; 33 | } 34 | return facts.reduce(function (prev, fact) { 35 | if (contains(fact.properties.edges, tag)) { 36 | prev.push({ ...fact }); 37 | } 38 | return prev; 39 | }, []); 40 | } 41 | 42 | export function arrayToFragment(array) { 43 | return (array) ? array.join('+') : ""; 44 | } 45 | 46 | export function fragmentToArray(fragment) { 47 | return (fragment) ? fragment.split('+') : []; 48 | } 49 | 50 | export function changeFactsUrl(category, selectedTags) { 51 | let url = "/site/{0}/facts/".format(category); 52 | if (selectedTags.length > 0) { 53 | const fragment = arrayToFragment(selectedTags); 54 | url = "/site/{0}/facts/tags/{1}".format(category, fragment.toLowerCase()); 55 | } 56 | window.location.hash = url; 57 | } 58 | -------------------------------------------------------------------------------- /project-fortis-interfaces/travis/ci.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | pushd "$(dirname $0)/.." 6 | 7 | err=0 8 | 9 | npm install 10 | 11 | if ! ./node_modules/.bin/eslint src *.js; then 12 | err=1 13 | fi 14 | 15 | if ./node_modules/.bin/depcheck | grep -q '^Unused dependencies$'; then 16 | err=2 17 | fi 18 | 19 | popd 20 | 21 | exit "$err" 22 | -------------------------------------------------------------------------------- /project-fortis-pipeline/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-pipeline/.gitignore -------------------------------------------------------------------------------- /project-fortis-pipeline/localdeploy/parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "namespaces_fortiseventhubs_name": { 6 | "value": "fortiseh" 7 | }, 8 | "namespaces_fortisservicebus_name": { 9 | "value": "fortissb" 10 | }, 11 | "storageAccounts_fortisblobstorage_name": { 12 | "value": "fortisblob" 13 | }, 14 | "accounts_fortistranslator_name": { 15 | "value": "fortistrans" 16 | }, 17 | "accounts_fortisspeechtotext_name": { 18 | "value": "fortisstt" 19 | }, 20 | "components_fortisapplicationinsights_name": { 21 | "value": "fortislog" 22 | }, 23 | "accounts_fortistextanalytics_name": { 24 | "value": "fortisnlp" 25 | }, 26 | "accounts_fortiscomputervision_name": { 27 | "value": "fortiscv" 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /project-fortis-pipeline/localdeploy/parse-output.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import argparse 4 | import json 5 | 6 | parser = argparse.ArgumentParser() 7 | parser.add_argument('file_to_parse', type=argparse.FileType('r')) 8 | args = parser.parse_args() 9 | 10 | json_payload = json.load(args.file_to_parse) 11 | 12 | outputs = json_payload.get('properties', {}).get('outputs', {}) 13 | for key, value in outputs.items(): 14 | value = value.get('value', '') 15 | if key and value: 16 | # upper-case output key names sometimes get messed up with some 17 | # characters being flipped to lower-case; correcting for that below 18 | key = key if key.lower() == key else key.upper() 19 | print('%s=%s' % (key, value)) 20 | -------------------------------------------------------------------------------- /project-fortis-pipeline/localdeploy/seed-data/seed-data-no-events.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-pipeline/localdeploy/seed-data/seed-data-no-events.tar.gz -------------------------------------------------------------------------------- /project-fortis-pipeline/localdeploy/seed-data/seed-data-twitter-clewolff.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-pipeline/localdeploy/seed-data/seed-data-twitter-clewolff.tar.gz -------------------------------------------------------------------------------- /project-fortis-pipeline/localdeploy/seed-data/seed-data-twitter-steph.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-pipeline/localdeploy/seed-data/seed-data-twitter-steph.tar.gz -------------------------------------------------------------------------------- /project-fortis-pipeline/localdeploy/seed-data/seed-data-twitter.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-pipeline/localdeploy/seed-data/seed-data-twitter.tar.gz -------------------------------------------------------------------------------- /project-fortis-pipeline/localdeploy/seed-data/seed-data.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-pipeline/localdeploy/seed-data/seed-data.tar.gz -------------------------------------------------------------------------------- /project-fortis-pipeline/ops/charts/cassandra/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | -------------------------------------------------------------------------------- /project-fortis-pipeline/ops/charts/cassandra/Chart.yaml: -------------------------------------------------------------------------------- 1 | name: cassandra 2 | home: http://cassandra.apache.org 3 | version: 0.1.0 4 | description: A highly scalable, high-performance distributed database designed to handle large amounts of data across many commodity servers, providing high availability with no single point of failure. 5 | icon: http://cassandra.apache.org/img/cassandra_logo.png 6 | sources: 7 | - https://github.com/apache/cassandra 8 | keywords: 9 | - cassandra 10 | - nosql 11 | - database 12 | -------------------------------------------------------------------------------- /project-fortis-pipeline/ops/charts/cassandra/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 24 | trimSuffix "-" -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 24 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | */}} 13 | {{- define "fullname" -}} 14 | {{- $name := default .Chart.Name .Values.nameOverride -}} 15 | {{- printf "%s-%s" .Release.Name $name | trunc 24 | trimSuffix "-" -}} 16 | {{- end -}} 17 | -------------------------------------------------------------------------------- /project-fortis-pipeline/ops/charts/cassandra/templates/svc.yaml: -------------------------------------------------------------------------------- 1 | # Headless service for stable DNS entries of StatefulSet members. 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: {{ template "fullname" . }} 6 | labels: 7 | app: {{ template "fullname" . }} 8 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | spec: 12 | ports: 13 | - name: {{ template "fullname" . }} 14 | port: 9042 15 | clusterIP: None 16 | selector: 17 | app: {{ template "fullname" . }} 18 | --- 19 | apiVersion: v1 20 | kind: Service 21 | metadata: 22 | name: "{{ template "fullname" . }}-ext" 23 | labels: 24 | app: {{ template "fullname" . }} 25 | chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" 26 | release: "{{ .Release.Name }}" 27 | heritage: "{{ .Release.Service }}" 28 | annotations: 29 | service.beta.kubernetes.io/azure-load-balancer-internal: "true" 30 | spec: 31 | ports: 32 | - name: {{ template "fullname" . }} 33 | port: 9042 34 | selector: 35 | app: {{ template "fullname" . }} 36 | type: "LoadBalancer" -------------------------------------------------------------------------------- /project-fortis-pipeline/ops/charts/cassandra/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for cassandra. 2 | # This is a YAML-formatted file. 3 | # Declare name/value pairs to be passed into your templates. 4 | # name: value 5 | 6 | Name: cassandra 7 | Component: "cassandra" 8 | replicaCount: 6 9 | Image: "erikschlegel/cassandra" 10 | VmInstanceType: "Standard_L4s" 11 | ImageTag: "v12" 12 | ImagePullPolicy: "Always" 13 | 14 | # Cassandra configuration options 15 | # For chart deployment, the value for sending to the Seed Provider is 16 | # constructed using a template in the statefulset.yaml template 17 | cassandra: 18 | MaxHeapSize: "4000M" 19 | HeapNewSize: "100M" 20 | ClusterName: "cassandra" 21 | DC: "dc-eastus2-cassandra" 22 | Rack: "rack-eastus2-cassandra" 23 | AutoBootstrap: "false" 24 | 25 | # Persistence information 26 | persistence: 27 | enabled: true 28 | storageClass: fast 29 | accessMode: ReadWriteOnce 30 | size: 512Gi 31 | 32 | # Instance resources 33 | resources: 34 | requests: 35 | cpu: "1000m" 36 | memory: "4Gi" 37 | limits: 38 | cpu: "2000m" 39 | memory: "8Gi" 40 | -------------------------------------------------------------------------------- /project-fortis-pipeline/ops/charts/spark/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | -------------------------------------------------------------------------------- /project-fortis-pipeline/ops/charts/spark/Chart.yaml: -------------------------------------------------------------------------------- 1 | name: spark 2 | home: http://spark.apache.org/ 3 | version: 0.1.3 4 | description: Fast and general-purpose cluster computing system. 5 | home: http://spark.apache.org 6 | icon: http://spark.apache.org/images/spark-logo-trademark.png 7 | sources: 8 | - https://github.com/kubernetes/kubernetes/tree/master/examples/spark 9 | - https://github.com/apache/spark 10 | maintainers: 11 | - name: Erik Schlegel 12 | email: erik.schlegel@gmail.com 13 | -------------------------------------------------------------------------------- /project-fortis-pipeline/ops/charts/spark/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | 1. Get the Spark URL to visit by running these commands in the same shell: 2 | 3 | NOTE: It may take a few minutes for the LoadBalancer IP to be available. 4 | You can watch the status of by running 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ template "webui-fullname" . }}' 5 | 6 | export SPARK_SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "webui-fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') 7 | echo http://$SPARK_SERVICE_IP:{{ .Values.WebUi.ServicePort }} 8 | 9 | 2. Get the Zeppelin URL to visit by running these commands in the same shell: 10 | 11 | NOTE: It may take a few minutes for the LoadBalancer IP to be available. 12 | You can watch the status of by running 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ template "zeppelin-fullname" . }}' 13 | 14 | export ZEPPELIN_SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "zeppelin-fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') 15 | echo http://$ZEPPELIN_SERVICE_IP:{{ .Values.Zeppelin.ServicePort }} 16 | 17 | -------------------------------------------------------------------------------- /project-fortis-pipeline/ops/charts/spark/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 24 -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create fully qualified names. 11 | We truncate at 24 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | */}} 13 | {{- define "master-fullname" -}} 14 | {{- $name := default .Chart.Name .Values.Master.Name -}} 15 | {{- printf "%s-%s" .Release.Name $name | trunc 24 -}} 16 | {{- end -}} 17 | 18 | {{- define "webui-fullname" -}} 19 | {{- $name := default .Chart.Name .Values.WebUi.Name -}} 20 | {{- printf "%s-%s" .Release.Name $name | trunc 24 -}} 21 | {{- end -}} 22 | 23 | {{- define "worker-fullname" -}} 24 | {{- $name := default .Chart.Name .Values.Worker.Name -}} 25 | {{- printf "%s-%s" .Release.Name $name | trunc 24 -}} 26 | {{- end -}} 27 | 28 | {{- define "zeppelin-fullname" -}} 29 | {{- $name := default .Chart.Name .Values.Zeppelin.Name -}} 30 | {{- printf "%s-%s" .Release.Name $name | trunc 24 -}} 31 | {{- end -}} -------------------------------------------------------------------------------- /project-fortis-pipeline/ops/charts/spark/templates/secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: checkpointing-pvc-secret 5 | type: Opaque 6 | data: 7 | {{ if .Values.Persistence.PvcAcctName }} 8 | azurestorageaccountname: {{ .Values.Persistence.PvcAcctName | b64enc | quote }} 9 | {{ else }} 10 | azurestorageaccountname: {{ randAlphaNum 10 | b64enc | quote }} 11 | {{ end }} 12 | {{ if .Values.Persistence.PvcPwd }} 13 | azurestorageaccountkey: {{ .Values.Persistence.PvcPwd | b64enc | quote }} 14 | {{ else }} 15 | azurestorageaccountkey: {{ randAlphaNum 10 | b64enc | quote }} 16 | {{ end }} -------------------------------------------------------------------------------- /project-fortis-pipeline/ops/charts/spark/templates/spark-master-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{.Values.Master.Component}} 5 | labels: 6 | heritage: {{.Release.Service | quote }} 7 | release: {{.Release.Name | quote }} 8 | chart: "{{.Chart.Name}}-{{.Chart.Version}}" 9 | component: "{{.Values.Master.Component}}" 10 | annotations: 11 | service.beta.kubernetes.io/azure-load-balancer-internal: "true" 12 | spec: 13 | ports: 14 | - port: {{.Values.Master.ServicePort}} 15 | targetPort: {{.Values.Master.ContainerPort}} 16 | name: spark 17 | - port: {{.Values.WebUi.ServicePort}} 18 | targetPort: {{.Values.WebUi.ContainerPort}} 19 | name: http 20 | selector: 21 | component: "{{.Values.Master.Component}}" 22 | type: "LoadBalancer" -------------------------------------------------------------------------------- /project-fortis-pipeline/ops/charts/spark/templates/spark-zeppelin-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: zeppelin 5 | annotations: 6 | service.beta.kubernetes.io/azure-load-balancer-internal: "true" 7 | spec: 8 | ports: 9 | - port: {{.Values.Zeppelin.ServicePort}} 10 | targetPort: {{.Values.Zeppelin.ContainerPort}} 11 | selector: 12 | component: zeppelin 13 | type: "LoadBalancer" 14 | --- 15 | apiVersion: v1 16 | kind: ReplicationController 17 | metadata: 18 | name: zeppelin-controller 19 | spec: 20 | replicas: {{default 1 .Values.Zeppelin.Replicas}} 21 | selector: 22 | component: zeppelin 23 | template: 24 | metadata: 25 | labels: 26 | component: zeppelin 27 | spec: 28 | containers: 29 | - name: zeppelin 30 | image: "{{.Values.Zeppelin.Image}}:{{.Values.Zeppelin.ImageTag}}" 31 | ports: 32 | - containerPort: {{.Values.Zeppelin.ContainerPort}} 33 | resources: 34 | requests: 35 | cpu: "{{.Values.Zeppelin.Cpu}}" 36 | -------------------------------------------------------------------------------- /project-fortis-pipeline/ops/charts/spark/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for spark. 2 | # This is a YAML-formatted file. 3 | # Declare name/value pairs to be passed into your templates. 4 | # name: value 5 | 6 | Master: 7 | Name: master 8 | Image: "erikschlegel/spark-master" 9 | ImageTag: "2.2" 10 | Component: "spark-master" 11 | ImagePullPolicy: "Always" 12 | ServicePort: 7077 13 | ContainerPort: 7077 14 | #SparkSubmitCommand: ["spark-submit", "--master local[2]", "--driver-memory 4g", "enter-your-fat.jar"] 15 | #ConfigMapName: spark-master-conf 16 | Resources: 17 | Requests: 18 | Cpu: "700m" 19 | Memory: "3Gi" 20 | Limits: 21 | Cpu: "700m" 22 | Memory: "3Gi" 23 | # Set Master JVM memory. Default 1g 24 | DaemonMemory: 1g 25 | 26 | WebUi: 27 | Name: webui 28 | ServicePort: 8080 29 | Component: "spark-webui" 30 | ProxyPort: 80 31 | ContainerPort: 8080 32 | Image: "elsonrodriguez/spark-ui-proxy:1.0" 33 | 34 | Worker: 35 | Name: worker 36 | Image: "erikschlegel/spark-worker" 37 | ImageTag: "2.2" 38 | ImagePullPolicy: "Always" 39 | VmInstanceType: "Standard_L4s" 40 | Replicas: 6 41 | Component: "spark-worker" 42 | WorkingDirectory: "/opt/spark/work" 43 | ContainerPort: 8081 44 | #ConfigMapName: spark-master-conf 45 | Resources: 46 | Requests: 47 | Cpu: "700m" 48 | Memory: "3Gi" 49 | Limits: 50 | Cpu: "700m" 51 | Memory: "3Gi" 52 | Environment: 53 | - name: SPARK_DAEMON_MEMORY 54 | value: 1g 55 | - name: SPARK_WORKER_MEMORY 56 | value: 1g 57 | 58 | Zeppelin: 59 | Name: zeppelin 60 | Image: "srfrnk/zeppelin" 61 | ImageTag: "0.7.0" 62 | Component: "zeppelin" 63 | Cpu: "100m" 64 | ServicePort: 8080 65 | ContainerPort: 8080 66 | 67 | Persistence: 68 | # PvcAcctName: Secret 69 | # PvcPwd: Secret 70 | CheckpointDirectory: "/opt/checkpoint" 71 | #CheckpointShare: "checkpoint" 72 | -------------------------------------------------------------------------------- /project-fortis-pipeline/ops/create-tags.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | readonly k8resource_group="${1}" 4 | readonly fortis_interface_host="${2}" 5 | readonly site_name="${3}" 6 | readonly graphql_service_host="${4}" 7 | 8 | az group update --name "${k8resource_group}" --set tags.FORTIS_INTERFACE_URL="${fortis_interface_host}/index.html#/dashboard" 9 | az group update --name "${k8resource_group}" --set tags.FORTIS_ADMIN_INTERFACE_URL="${fortis_interface_host}/index.html#/settings" 10 | az group update --name "${k8resource_group}" --set tags.FORTIS_AAD_REDIRECT_URL="${fortis_interface_host}/index.html" 11 | az group update --name "${k8resource_group}" --set tags.FORTIS_SERVICE_HOST="${graphql_service_host}" 12 | -------------------------------------------------------------------------------- /project-fortis-pipeline/ops/install-cassandra.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | readonly k8cassandra_node_count="$1" 4 | readonly agent_vm_size="$2" 5 | 6 | # setup 7 | cd charts || exit -2 8 | readonly cluster_name="FORTIS_CASSANDRA" 9 | readonly storageClass="fast" 10 | 11 | install_cassandra() { 12 | helm install \ 13 | --set replicaCount="${k8cassandra_node_count}" \ 14 | --set VmInstanceType="${agent_vm_size}" \ 15 | --set cassandra.ClusterName="${cluster_name}" \ 16 | --set persistence.storageClass="${storageClass}" \ 17 | --namespace cassandra \ 18 | --name cassandra-cluster \ 19 | ./cassandra 20 | } 21 | 22 | while ! install_cassandra; do 23 | echo "Failed to set up cassandra helm chart, retrying" 24 | sleep 30s 25 | done 26 | 27 | # wait for all cassandra nodes to be ready 28 | while [ -z "$(kubectl --namespace=cassandra get svc cassandra-cluster-cassan-ext -o jsonpath='{..ip}')" ]; do 29 | echo "Waiting for Cassandra to get ready" 30 | sleep 10s 31 | done 32 | 33 | # cleanup 34 | cd .. 35 | -------------------------------------------------------------------------------- /project-fortis-pipeline/ops/install-fortis-backup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | readonly cassandra_ip="${1}" 4 | readonly cassandra_port="${2}" 5 | readonly cassandra_username="${3}" 6 | readonly cassandra_password="${4}" 7 | readonly storage_account_name="${5}" 8 | readonly storage_account_key="${6}" 9 | readonly fortis_backup_container="${7}" 10 | readonly latest_version="${8}" 11 | 12 | # setup 13 | readonly install_dir="$(mktemp -d /tmp/fortis-backup-XXXXXX)" 14 | readonly deployment_yaml="${install_dir}/kubernetes-deployment.yaml" 15 | 16 | # deploy the service to the kubernetes cluster 17 | cat > "${deployment_yaml}" << EOF 18 | apiVersion: extensions/v1beta1 19 | kind: Deployment 20 | metadata: 21 | creationTimestamp: null 22 | name: project-fortis-backup 23 | labels: 24 | io.kompose.service: project-fortis-backup 25 | spec: 26 | replicas: 1 27 | strategy: {} 28 | template: 29 | metadata: 30 | creationTimestamp: null 31 | labels: 32 | io.kompose.service: project-fortis-backup 33 | spec: 34 | containers: 35 | - env: 36 | - name: FORTIS_CASSANDRA_HOST 37 | value: ${cassandra_ip} 38 | - name: FORTIS_CASSANDRA_PORT 39 | value: "${cassandra_port}" 40 | - name: FORTIS_CASSANDRA_USERNAME 41 | value: ${cassandra_username} 42 | - name: FORTIS_CASSANDRA_PASSWORD 43 | value: ${cassandra_password} 44 | - name: USER_FILES_BLOB_ACCOUNT_NAME 45 | value: ${storage_account_name} 46 | - name: USER_FILES_BLOB_ACCOUNT_KEY 47 | value: ${storage_account_key} 48 | - name: BACKUP_CONTAINER_NAME 49 | value: ${fortis_backup_container} 50 | image: cwolff/project_fortis_backup:${latest_version} 51 | imagePullPolicy: "Always" 52 | name: project-fortis-backup 53 | resources: {} 54 | restartPolicy: Always 55 | status: {} 56 | EOF 57 | kubectl create -f "${deployment_yaml}" 58 | -------------------------------------------------------------------------------- /project-fortis-pipeline/ops/install-fortis-interfaces.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | readonly graphql_service_host="$1" 4 | readonly blob_account_name="$2" 5 | readonly blob_account_key="$3" 6 | readonly blob_container_name="$4" 7 | readonly fortis_interface_host="$5" 8 | readonly aad_client="$6" 9 | readonly mapbox_tile_layer_url="$7" 10 | readonly latest_version="$8" 11 | 12 | # setup 13 | if ! (command -v jq >/dev/null); then sudo apt-get -qq install -y jq; fi 14 | if ! (command -v npm >/dev/null); then curl -sL 'https://deb.nodesource.com/setup_8.x' | sudo -E bash -; sudo apt-get -qq install -y nodejs; fi 15 | 16 | readonly install_dir="$(mktemp -d /tmp/fortis-interfaces-XXXXXX)" 17 | pushd "${install_dir}" 18 | 19 | wget "https://github.com/CatalystCode/project-fortis/archive/${latest_version}.tar.gz" 20 | tar xf "${latest_version}.tar.gz" "project-fortis-${latest_version}/project-fortis-interfaces" --strip-components=2 21 | 22 | npm install 23 | 24 | # add site root to package.json so that the frontend build can include the 25 | # correct relative links to resources like js, css, static files, etc. 26 | readonly package_json="$(mktemp)" 27 | jq --arg homepage "$fortis_interface_host" ". + {homepage: \$homepage}" > "$package_json" < package.json 28 | mv "$package_json" ./package.json 29 | 30 | # build the frontend 31 | REACT_APP_SERVICE_HOST="${graphql_service_host}" \ 32 | REACT_APP_AD_CLIENT_ID="${aad_client}" \ 33 | REACT_APP_MAPBOX_TILE_LAYER_URL="${mapbox_tile_layer_url}" \ 34 | npm run build 35 | 36 | # deploy the frontend to blob storage 37 | az storage container create \ 38 | --account-key "$blob_account_key" \ 39 | --account-name "$blob_account_name" \ 40 | --name "$blob_container_name" \ 41 | --public-access "container" 42 | az storage blob upload-batch \ 43 | --account-key "$blob_account_key" \ 44 | --account-name "$blob_account_name" \ 45 | --destination "$blob_container_name" \ 46 | --source "./build" 47 | 48 | # cleanup 49 | popd 50 | -------------------------------------------------------------------------------- /project-fortis-pipeline/travis/ci.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | # shellcheck disable=SC2086 6 | pushd "$(dirname $0)/.." 7 | 8 | # shellcheck disable=SC2046 9 | shellcheck $(find . -name '*.sh') 10 | 11 | popd 12 | -------------------------------------------------------------------------------- /project-fortis-services/.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | travis/ 3 | README.md 4 | .gitignore 5 | .gitattributes 6 | -------------------------------------------------------------------------------- /project-fortis-services/.eslintignore: -------------------------------------------------------------------------------- 1 | coverage 2 | templates 3 | -------------------------------------------------------------------------------- /project-fortis-services/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "rules": { 3 | "no-console": "off", 4 | "indent": [ 5 | "error", 6 | 2 7 | ], 8 | "quotes": [ 9 | 2, 10 | "single" 11 | ], 12 | "linebreak-style": [ 13 | 2, 14 | "unix" 15 | ], 16 | "semi": [ 17 | 2, 18 | "always" 19 | ] 20 | }, 21 | "env": { 22 | "es6": true, 23 | "node": true 24 | }, 25 | "extends": [ 26 | "plugin:require-path-exists/recommended", 27 | "eslint:recommended" 28 | ], 29 | "plugins": [ 30 | "require-path-exists" 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /project-fortis-services/.gitattributes: -------------------------------------------------------------------------------- 1 | *.js eol=lf 2 | -------------------------------------------------------------------------------- /project-fortis-services/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # nyc test coverage 18 | .nyc_output 19 | 20 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 21 | .grunt 22 | 23 | # node-waf configuration 24 | .lock-wscript 25 | 26 | # Compiled binary addons (http://nodejs.org/api/addons.html) 27 | build/Release 28 | 29 | # Dependency directories 30 | node_modules 31 | jspm_packages 32 | 33 | # Optional npm cache directory 34 | .npm 35 | 36 | # Optional REPL history 37 | .node_repl_history 38 | 39 | # IDEA config files 40 | .idea/ 41 | 42 | # VS code config 43 | .vscode 44 | -------------------------------------------------------------------------------- /project-fortis-services/docker/run-cqlsh.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | "$CASSANDRA_HOME/bin/cqlsh" \ 4 | --request-timeout=3600 \ 5 | --username="$FORTIS_CASSANDRA_USERNAME" \ 6 | --password="$FORTIS_CASSANDRA_PASSWORD" \ 7 | "$FORTIS_CASSANDRA_HOST" \ 8 | "$FORTIS_CASSANDRA_PORT" 9 | -------------------------------------------------------------------------------- /project-fortis-services/src/clients/appinsights/AppInsightsConstants.js: -------------------------------------------------------------------------------- 1 | const CLIENTS = { 2 | cassandra: 'cassandra', 3 | appInsights: 'appInsights', 4 | eventHub: 'eventHub', 5 | facebookAnalytics: 'facebookAnalytics', 6 | featureService: 'featureService', 7 | tileService: 'tileService', 8 | postres: 'postgres', 9 | serviceBus: 'serviceBus', 10 | blobStorage: 'blobStorage', 11 | translator: 'translator' 12 | }; 13 | 14 | module.exports = { 15 | CLIENTS: CLIENTS 16 | }; -------------------------------------------------------------------------------- /project-fortis-services/src/clients/eventhub/EventHubSender.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const EventHubClient = require('azure-event-hubs').Client; 4 | const Promise = require('promise'); 5 | const trackDependency = require('../appinsights/AppInsightsClient').trackDependency; 6 | 7 | const { 8 | publishEventsEventhubConnectionString, publishEventsEventhubPath, 9 | publishEventsEventhubPartition 10 | } = require('../../../config').eventHub; 11 | 12 | function sendMessages(messages) { 13 | return new Promise((resolve, reject) => { 14 | if (!messages || !messages.length) { 15 | return reject('No messages to be sent'); 16 | } 17 | 18 | let payloads; 19 | try { 20 | payloads = messages.map(message => ({contents: JSON.stringify(message)})); 21 | } catch (err) { 22 | return reject(`Unable to create payloads for EventHub: ${err}`); 23 | } 24 | 25 | const eventHubClient = EventHubClient.fromConnectionString( 26 | publishEventsEventhubConnectionString, publishEventsEventhubPath); 27 | 28 | if (!eventHubClient) return reject('No event hub connection string provided.'); 29 | 30 | eventHubClient.open() 31 | .then(() => eventHubClient.createSender()) 32 | .then(eventHubSender => { 33 | eventHubSender.on('errorReceived', err => reject(`Error talking to EventHub: ${err}`)); 34 | Promise.all(payloads.map(payload => eventHubSender.send(payload, publishEventsEventhubPartition))) 35 | .then(() => resolve([])) 36 | .catch((err) => reject(`Error sending EventHub message: ${err}`)); 37 | }); 38 | }); 39 | } 40 | 41 | module.exports = { 42 | sendMessages: trackDependency(sendMessages, 'EventHub', 'send') 43 | }; -------------------------------------------------------------------------------- /project-fortis-services/src/clients/streaming/ServiceBusClient.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Promise = require('promise'); 4 | const azure = require('azure-sb'); 5 | const { trackDependency } = require('../appinsights/AppInsightsClient'); 6 | 7 | const { 8 | fortisSbConnStr 9 | } = require('../../../config').serviceBus; 10 | 11 | let client; 12 | 13 | function sendQueueMessage(queue, serviceBusMessage) { 14 | return new Promise((resolve, reject) => { 15 | if (!client) { 16 | try { 17 | client = azure.createServiceBusService(fortisSbConnStr); 18 | } catch (exception) { 19 | return reject(exception); 20 | } 21 | } 22 | 23 | try { 24 | client.sendQueueMessage(queue, serviceBusMessage, (error) => { 25 | if (error) reject(error); 26 | else resolve(serviceBusMessage); 27 | }); 28 | } catch (exception) { 29 | reject(exception); 30 | } 31 | }); 32 | } 33 | 34 | module.exports = { 35 | sendQueueMessage: trackDependency(sendQueueMessage, 'ServiceBus', 'send'), 36 | }; 37 | -------------------------------------------------------------------------------- /project-fortis-services/src/clients/streaming/StreamingController.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { trackEvent } = require('../appinsights/AppInsightsClient'); 4 | const { sendQueueMessage } = require('./ServiceBusClient'); 5 | 6 | const { 7 | fortisSbCommandQueue, fortisSbConfigQueue 8 | } = require('../../../config').serviceBus; 9 | 10 | function restartPipeline() { 11 | return notifyUpdate(fortisSbCommandQueue); 12 | } 13 | 14 | function notifyWatchlistUpdate() { 15 | return notifyUpdate(fortisSbConfigQueue, { 'dirty': 'watchlist' }); 16 | } 17 | 18 | function notifyBlacklistUpdate() { 19 | return notifyUpdate(fortisSbConfigQueue, { 'dirty': 'blacklist' }); 20 | } 21 | 22 | function notifySiteSettingsUpdate() { 23 | return notifyUpdate(fortisSbConfigQueue, { 'dirty': 'sitesettings' }); 24 | } 25 | 26 | function notifyUpdate(queue, properties) { 27 | const serviceBusMessage = {}; 28 | 29 | if (properties) serviceBusMessage.customProperties = properties; 30 | 31 | return sendQueueMessage(queue, serviceBusMessage); 32 | } 33 | 34 | module.exports = { 35 | restartPipeline: trackEvent(restartPipeline, 'notifyRestartPipeline'), 36 | notifyWatchlistUpdate: trackEvent(notifyWatchlistUpdate, 'notifyWatchlistChanged'), 37 | notifyBlacklistUpdate: trackEvent(notifyBlacklistUpdate, 'notifyBlacklistChanged'), 38 | notifySiteSettingsUpdate: trackEvent(notifySiteSettingsUpdate, 'notifySettingsChanged') 39 | }; 40 | -------------------------------------------------------------------------------- /project-fortis-services/src/resolvers/Edges/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const queries = require('./queries'); 4 | 5 | module.exports = { 6 | topLocations: queries.popularLocations, 7 | timeSeries: queries.timeSeries, 8 | topSources: queries.topSources, 9 | topTerms: queries.topTerms, 10 | geofenceplaces: queries.geofenceplaces, 11 | conjunctiveTerms: queries.conjunctiveTopics 12 | }; 13 | -------------------------------------------------------------------------------- /project-fortis-services/src/resolvers/Messages/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const mutations = require('./mutations'); 4 | const queries = require('./queries'); 5 | 6 | module.exports = { 7 | publishEvents: mutations.publishEvents, 8 | restartPipeline: mutations.restartPipeline, 9 | 10 | byLocation: queries.byLocation, 11 | byBbox: queries.byBbox, 12 | byEdges: queries.byEdges, 13 | byPipeline: queries.byPipeline, 14 | event: queries.event, 15 | translate: queries.translate, 16 | translateWords: queries.translateWords 17 | }; 18 | -------------------------------------------------------------------------------- /project-fortis-services/src/resolvers/Messages/mutations.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const streamingController = require('../../clients/streaming/StreamingController'); 4 | const eventHubSender = require('../../clients/eventhub/EventHubSender'); 5 | const trackEvent = require('../../clients/appinsights/AppInsightsClient').trackEvent; 6 | const restartPipelineExtraProps = require('../../clients/appinsights/LoggingClient').restartPipelineExtraProps; 7 | const { requiresRole } = require('../../auth'); 8 | 9 | function restartPipeline(args, res) { // eslint-disable-line no-unused-vars 10 | return streamingController.restartPipeline(); 11 | } 12 | 13 | function publishEvents(args, res) { // eslint-disable-line no-unused-vars 14 | return eventHubSender.sendMessages(args && args.input && args.input.messages); 15 | } 16 | 17 | module.exports = { 18 | restartPipeline: requiresRole(trackEvent(restartPipeline, 'restartPipeline', restartPipelineExtraProps()), 'admin'), 19 | publishEvents: requiresRole(trackEvent(publishEvents, 'publishEvents'), 'admin') 20 | }; 21 | -------------------------------------------------------------------------------- /project-fortis-services/src/resolvers/Settings/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const mutations = require('./mutations'); 4 | const queries = require('./queries'); 5 | 6 | module.exports = { 7 | addUsers: mutations.addUsers, 8 | removeUsers: mutations.removeUsers, 9 | removeSite: mutations.removeSite, 10 | editSite: mutations.editSite, 11 | modifyStreams: mutations.modifyStreams, 12 | removeStreams: mutations.removeStreams, 13 | modifyBlacklist: mutations.modifyBlacklist, 14 | removeBlacklist: mutations.removeBlacklist, 15 | removeKeywords: mutations.removeKeywords, 16 | addKeywords: mutations.addKeywords, 17 | addTrustedSources: mutations.addTrustedSources, 18 | removeTrustedSources: mutations.removeTrustedSources, 19 | 20 | exportSite: queries.exportSite, 21 | users: queries.users, 22 | siteTerms: queries.siteTerms, 23 | sites: queries.sites, 24 | trustedSources: queries.trustedSources, 25 | streams: queries.streams, 26 | termBlacklist: queries.termBlacklist 27 | }; 28 | -------------------------------------------------------------------------------- /project-fortis-services/src/resolvers/Settings/shared.js: -------------------------------------------------------------------------------- 1 | const PlaceholderForSecret = 'secretHidden'; 2 | 3 | const SecretStreamParams = new Set([ 4 | 'consumerKey', 5 | 'consumerSecret', 6 | 'accessToken', 7 | 'accessTokenSecret', 8 | ]); 9 | 10 | function isSecretUnchanged(value) { 11 | return value === PlaceholderForSecret; 12 | } 13 | 14 | function hideSecret(obj, key) { 15 | if (obj[key]) { 16 | obj[key] = PlaceholderForSecret; 17 | } 18 | } 19 | 20 | function isSecretParam(param) { 21 | return SecretStreamParams.has(param); 22 | } 23 | 24 | function paramsToParamsEntries(params) { 25 | return Object.keys(params).map(key => ({ key, value: params[key] })); 26 | } 27 | 28 | function cassandraRowToStream(row) { 29 | if (row.enabled == null) { 30 | row.enabled = false; 31 | } 32 | 33 | let params; 34 | try { 35 | params = row.params_json ? JSON.parse(row.params_json) : {}; 36 | } catch (err) { 37 | console.error(`Unable to parse params '${row.params_json}' for stream ${row.streamid}`); 38 | params = {}; 39 | } 40 | 41 | return { 42 | streamId: row.streamid, 43 | pipelineKey: row.pipelinekey, 44 | pipelineLabel: row.pipelinelabel, 45 | pipelineIcon: row.pipelineicon, 46 | streamFactory: row.streamfactory, 47 | params: paramsToParamsEntries(params), 48 | enabled: row.enabled 49 | }; 50 | } 51 | 52 | module.exports = { 53 | isSecretUnchanged, 54 | isSecretParam, 55 | hideSecret, 56 | cassandraRowToStream 57 | }; 58 | -------------------------------------------------------------------------------- /project-fortis-services/src/resolvers/Tiles/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const queries = require('./queries'); 4 | 5 | module.exports = { 6 | heatmapFeaturesByTile: queries.heatmapFeaturesByTile, 7 | fetchTileIdsByPlaceId: queries.fetchTileIdsByPlaceId 8 | }; 9 | -------------------------------------------------------------------------------- /project-fortis-services/src/routes/healthcheck.js: -------------------------------------------------------------------------------- 1 | const cassandraStatus = require('../clients/cassandra/CassandraConnector').status; 2 | 3 | function healthcheckHandler(req, res) { 4 | return res.json({ 5 | cassandraIsInitialized: cassandraStatus.isInitialized 6 | }); 7 | } 8 | 9 | module.exports = healthcheckHandler; 10 | -------------------------------------------------------------------------------- /project-fortis-services/src/schemas/TilesSchema.js: -------------------------------------------------------------------------------- 1 | const graphql = require('graphql'); 2 | 3 | module.exports = graphql.buildSchema(` 4 | type Query { 5 | heatmapFeaturesByTile(fromDate: String!, toDate: String!, periodType: String!, pipelinekeys: [String]!, maintopic: String!, conjunctivetopics: [String], tileid: String!, zoomLevel: Int!, bbox: [Float], externalsourceid: String!): FeatureCollection, 6 | fetchTileIdsByPlaceId(placeid: String!, zoomLevel: Int!): [TileId], 7 | } 8 | 9 | enum TypeEnum { 10 | FeatureCollection 11 | } 12 | 13 | enum FeatureType { 14 | Point 15 | } 16 | 17 | type FeatureCollection { 18 | runTime: String, 19 | type: TypeEnum!, 20 | features: [Feature]! 21 | } 22 | 23 | type TileId { 24 | id: String 25 | zoom: Int 26 | row: Int 27 | column: Int 28 | } 29 | 30 | type Feature { 31 | type: FeatureType, 32 | coordinates: [Float], 33 | properties: Tile! 34 | } 35 | 36 | type Tile { 37 | mentions: Int 38 | date: String 39 | avgsentiment: Float 40 | tile: TileId 41 | } 42 | `); -------------------------------------------------------------------------------- /project-fortis-services/src/scripts/addusers.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | 'use strict'; 4 | 5 | const Promise = require('promise'); 6 | const cassandraConnector = require('../clients/cassandra/CassandraConnector'); 7 | 8 | function addUsers(role, users) { 9 | return new Promise((resolve, reject) => { 10 | if (!role || !role.length) return reject('role is not defined'); 11 | if (!users || !users.length) return reject('users is not defined'); 12 | 13 | const mutations = users.map(user => ({ 14 | query: 'INSERT INTO settings.users(identifier, role) VALUES (?, ?) IF NOT EXISTS', 15 | params: [user, role] 16 | })); 17 | const queries = users.map(user => ({ 18 | query: 'SELECT * FROM settings.users WHERE identifier = ? AND role = ?', 19 | params: [user, role] 20 | })); 21 | 22 | Promise.all(mutations.map(mutation => cassandraConnector.executeBatchMutations([mutation]))) 23 | .then(() => Promise.all(queries.map(({ query, params }) => cassandraConnector.executeQuery(query, params)))) 24 | .then(addedUsers => { 25 | if (addedUsers.length === users.length) { 26 | resolve(`${addedUsers.length} users have role ${role}`); 27 | } else { 28 | reject('Tried to add users but query-back did not return them'); 29 | } 30 | }) 31 | .catch(reject); 32 | }); 33 | } 34 | 35 | function cli() { 36 | if (process.argv.length !== 4) { 37 | console.error(`Usage: ${process.argv[0]} ${process.argv[1]} [,user2,user3]`); 38 | process.exit(1); 39 | } 40 | 41 | const role = process.argv[2]; 42 | const users = process.argv[3].split(','); 43 | 44 | cassandraConnector.initialize() 45 | .then(() => addUsers(role, users)) 46 | .then(result => { 47 | console.log(result); 48 | process.exit(0); 49 | }) 50 | .catch(error => { 51 | console.error('Failed to add users'); 52 | console.error(error); 53 | process.exit(1); 54 | }); 55 | } 56 | 57 | cli(); 58 | -------------------------------------------------------------------------------- /project-fortis-services/src/utils/collections.js: -------------------------------------------------------------------------------- 1 | const Long = require('cassandra-driver').types.Long; 2 | 3 | function _sortByMentionCount(rows) { 4 | return rows.sort((a, b) => b.mentions - a.mentions); 5 | } 6 | 7 | function _computeWeightedSentiment(rows) { 8 | return rows.map(row => Object.assign({}, row, { avgsentiment: computeWeightedAvg(row.mentions, row.avgsentimentnumerator) })); 9 | } 10 | 11 | function computeWeightedAvg(mentioncount, weightedavgnumerator) { 12 | const DoubleToLongConversionFactor = 1000; 13 | 14 | return !mentioncount.isZero() ? (weightedavgnumerator / DoubleToLongConversionFactor) / mentioncount : 0; 15 | } 16 | 17 | function makeMap(iterable, keyFunc, valueFunc) { 18 | const map = {}; 19 | iterable.forEach(item => { 20 | const key = keyFunc(item); 21 | const value = valueFunc(item); 22 | map[key] = value; 23 | }); 24 | return map; 25 | } 26 | 27 | function makeSet(iterable, func) { 28 | const set = new Set(); 29 | iterable.forEach(item => set.add(func(item))); 30 | return set; 31 | } 32 | 33 | function aggregateBy(rows, aggregateKey, aggregateValue) { 34 | let accumulationMap = new Map(); 35 | 36 | rows.forEach(row => { 37 | const key = aggregateKey(row); 38 | const mapEntry = accumulationMap.has(key) ? accumulationMap.get(key) : aggregateValue(row); 39 | const mutatedRow = Object.assign({}, mapEntry, { 40 | mentions: (mapEntry.mentions || Long.ZERO).add(row.mentioncount), 41 | avgsentimentnumerator: (mapEntry.avgsentimentnumerator || Long.ZERO).add(row.avgsentimentnumerator || Long.ZERO) 42 | }); 43 | 44 | accumulationMap.set(key, mutatedRow); 45 | }); 46 | 47 | return _sortByMentionCount(_computeWeightedSentiment(Array.from(accumulationMap.values()))); 48 | } 49 | 50 | module.exports = { 51 | makeMap, 52 | aggregateBy, 53 | makeSet, 54 | computeWeightedAvg 55 | }; 56 | -------------------------------------------------------------------------------- /project-fortis-services/src/utils/request.js: -------------------------------------------------------------------------------- 1 | const anonymousUser = 'anonymous@fortis'; 2 | 3 | function getUserFromArgs(...args) { 4 | return (args && args.length >= 2 && args[1].user && args[1].user.identifier) || anonymousUser; 5 | } 6 | 7 | module.exports = { 8 | getUserFromArgs 9 | }; 10 | -------------------------------------------------------------------------------- /project-fortis-services/travis/ci.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | pushd "$(dirname $0)/.." 6 | 7 | err=0 8 | 9 | npm install 10 | 11 | if ! ./node_modules/.bin/eslint --max-warnings=0 src *.js; then 12 | err=1 13 | fi 14 | 15 | if ./node_modules/.bin/depcheck | grep -q '^Unused dependencies$'; then 16 | err=2 17 | fi 18 | 19 | popd 20 | 21 | exit "$err" 22 | -------------------------------------------------------------------------------- /project-fortis-services/travis/publish.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | log() { 6 | echo "$@" >&2 7 | } 8 | 9 | check_preconditions() { 10 | if [ -z "${TRAVIS_TAG}" ]; then 11 | log "Build is not a tag, skipping publish" 12 | exit 0 13 | fi 14 | if [ -z "${DOCKER_USERNAME}" ] || [ -z "${DOCKER_PASSWORD}" ]; then 15 | log "Docker credentials not provided, unable to publish builds" 16 | exit 1 17 | fi 18 | } 19 | 20 | create_image() { 21 | touch .env-secrets 22 | BUILD_TAG="${TRAVIS_TAG}" docker-compose build project_fortis_services 23 | } 24 | 25 | publish_image() { 26 | docker login --username="${DOCKER_USERNAME}" --password="${DOCKER_PASSWORD}" 27 | BUILD_TAG="${TRAVIS_TAG}" docker-compose push project_fortis_services 28 | } 29 | 30 | pushd "$(dirname $0)/../.." 31 | 32 | check_preconditions 33 | create_image 34 | publish_image 35 | 36 | popd 37 | -------------------------------------------------------------------------------- /project-fortis-spark/.dockerignore: -------------------------------------------------------------------------------- 1 | travis/ 2 | target/ 3 | .idea/ 4 | README.md 5 | .gitignore 6 | version.sbt 7 | -------------------------------------------------------------------------------- /project-fortis-spark/.gitignore: -------------------------------------------------------------------------------- 1 | dist/* 2 | target/ 3 | lib_managed/ 4 | src_managed/ 5 | project/boot/ 6 | project/plugins/project/ 7 | .history 8 | .cache 9 | .lib/ 10 | -------------------------------------------------------------------------------- /project-fortis-spark/docker/run-cqlsh.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | "$CASSANDRA_HOME/bin/cqlsh" \ 4 | --request-timeout=3600 \ 5 | --username="$FORTIS_CASSANDRA_USERNAME" \ 6 | --password="$FORTIS_CASSANDRA_PASSWORD" \ 7 | "$FORTIS_CASSANDRA_HOST" \ 8 | "$FORTIS_CASSANDRA_PORT" 9 | -------------------------------------------------------------------------------- /project-fortis-spark/docker/run-spark.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | log() { 4 | echo "[$(date)] $1" 5 | } 6 | 7 | has_site() { 8 | echo 'SELECT * FROM settings.sitesettings;' | /app/cqlsh | grep -q '(1 rows)' 9 | } 10 | 11 | get_token() { 12 | echo "COPY settings.sitesettings($1) TO STDOUT;" | /app/cqlsh | tr -dC 'A-Za-z0-9' 13 | } 14 | 15 | wait_for_token() { 16 | local token="$1" 17 | local value="" 18 | 19 | while :; do 20 | value="$(get_token ${token})" 21 | if [ -n "${value}" ]; then break; else log "Cognitive Services token ${token} not yet available, waiting..."; sleep 10s; fi 22 | done 23 | log "...done, token ${token} is now available with value '${value}'" 24 | } 25 | 26 | # wait for cassandra to start 27 | while ! /app/cqlsh; do 28 | log "Cassandra not yet available, waiting..." 29 | sleep 10s 30 | done 31 | log "...done, Cassandra is now available" 32 | 33 | # wait for cassandra site to be defined 34 | while ! has_site; do 35 | log "Cassandra site is not yet set up, waiting..." 36 | sleep 10s 37 | done 38 | log "...done, Cassandra site is now set up" 39 | 40 | # wait for cognitive services secrets if preconfigured 41 | if [ -n "$COGNITIVE_TRANSLATION_SERVICE_TOKEN" ]; then 42 | wait_for_token "translationsvctoken" 43 | fi 44 | if [ -n "$COGNITIVE_SPEECH_SERVICE_TOKEN" ]; then 45 | wait_for_token "cogspeechsvctoken" 46 | fi 47 | if [ -n "$COGNITIVE_VISION_SERVICE_TOKEN" ]; then 48 | wait_for_token "cogvisionsvctoken" 49 | fi 50 | if [ -n "$COGNITIVE_TEXT_SERVICE_TOKEN" ]; then 51 | wait_for_token "cogtextsvctoken" 52 | fi 53 | 54 | # wait for featureservice 55 | while ! wget -qO- "$FORTIS_FEATURE_SERVICE_HOST/features/name/paris" > /dev/null; do 56 | log "featureService not yet available, waiting..." 57 | sleep 30s 58 | done 59 | log "...done, featureService is now available" 60 | 61 | while ! spark-submit --driver-memory "${SPARK_DRIVER_MEMORY}" --class "${SPARK_MAINCLASS}" /app/job.jar; do 62 | log "Spark job finished, restarting." 63 | sleep 3s 64 | done 65 | -------------------------------------------------------------------------------- /project-fortis-spark/lib/spark-streaming-twitter_2.11-2.2.0-SNAPSHOT.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-spark/lib/spark-streaming-twitter_2.11-2.2.0-SNAPSHOT.jar -------------------------------------------------------------------------------- /project-fortis-spark/lib/tritonus_remaining-0.3.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-spark/lib/tritonus_remaining-0.3.6.jar -------------------------------------------------------------------------------- /project-fortis-spark/lib/tritonus_share-0.3.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CatalystCode/project-fortis/e9843b080e33c2959b1021cb20247ba51a695a6a/project-fortis-spark/lib/tritonus_share-0.3.6.jar -------------------------------------------------------------------------------- /project-fortis-spark/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.13 -------------------------------------------------------------------------------- /project-fortis-spark/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3") -------------------------------------------------------------------------------- /project-fortis-spark/src/main/resources/ApplicationInsights.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/Constants.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark 2 | 3 | object Constants { 4 | val SparkAppName = "project-fortis-spark" 5 | val SparkMasterDefault = "local[*]" 6 | val SparkStreamingBatchSizeDefault = 30 7 | 8 | val EventHubProgressDir = "eventhubProgress" 9 | val SscInitRetryAfterMillis = 60*1000 10 | val SscShutdownDelayMillis = 60*1000 11 | 12 | val maxKeywordsPerEventDefault = 5 13 | val maxLocationsPerEventDefault = 4 14 | 15 | object Env { 16 | val SparkStreamingBatchSize = "FORTIS_STREAMING_DURATION_IN_SECONDS" 17 | val HighlyAvailableProgressDir = "HA_PROGRESS_DIR" 18 | val AppInsightsKey = "APPINSIGHTS_INSTRUMENTATIONKEY" 19 | val FeatureServiceUrlBase = "FORTIS_FEATURE_SERVICE_HOST" 20 | val BlobUrlBase = "FORTIS_CENTRAL_ASSETS_HOST" 21 | val CassandraHost = "FORTIS_CASSANDRA_HOST" 22 | val CassandraPort = "FORTIS_CASSANDRA_PORT" 23 | val Location = "FORTIS_RESOURCE_GROUP_LOCATION" 24 | val CassandraUsername = "FORTIS_CASSANDRA_USERNAME" 25 | val CassandraPassword = "FORTIS_CASSANDRA_PASSWORD" 26 | val ManagementBusConnectionString = "FORTIS_SB_CONN_STR" 27 | val ManagementBusConfigQueueName = "FORTIS_SB_CONFIG_QUEUE" 28 | val ManagementBusCommandQueueName = "FORTIS_SB_COMMAND_QUEUE" 29 | val SscInitRetryAfterMillis = "FORTIS_SSC_INIT_RETRY_AFTER_MILLIS" 30 | val SscShutdownDelayMillis = "FORTIS_SSC_SHUTDOWN_DELAY_MILLIS" 31 | val MaxKeywordsPerEvent = "FORTIS_EVENT_MAX_KEYWORDS" 32 | val MaxLocationsPerEvent = "FORTIS_EVENT_MAX_LOCATIONS" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/FortisSettings.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark 2 | 3 | case class FortisSettings( 4 | progressDir: String, 5 | featureServiceUrlBase: String, 6 | cognitiveUrlBase: String, 7 | blobUrlBase: String, 8 | cassandraHosts: String, 9 | cassandraPorts: String, 10 | cassandraUsername: String, 11 | cassandraPassword: String, 12 | managementBusConnectionString: String, 13 | managementBusConfigQueueName: String, 14 | managementBusCommandQueueName: String, 15 | appInsightsKey: Option[String], 16 | sscInitRetryAfterMillis: Long, 17 | sscShutdownDelayMillis: Long, 18 | maxKeywordsPerEvent: Int, 19 | maxLocationsPerEvent: Int 20 | ) -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/analyzer/Analyzer.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.analyzer 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.dto.{Analysis, Location, Tag} 4 | import com.microsoft.partnercatalyst.fortis.spark.transforms.image.ImageAnalyzer 5 | import com.microsoft.partnercatalyst.fortis.spark.transforms.language.LanguageDetector 6 | import com.microsoft.partnercatalyst.fortis.spark.transforms.locations.LocationsExtractor 7 | import com.microsoft.partnercatalyst.fortis.spark.transforms.people.PeopleRecognizer 8 | import com.microsoft.partnercatalyst.fortis.spark.transforms.sentiment.SentimentDetector 9 | import com.microsoft.partnercatalyst.fortis.spark.transforms.summary.Summarizer 10 | import com.microsoft.partnercatalyst.fortis.spark.transforms.topic.{Blacklist, KeywordExtractor} 11 | 12 | trait Analyzer[T] { 13 | type LocationFetcher = (Double, Double) => Iterable[Location] 14 | 15 | def toSchema(item: T, locationFetcher: LocationFetcher, imageAnalyzer: ImageAnalyzer): ExtendedDetails[T] 16 | def hasBlacklistedTerms(details: ExtendedDetails[T], blacklist: Blacklist): Boolean 17 | def hasBlacklistedEntities(analysis: Analysis, blacklist: Blacklist): Boolean 18 | def extractKeywords(details: ExtendedDetails[T], keywordExtractor: KeywordExtractor): List[Tag] 19 | def extractLocations(details: ExtendedDetails[T], locationsExtractor: LocationsExtractor): List[Location] 20 | def extractEntities(details: ExtendedDetails[T], peopleRecognizer: PeopleRecognizer): List[Tag] 21 | def detectLanguage(details: ExtendedDetails[T], languageDetector: LanguageDetector): Option[String] 22 | def detectSentiment(details: ExtendedDetails[T], sentimentDetector: SentimentDetector): List[Double] 23 | def createSummary(details: ExtendedDetails[T], summarizer: Summarizer): Option[String] 24 | } -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/analyzer/BingAnalyzer.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.analyzer 2 | 3 | import java.net.URL 4 | import java.text.SimpleDateFormat 5 | import java.util.TimeZone 6 | 7 | import com.github.catalystcode.fortis.spark.streaming.bing.dto.BingPost 8 | import com.microsoft.partnercatalyst.fortis.spark.transforms.image.ImageAnalyzer 9 | 10 | @SerialVersionUID(100L) 11 | class BingAnalyzer extends Analyzer[BingPost] with Serializable 12 | with AnalysisDefaults.EnableAll[BingPost] { 13 | 14 | private val DefaultFormat = "yyyy-MM-dd'T'HH:mm:ss" 15 | private val DefaultTimezone = "UTC" 16 | 17 | override def toSchema(item: BingPost, locationFetcher: LocationFetcher, imageAnalyzer: ImageAnalyzer): ExtendedDetails[BingPost] = { 18 | ExtendedDetails( 19 | eventid = s"Bing.${item.url}", 20 | sourceeventid = item.url, 21 | eventtime = convertDatetimeStringToEpochLong(item.dateLastCrawled), 22 | externalsourceid = new URL(item.url).getHost, 23 | body = item.snippet, 24 | title = item.name, 25 | imageurl = None, 26 | pipelinekey = "Bing", 27 | sourceurl = item.url, 28 | original = item 29 | ) 30 | } 31 | 32 | private def convertDatetimeStringToEpochLong(dateStr: String, format: Option[String] = None, timezone: Option[String] = None): Long ={ 33 | val sdf = new SimpleDateFormat(format.getOrElse(DefaultFormat)) 34 | sdf.setTimeZone(TimeZone.getTimeZone(timezone.getOrElse(DefaultTimezone))) 35 | 36 | sdf.parse(dateStr).getTime 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/analyzer/CustomEventAnalyzer.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.analyzer 2 | 3 | import java.util.UUID.randomUUID 4 | 5 | import com.microsoft.partnercatalyst.fortis.spark.sources.streamwrappers.customevents.CustomEvent 6 | import com.microsoft.partnercatalyst.fortis.spark.transforms.image.ImageAnalyzer 7 | 8 | @SerialVersionUID(100L) 9 | class CustomEventAnalyzer extends Analyzer[CustomEvent] with Serializable 10 | with AnalysisDefaults.EnableAll[CustomEvent] { 11 | override def toSchema(item: CustomEvent, locationFetcher: LocationFetcher, imageAnalyzer: ImageAnalyzer): ExtendedDetails[CustomEvent] = { 12 | ExtendedDetails( 13 | eventid = s"${item.source.getOrElse("CustomEvent")}.${randomUUID()}", 14 | sourceeventid = item.RowKey, 15 | externalsourceid = item.source.getOrElse("N/A"), 16 | eventtime = item.created_at.toLong, 17 | body = item.message, 18 | title = item.title.getOrElse(""), 19 | imageurl = None, 20 | pipelinekey = item.source.getOrElse("CustomEvent"), 21 | sourceurl = item.link.getOrElse(""), 22 | original = item 23 | ) 24 | } 25 | } -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/analyzer/ExtendedFortisEvent.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.analyzer 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.dto.{Analysis, Details, FortisEvent, Location} 4 | 5 | case class ExtendedFortisEvent[T]( 6 | details: ExtendedDetails[T], 7 | analysis: Analysis 8 | ) extends FortisEvent { 9 | override def copy(analysis: Analysis) = { 10 | ExtendedFortisEvent[T]( 11 | details = details, 12 | analysis = Option(analysis).getOrElse(this.analysis)) 13 | } 14 | } 15 | 16 | case class ExtendedDetails[T]( 17 | eventid: String, 18 | sourceeventid: String, 19 | eventtime: Long, 20 | body: String, 21 | title: String, 22 | imageurl: Option[String], 23 | pipelinekey: String, 24 | externalsourceid: String, 25 | sourceurl: String, 26 | sharedLocations: List[Location] = List(), 27 | original: T 28 | ) extends Details -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/analyzer/FacebookCommentAnalyzer.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.analyzer 2 | 3 | import java.util.Date 4 | 5 | import com.github.catalystcode.fortis.spark.streaming.facebook.dto.FacebookComment 6 | import com.microsoft.partnercatalyst.fortis.spark.transforms.image.ImageAnalyzer 7 | 8 | @SerialVersionUID(100L) 9 | class FacebookCommentAnalyzer extends Analyzer[FacebookComment] with Serializable 10 | with AnalysisDefaults.EnableAll[FacebookComment] { 11 | override def toSchema(item: FacebookComment, locationFetcher: LocationFetcher, imageAnalyzer: ImageAnalyzer): ExtendedDetails[FacebookComment] = { 12 | ExtendedDetails( 13 | eventid = s"Facebook.comment.${item.comment.getId}", 14 | sourceeventid = item.comment.getId, 15 | eventtime = Option(item.comment.getCreatedTime).getOrElse(new Date()).getTime, 16 | body = Option(item.comment.getMessage).getOrElse(""), 17 | title = s"Post ${item.postId}: Comment", 18 | externalsourceid = item.pageId, 19 | pipelinekey = "Facebook", 20 | imageurl = None, 21 | sourceurl = "", 22 | original = item 23 | ) 24 | } 25 | } -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/analyzer/FacebookPostAnalyzer.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.analyzer 2 | 3 | import java.util.Date 4 | 5 | import com.github.catalystcode.fortis.spark.streaming.facebook.dto.FacebookPost 6 | import com.microsoft.partnercatalyst.fortis.spark.transforms.image.ImageAnalyzer 7 | 8 | import scala.util.Try 9 | 10 | @SerialVersionUID(100L) 11 | class FacebookPostAnalyzer extends Analyzer[FacebookPost] with Serializable 12 | with AnalysisDefaults.EnableAll[FacebookPost] { 13 | override def toSchema(item: FacebookPost, locationFetcher: LocationFetcher, imageAnalyzer: ImageAnalyzer): ExtendedDetails[FacebookPost] = { 14 | ExtendedDetails( 15 | eventid = s"Facebook.post.${item.post.getId}", 16 | sourceeventid = item.post.getId, 17 | eventtime = Option(Option(item.post.getUpdatedTime).getOrElse(item.post.getCreatedTime)).getOrElse(new Date()).getTime, 18 | body = Option(item.post.getMessage).getOrElse(""), 19 | title = Option(item.post.getCaption).getOrElse(""), 20 | imageurl = Option(item.post.getIcon) match { 21 | case Some(icon) => Option(icon.toString) 22 | case None => Some("") 23 | }, 24 | externalsourceid = item.pageId, 25 | pipelinekey = "Facebook", 26 | sharedLocations = Option(item.post.getPlace).map(_.getLocation) match { 27 | case Some(location) => locationFetcher(location.getLatitude, location.getLongitude).toList 28 | case None => List() 29 | }, 30 | sourceurl = Try(item.post.getPermalinkUrl.toString).getOrElse(""), 31 | original = item 32 | ) 33 | } 34 | } -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/analyzer/HTMLAnalyzer.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.analyzer 2 | 3 | import java.net.URL 4 | import java.text.SimpleDateFormat 5 | import java.util.Date 6 | 7 | import com.github.catalystcode.fortis.spark.streaming.html.HTMLPage 8 | import com.microsoft.partnercatalyst.fortis.spark.transforms.image.ImageAnalyzer 9 | import org.jsoup.Jsoup 10 | import org.jsoup.nodes.Document 11 | 12 | @SerialVersionUID(100L) 13 | class HTMLAnalyzer extends Analyzer[HTMLPage] with Serializable with AnalysisDefaults.EnableAll[HTMLPage] { 14 | 15 | override def toSchema(item: HTMLPage, locationFetcher: LocationFetcher, imageAnalyzer: ImageAnalyzer): ExtendedDetails[HTMLPage] = { 16 | val document = Jsoup.parse(item.html) 17 | val url = new URL(item.url) 18 | val body = document match { 19 | case null => item.html 20 | case _ => document.body().text() 21 | } 22 | ExtendedDetails( 23 | eventid = s"HTML.${item.url}", 24 | sourceeventid = item.url, 25 | eventtime = getDate(item, document), 26 | body = body, 27 | imageurl = None, 28 | title = document.title(), 29 | externalsourceid = url.getHost, 30 | pipelinekey = "HTML", 31 | sourceurl = item.url, 32 | sharedLocations = List(), 33 | original = item 34 | ) 35 | } 36 | 37 | private def getDate(item: HTMLPage, document: Document): Long = { 38 | try { 39 | val metaElements = document.select("meta[itemprop=datePublished]") 40 | if (metaElements != null) { 41 | val dateString = metaElements.first().attr("content") 42 | if (dateString != null) { 43 | return new SimpleDateFormat("YYYY-MM-dd").parse(dateString.trim).getTime 44 | } 45 | } 46 | new Date().getTime 47 | } catch { 48 | case _: Exception => new Date().getTime 49 | } 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/analyzer/RadioAnalyzer.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.analyzer 2 | 3 | import java.time.Instant.now 4 | import java.util.UUID.randomUUID 5 | 6 | import com.microsoft.partnercatalyst.fortis.spark.sources.streamwrappers.radio.RadioTranscription 7 | import com.microsoft.partnercatalyst.fortis.spark.transforms.image.ImageAnalyzer 8 | 9 | @SerialVersionUID(100L) 10 | class RadioAnalyzer extends Analyzer[RadioTranscription] with Serializable 11 | with AnalysisDefaults.EnableAll[RadioTranscription] { 12 | override def toSchema(item: RadioTranscription, locationFetcher: LocationFetcher, imageAnalyzer: ImageAnalyzer): ExtendedDetails[RadioTranscription] = { 13 | ExtendedDetails( 14 | eventid = s"Radio.${randomUUID()}", 15 | sourceeventid = "", 16 | eventtime = now.getEpochSecond, 17 | externalsourceid = item.radioUrl, 18 | body = item.text, 19 | title = "", 20 | imageurl = None, 21 | pipelinekey = "Radio", 22 | sourceurl = item.radioUrl, 23 | original = item 24 | ) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/analyzer/RedditAnalyzer.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.analyzer 2 | 3 | import java.util.UUID.randomUUID 4 | 5 | import com.github.catalystcode.fortis.spark.streaming.reddit.dto.RedditObject 6 | import com.microsoft.partnercatalyst.fortis.spark.transforms.image.ImageAnalyzer 7 | 8 | @SerialVersionUID(100L) 9 | class RedditAnalyzer extends Analyzer[RedditObject] with Serializable 10 | with AnalysisDefaults.EnableAll[RedditObject] { 11 | override def toSchema(item: RedditObject, locationFetcher: LocationFetcher, imageAnalyzer: ImageAnalyzer): ExtendedDetails[RedditObject] = { 12 | ExtendedDetails( 13 | eventid = s"Reddit.${item.data.id.getOrElse(randomUUID()).toString}", 14 | sourceeventid = item.data.id.getOrElse("").toString, 15 | eventtime = item.data.created_utc.get.toLong, 16 | body = item.data.description.getOrElse(""), 17 | title = item.data.title.getOrElse(""), 18 | imageurl = None, 19 | externalsourceid = item.data.author.getOrElse(""), 20 | pipelinekey = "Reddit", 21 | sourceurl = item.data.url.getOrElse(""), 22 | original = item 23 | ) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/analyzer/TadawebAnalyzer.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.analyzer 2 | 3 | import java.time.Instant.now 4 | 5 | import com.microsoft.partnercatalyst.fortis.spark.sources.streamwrappers.tadaweb.TadawebEvent 6 | import com.microsoft.partnercatalyst.fortis.spark.transforms.image.ImageAnalyzer 7 | import com.microsoft.partnercatalyst.fortis.spark.transforms.sentiment.SentimentDetector 8 | 9 | @SerialVersionUID(100L) 10 | class TadawebAnalyzer extends Analyzer[TadawebEvent] with Serializable 11 | with AnalysisDefaults.EnableAll[TadawebEvent] { 12 | override def toSchema(item: TadawebEvent, locationFetcher: LocationFetcher, imageAnalyzer: ImageAnalyzer): ExtendedDetails[TadawebEvent] = { 13 | ExtendedDetails( 14 | eventid = s"TadaWeb.${item.tada.id}", 15 | sourceeventid = item.tada.id, 16 | externalsourceid = item.tada.name, 17 | eventtime = now.getEpochSecond, 18 | body = item.text, 19 | title = item.title, 20 | imageurl = None, 21 | pipelinekey = "TadaWeb", 22 | sourceurl = item.link, 23 | sharedLocations = item.cities.flatMap(city => city.coordinates match { 24 | case Seq(latitude, longitude) => locationFetcher(latitude, longitude) 25 | case _ => None 26 | }).toList, 27 | original = item 28 | ) 29 | } 30 | 31 | override def detectSentiment(details: ExtendedDetails[TadawebEvent], sentimentDetector: SentimentDetector): List[Double] = { 32 | details.original.sentiment match { 33 | case "negative" => List(SentimentDetector.Negative) 34 | case "neutral" => List(SentimentDetector.Neutral) 35 | case "positive" => List(SentimentDetector.Positive) 36 | case _ => super.detectSentiment(details, sentimentDetector) 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/analyzer/TwitterAnalyzer.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.analyzer 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.transforms.image.ImageAnalyzer 4 | import com.microsoft.partnercatalyst.fortis.spark.transforms.language.LanguageDetector 5 | import twitter4j.{Status => TwitterStatus} 6 | 7 | @SerialVersionUID(100L) 8 | class TwitterAnalyzer extends Analyzer[TwitterStatus] with Serializable 9 | with AnalysisDefaults.EnableAll[TwitterStatus] { 10 | 11 | override def toSchema(item: TwitterStatus, locationFetcher: LocationFetcher, imageAnalyzer: ImageAnalyzer): ExtendedDetails[TwitterStatus] = { 12 | ExtendedDetails( 13 | eventid = s"Twitter.${item.getId}", 14 | sourceeventid = item.getId.toString, 15 | eventtime = item.getCreatedAt.getTime, 16 | body = item.getText, 17 | title = "", 18 | imageurl = None, 19 | externalsourceid = item.getUser.getScreenName, 20 | pipelinekey = "Twitter", 21 | sourceurl = s"https://twitter.com/statuses/${item.getId}", 22 | sharedLocations = Option(item.getGeoLocation) match { 23 | case Some(location) => locationFetcher(location.getLatitude, location.getLongitude).toList 24 | case None => List() 25 | }, 26 | original = item 27 | ) 28 | } 29 | 30 | override def detectLanguage(details: ExtendedDetails[TwitterStatus], languageDetector: LanguageDetector): Option[String] = { 31 | Option(details.original.getLang) match { 32 | case Some(lang) => Some(lang) 33 | case None => super.detectLanguage(details, languageDetector) 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/dba/CassandraSchema.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.dba 2 | 3 | object CassandraSchema { 4 | val KeyspaceName = "settings" 5 | 6 | object Table { 7 | val BlacklistName = "blacklist" 8 | val WatchlistName = "watchlist" 9 | val SiteSettingsName = "sitesettings" 10 | val StreamsName = "streams" 11 | val TrustedSourcesName = "trustedsources" 12 | 13 | case class Stream( 14 | pipelinekey: String, 15 | streamid: String, 16 | enabled: Option[Boolean], 17 | params_json: String, 18 | pipelineicon: String, 19 | pipelinelabel: String, 20 | streamfactory: String 21 | ) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/dba/ConfigurationManager.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.dba 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.dto.{BlacklistedItem, SiteSettings} 4 | import com.microsoft.partnercatalyst.fortis.spark.sources.streamprovider.ConnectorConfig 5 | import org.apache.spark.SparkContext 6 | 7 | trait ConfigurationManager { 8 | def fetchConnectorConfigs(sparkContext: SparkContext, pipeline: String): List[ConnectorConfig] 9 | def fetchSiteSettings(sparkContext: SparkContext): SiteSettings 10 | 11 | def fetchWatchlist(sparkContext: SparkContext): Map[String, Seq[String]] 12 | def fetchBlacklist(sparkContext: SparkContext): Seq[BlacklistedItem] 13 | } 14 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/dto/BlacklistedItem.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.dto 2 | 3 | case class BlacklistedItem( 4 | conjunctiveFilter: Set[String], 5 | isLocation: Boolean 6 | ) -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/dto/ComputedTile.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.dto 2 | 3 | case class ComputedTile(periodstartdate: Long, 4 | periodenddate: Long, 5 | periodtype: String, 6 | period: String, 7 | pipelinekey: String, 8 | tilez: Int, 9 | tilex: Int, 10 | tiley: Int, 11 | externalsourceid: String, 12 | mentioncount: Int, 13 | avgsentiment: Int, 14 | heatmap: String, 15 | placeids: Seq[String], 16 | insertiontime: Long, 17 | conjunctiontopics: (Option[String], Option[String], Option[String])) extends Serializable 18 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/dto/ComputedTrend.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.dto 2 | 3 | import java.util.Date 4 | 5 | case class ComputedTrend(topic: String, 6 | periodstartdate: Long, 7 | periodtype: String, 8 | period: String, 9 | pipelinekey: String, 10 | tilez: Int, 11 | tilex: Int, 12 | tiley: Int, 13 | score: Double, 14 | insertiontime: Long) extends Serializable { 15 | def this(tile: ComputedTile, score: Double) = this( 16 | tile.conjunctiontopics._1.get, 17 | tile.periodstartdate, 18 | tile.periodtype, 19 | tile.period, 20 | tile.pipelinekey, 21 | tile.tilez, 22 | tile.tilex, 23 | tile.tiley, 24 | score, 25 | new Date().getTime 26 | ) 27 | } -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/dto/Geofence.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.dto 2 | 3 | case class Geofence(north: Double, west: Double, south: Double, east: Double) -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/dto/SiteSettings.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.dto 2 | 3 | import net.liftweb.json 4 | 5 | case class SiteSettings( 6 | sitename: String, 7 | geofence_json: String, 8 | defaultlanguage: Option[String], 9 | languages_json: String, 10 | defaultzoom: Int, 11 | featureservicenamespace: Option[String], 12 | title: String, 13 | logo: String, 14 | translationsvctoken: String, 15 | cogspeechsvctoken: String, 16 | cogvisionsvctoken: String, 17 | cogtextsvctoken: String, 18 | insertiontime: Long 19 | ) 20 | { 21 | 22 | def getAllLanguages(): Seq[String] = { 23 | implicit val formats = json.DefaultFormats 24 | 25 | val languages = json.parse(languages_json).extract[List[String]] 26 | 27 | defaultlanguage match { 28 | case None => languages 29 | case Some(language) => (Set(language) ++ languages.toSet).toSeq 30 | } 31 | } 32 | 33 | def getGeofence(): Geofence = { 34 | implicit val formats = json.DefaultFormats 35 | 36 | val geofence = json.parse(geofence_json).extract[List[Double]] 37 | 38 | Geofence( 39 | north = geofence(0), 40 | west = geofence(1), 41 | south = geofence(2), 42 | east = geofence(3) 43 | ) 44 | } 45 | 46 | } -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/logging/FortisTelemetry.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.logging 2 | 3 | trait FortisTelemetry { 4 | def logDebug(trace: String): Unit 5 | def logInfo(trace: String): Unit 6 | def logError(trace: String, exception: Throwable=null): Unit 7 | def logEvent(name: String, properties: Map[String, String]=Map(), metrics: Map[String, Double]=Map()): Unit 8 | def logDependency(name: String, method: String, success: Boolean, durationInMs: Long) 9 | } 10 | 11 | object FortisTelemetry { 12 | private lazy val telemetry: FortisTelemetry = new AppInsightsTelemetry() 13 | 14 | def get: FortisTelemetry = telemetry 15 | } -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/logging/Timer.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.logging 2 | 3 | import scala.util.{Failure, Success, Try} 4 | 5 | object Timer { 6 | def time[R](callback: (Boolean, Long) => Unit)(block: => R): R = { 7 | val startTime = System.nanoTime() 8 | val result = Try(block) 9 | val endTime = System.nanoTime() 10 | 11 | val duration = endTime - startTime 12 | callback(result.isSuccess, duration) 13 | 14 | result match { 15 | case Success(res) => res 16 | case Failure(th) => throw th 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sinks/cassandra/CassandraConfig.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sinks.cassandra 2 | 3 | import com.datastax.driver.core.ConsistencyLevel 4 | import com.microsoft.partnercatalyst.fortis.spark.FortisSettings 5 | import org.apache.spark.SparkConf 6 | import org.apache.spark.streaming.Duration 7 | 8 | import scala.util.Properties.envOrElse 9 | 10 | object CassandraConfig { 11 | def init(conf: SparkConf, batchDuration: Duration, fortisSettings: FortisSettings): SparkConf = { 12 | conf 13 | .setIfMissing("spark.cassandra.connection.host", fortisSettings.cassandraHosts) 14 | .setIfMissing("spark.cassandra.connection.port", fortisSettings.cassandraPorts) 15 | .setIfMissing("spark.cassandra.auth.username", fortisSettings.cassandraUsername) 16 | .setIfMissing("spark.cassandra.auth.password", fortisSettings.cassandraPassword) 17 | .setIfMissing("spark.cassandra.input.consistency.level", ConsistencyLevel.LOCAL_QUORUM.toString) 18 | .setIfMissing("spark.cassandra.connection.keep_alive_ms", envOrElse("CASSANDRA_KEEP_ALIVE_MS", (batchDuration.milliseconds * 2).toString)) 19 | .setIfMissing("spark.cassandra.connection.factory", "com.microsoft.partnercatalyst.fortis.spark.sinks.cassandra.FortisConnectionFactory") 20 | .set("spark.cassandra.output.batch.size.bytes", "5120") 21 | .set("spark.cassandra.output.concurrent.writes", "16") 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sinks/cassandra/CassandraExtensions.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sinks.cassandra 2 | 3 | import com.datastax.spark.connector.cql.{CassandraConnector, Schema, TableDef} 4 | import com.datastax.spark.connector.writer._ 5 | import org.apache.spark.rdd.RDD 6 | 7 | import scala.reflect.ClassTag 8 | 9 | object CassandraExtensions { 10 | implicit class CassandraRDD[K, V](val rdd: RDD[(K, V)]) extends AnyVal { 11 | def deDupValuesByCassandraTable(keyspaceName: String, tableName: String) 12 | (implicit connector: CassandraConnector = CassandraConnector(rdd.sparkContext), rwf: RowWriterFactory[V], kt: ClassTag[K], vt: ClassTag[V], ord: Ordering[K]): RDD[(K, V)] = 13 | { 14 | val tableDef = Schema.tableFromCassandra(connector, keyspaceName, tableName) 15 | rdd.deDupValuesByCassandraTable(tableDef) 16 | } 17 | 18 | def deDupValuesByCassandraTable(tableDef: TableDef) 19 | (implicit rwf: RowWriterFactory[V], kt: ClassTag[K], vt: ClassTag[V], ord: Ordering[K]): RDD[(K, V)] = 20 | { 21 | val rowWriter = implicitly[RowWriterFactory[V]].rowWriter(tableDef, tableDef.primaryKey.map(_.ref)) 22 | val primaryKeySize = tableDef.primaryKey.length 23 | 24 | rdd.groupByKey().mapValues(eventRows => { 25 | eventRows.groupBy(value => { 26 | // Group by an ordered list of primary key column values. 27 | // Resulting groups will be rows that would collide. We take 'head' of each group in order to de-dup. 28 | val buffer = new Array[Any](primaryKeySize) 29 | rowWriter.readColumnValues(value, buffer) 30 | 31 | buffer.toList 32 | }).mapValues(_.head).values 33 | }).flatMap { case (event, uniqueRows) => uniqueRows.map((event, _)) } 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sinks/cassandra/aggregators/ConjunctiveTopicsOffineAggregator.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sinks.cassandra.aggregators 2 | 3 | import com.datastax.spark.connector._ 4 | import com.datastax.spark.connector.writer.SqlRowWriter 5 | import com.microsoft.partnercatalyst.fortis.spark.dba.ConfigurationManager 6 | import com.microsoft.partnercatalyst.fortis.spark.sinks.cassandra.CassandraConjunctiveTopics 7 | import com.microsoft.partnercatalyst.fortis.spark.sinks.cassandra.dto.{ConjunctiveTopic, Event} 8 | import org.apache.spark.rdd.RDD 9 | import com.microsoft.partnercatalyst.fortis.spark.sinks.cassandra.Constants._ 10 | import com.microsoft.partnercatalyst.fortis.spark.sinks.cassandra.CassandraExtensions._ 11 | 12 | class ConjunctiveTopicsOffineAggregator(configurationManager: ConfigurationManager) extends (RDD[Event] => Unit) { 13 | override def apply(events: RDD[Event]): Unit = { 14 | val topics = aggregate(events).cache() 15 | topics.count() match { 16 | case 0 => return 17 | case _ => 18 | implicit val rowWriter: SqlRowWriter.Factory.type = SqlRowWriter.Factory 19 | topics.saveToCassandra(KeyspaceName, Table.ConjunctiveTopics) 20 | } 21 | 22 | topics.unpersist(blocking = true) 23 | } 24 | 25 | private[aggregators] def aggregate(events: RDD[Event]): RDD[ConjunctiveTopic] = { 26 | val siteSettings = configurationManager.fetchSiteSettings(events.sparkContext) 27 | val conjunctiveTopicsByEvent = CassandraConjunctiveTopics(events, siteSettings.defaultzoom).keyBy(_.eventid) 28 | 29 | conjunctiveTopicsByEvent.deDupValuesByCassandraTable(KeyspaceName, Table.ConjunctiveTopics).values 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sinks/cassandra/dto/AggregationRecord.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sinks.cassandra.dto 2 | 3 | trait AggregationRecord { 4 | val perioddate: Long 5 | val periodtype: String 6 | val pipelinekey: String 7 | val mentioncount: Long 8 | val avgsentimentnumerator: Long 9 | val externalsourceid: String 10 | } 11 | 12 | trait AggregationRecordTile extends AggregationRecord { 13 | val tileid: String 14 | val tilez: Int 15 | } -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sinks/cassandra/dto/FortisRecords.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sinks.cassandra.dto 2 | 3 | case class Event( 4 | pipelinekey: String, 5 | computedfeatures_json: String, 6 | eventtime: Long, 7 | eventlangcode: String, 8 | eventid: String, 9 | sourceeventid: String, 10 | insertiontime: Long, 11 | body: String, 12 | summary: String, 13 | imageurl: Option[String], 14 | batchid: String, 15 | externalsourceid: String, 16 | sourceurl: String, 17 | title: String 18 | ) extends Serializable 19 | 20 | case class EventBatchEntry( 21 | eventid: String, 22 | pipelinekey: String 23 | ) extends Serializable 24 | 25 | case class TileRow( 26 | externalsourceid: String, 27 | perioddate: Long, 28 | periodtype: String, 29 | pipelinekey: String, 30 | mentioncount: Long, 31 | avgsentimentnumerator: Long, 32 | tilez: Int, 33 | tileid: String, 34 | heatmaptileid: String, 35 | centroidlat: Double, 36 | centroidlon: Double, 37 | conjunctiontopic1: String, 38 | conjunctiontopic2: String, 39 | conjunctiontopic3: String, 40 | eventtime: Long, 41 | placeid: String, 42 | eventid: String, 43 | insertiontime: Long 44 | ) extends Serializable 45 | 46 | 47 | case class ConjunctiveTopic( 48 | eventid: String, 49 | conjunctivetopic: String, 50 | externalsourceid: String, 51 | mentioncount: Long, 52 | perioddate: Long, 53 | periodtype: String, 54 | pipelinekey: String, 55 | tileid: String, 56 | tilez: Int, 57 | topic: String 58 | ) extends Serializable 59 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sinks/cassandra/dto/UserDefinedTypes.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sinks.cassandra.dto 2 | 3 | import net.liftweb.json 4 | 5 | case class Sentiment(neg_avg: Double) extends Serializable 6 | 7 | case class Gender( 8 | male_mentions: Long, 9 | female_mentions: Long 10 | ) extends Serializable 11 | 12 | case class Entities( 13 | name: String, 14 | externalsource: String, 15 | externalrefid: String, 16 | count: Long 17 | ) extends Serializable 18 | 19 | case class Place( 20 | placeid: String, 21 | centroidlat: Double, 22 | centroidlon: Double 23 | ) extends Serializable 24 | 25 | case class Features( 26 | mentions: Long, 27 | sentiment: Sentiment, 28 | keywords: Seq[String], 29 | places: Seq[Place], 30 | entities: Seq[Entities] 31 | ) extends Serializable 32 | 33 | object Features { 34 | def asJson(features: Features): String = { 35 | implicit val formats = json.DefaultFormats 36 | 37 | json.compactRender(json.Extraction.decompose(features)) 38 | } 39 | 40 | def fromJson(features: String): Features = { 41 | implicit val formats = json.DefaultFormats 42 | 43 | json.parse(features).extract[Features] 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamfactories/BingPageStreamFactory.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamfactories 2 | 3 | import com.github.catalystcode.fortis.spark.streaming.bing.dto.BingPost 4 | import com.github.catalystcode.fortis.spark.streaming.bing.{BingAuth, BingUtils} 5 | import com.microsoft.partnercatalyst.fortis.spark.sources.streamprovider.ConnectorConfig 6 | import org.apache.spark.streaming.StreamingContext 7 | import org.apache.spark.streaming.dstream.DStream 8 | 9 | class BingPageStreamFactory extends StreamFactoryBase[BingPost]{ 10 | override protected def canHandle(connectorConfig: ConnectorConfig): Boolean = { 11 | "BingPage".equalsIgnoreCase(connectorConfig.name) 12 | } 13 | 14 | override protected def buildStream(ssc: StreamingContext, connectorConfig: ConnectorConfig): DStream[BingPost] = { 15 | import ParameterExtensions._ 16 | 17 | val params = connectorConfig.parameters 18 | val auth = BingAuth(params.getAs[String]("accessToken")) 19 | val searchInstanceId = params.getAs[String]("searchInstanceId") 20 | val keywords = params.getAs[String]("keywords").split('|') 21 | 22 | BingUtils.createPageStream(ssc, auth, searchInstanceId, keywords) 23 | } 24 | } -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamfactories/FacebookCommentStreamFactory.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamfactories 2 | 3 | import com.github.catalystcode.fortis.spark.streaming.facebook.dto.FacebookComment 4 | import com.github.catalystcode.fortis.spark.streaming.facebook.{FacebookAuth, FacebookUtils} 5 | import com.microsoft.partnercatalyst.fortis.spark.sources.streamprovider.ConnectorConfig 6 | import org.apache.spark.streaming.StreamingContext 7 | import org.apache.spark.streaming.dstream.DStream 8 | 9 | class FacebookCommentStreamFactory extends StreamFactoryBase[FacebookComment] { 10 | override protected def canHandle(connectorConfig: ConnectorConfig): Boolean = { 11 | "FacebookComment".equalsIgnoreCase(connectorConfig.name) 12 | } 13 | 14 | override protected def buildStream(ssc: StreamingContext, connectorConfig: ConnectorConfig): DStream[FacebookComment] = { 15 | import ParameterExtensions._ 16 | 17 | val params = connectorConfig.parameters 18 | val facebookAuth = FacebookAuth( 19 | params.getAs[String]("appId"), 20 | params.getAs[String]("appSecret"), 21 | params.getAs[String]("accessToken") 22 | ) 23 | 24 | FacebookUtils.createCommentsStreams(ssc, facebookAuth, params.getTrustedSources.toSet) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamfactories/FacebookPageStreamFactory.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamfactories 2 | 3 | import com.github.catalystcode.fortis.spark.streaming.facebook.dto.FacebookPost 4 | import com.github.catalystcode.fortis.spark.streaming.facebook.{FacebookAuth, FacebookUtils} 5 | import com.microsoft.partnercatalyst.fortis.spark.sources.streamprovider.ConnectorConfig 6 | import org.apache.spark.streaming.StreamingContext 7 | import org.apache.spark.streaming.dstream.DStream 8 | 9 | class FacebookPageStreamFactory extends StreamFactoryBase[FacebookPost] { 10 | override protected def canHandle(connectorConfig: ConnectorConfig): Boolean = { 11 | "FacebookPage".equalsIgnoreCase(connectorConfig.name) 12 | } 13 | 14 | override protected def buildStream(ssc: StreamingContext, connectorConfig: ConnectorConfig): DStream[FacebookPost] = { 15 | import ParameterExtensions._ 16 | 17 | val params = connectorConfig.parameters 18 | val facebookAuth = FacebookAuth( 19 | params.getAs[String]("appId"), 20 | params.getAs[String]("appSecret"), 21 | params.getAs[String]("accessToken") 22 | ) 23 | 24 | FacebookUtils.createPageStreams(ssc, facebookAuth, params.getTrustedSources.toSet) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamfactories/HTMLStreamFactory.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamfactories 2 | 3 | import com.github.catalystcode.fortis.spark.streaming.html.{HTMLInputDStream, HTMLPage} 4 | import com.microsoft.partnercatalyst.fortis.spark.sources.streamprovider.ConnectorConfig 5 | import org.apache.spark.storage.StorageLevel 6 | import org.apache.spark.streaming.StreamingContext 7 | import org.apache.spark.streaming.dstream.DStream 8 | 9 | class HTMLStreamFactory extends StreamFactoryBase[HTMLPage] { 10 | 11 | override protected def canHandle(connectorConfig: ConnectorConfig): Boolean = { 12 | "HTML".equalsIgnoreCase(connectorConfig.name) 13 | } 14 | 15 | override protected def buildStream(ssc: StreamingContext, connectorConfig: ConnectorConfig): DStream[HTMLPage] = { 16 | val params = connectorConfig.parameters 17 | connectorConfig.parameters.get("feedUrls") match { 18 | case Some(feedUrls:String) => 19 | val urls = feedUrls.split("[|]") 20 | new HTMLInputDStream( 21 | urls, 22 | ssc, 23 | storageLevel = StorageLevel.MEMORY_ONLY, 24 | requestHeaders = Map( 25 | "User-Agent" -> params.getOrElse("userAgent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36").toString 26 | ), 27 | maxDepth = params.getOrElse("maxDepth", "1").toString.toInt, 28 | pollingPeriodInSeconds = params.getOrElse("pollingPeriodInSeconds", "3600").toString.toInt, 29 | cacheEditDistanceThreshold = params.getOrElse("cacheEditDistanceThreshold", "0.0001").toString.toDouble 30 | ) 31 | case _ => 32 | throw new Exception("No feedUrls present for HTML feed stream $connectorConfig.") 33 | } 34 | } 35 | 36 | } 37 | 38 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamfactories/InstagramLocationStreamFactory.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamfactories 2 | 3 | import com.github.catalystcode.fortis.spark.streaming.instagram.dto.InstagramItem 4 | import com.github.catalystcode.fortis.spark.streaming.instagram.{InstagramAuth, InstagramUtils} 5 | import com.microsoft.partnercatalyst.fortis.spark.sources.streamprovider.ConnectorConfig 6 | import org.apache.spark.streaming.StreamingContext 7 | import org.apache.spark.streaming.dstream.DStream 8 | 9 | class InstagramLocationStreamFactory extends StreamFactoryBase[InstagramItem]{ 10 | override protected def canHandle(connectorConfig: ConnectorConfig): Boolean = { 11 | "InstagramLocation".equalsIgnoreCase(connectorConfig.name) 12 | } 13 | 14 | override protected def buildStream(ssc: StreamingContext, connectorConfig: ConnectorConfig): DStream[InstagramItem] = { 15 | import ParameterExtensions._ 16 | 17 | val params = connectorConfig.parameters 18 | val auth = InstagramAuth(params.getAs[String]("authToken")) 19 | 20 | InstagramUtils.createLocationStream( 21 | ssc, 22 | auth, 23 | latitude = params.getAs[String]("latitude").toDouble, 24 | longitude = params.getAs[String]("longitude").toDouble) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamfactories/InstagramTagStreamFactory.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamfactories 2 | 3 | import com.github.catalystcode.fortis.spark.streaming.instagram.dto.InstagramItem 4 | import com.github.catalystcode.fortis.spark.streaming.instagram.{InstagramAuth, InstagramUtils} 5 | import com.microsoft.partnercatalyst.fortis.spark.sources.streamprovider.ConnectorConfig 6 | import org.apache.spark.streaming.StreamingContext 7 | import org.apache.spark.streaming.dstream.DStream 8 | 9 | class InstagramTagStreamFactory extends StreamFactoryBase[InstagramItem]{ 10 | override protected def canHandle(connectorConfig: ConnectorConfig): Boolean = { 11 | "InstagramTag".equalsIgnoreCase(connectorConfig.name) 12 | } 13 | 14 | override protected def buildStream(ssc: StreamingContext, connectorConfig: ConnectorConfig): DStream[InstagramItem] = { 15 | import ParameterExtensions._ 16 | 17 | val params = connectorConfig.parameters 18 | val auth = InstagramAuth(params.getAs[String]("authToken")) 19 | 20 | InstagramUtils.createTagStream(ssc, auth, params.getAs[String]("tag")) 21 | } 22 | } -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamfactories/ParameterExtensions.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamfactories 2 | 3 | object ParameterExtensions { 4 | implicit class Parameters(val bag: Map[String, Any]) extends AnyVal { 5 | def getAs[T](key: String): T = bag(key).asInstanceOf[T] 6 | def getTrustedSources: Seq[String] = bag("trustedSources").asInstanceOf[Seq[String]] 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamfactories/RSSStreamFactory.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamfactories 2 | 3 | import com.github.catalystcode.fortis.spark.streaming.rss._ 4 | import com.microsoft.partnercatalyst.fortis.spark.sources.streamprovider.ConnectorConfig 5 | import org.apache.spark.storage.StorageLevel 6 | import org.apache.spark.streaming.StreamingContext 7 | import org.apache.spark.streaming.dstream.DStream 8 | 9 | class RSSStreamFactory extends StreamFactoryBase[RSSEntry] { 10 | 11 | override protected def canHandle(connectorConfig: ConnectorConfig): Boolean = { 12 | "RSS".equalsIgnoreCase(connectorConfig.name) 13 | } 14 | 15 | override protected def buildStream(ssc: StreamingContext, connectorConfig: ConnectorConfig): DStream[RSSEntry] = { 16 | import ParameterExtensions._ 17 | 18 | val params = connectorConfig.parameters 19 | new RSSInputDStream( 20 | params.getTrustedSources, 21 | storageLevel = StorageLevel.MEMORY_ONLY, 22 | requestHeaders = Map( 23 | "User-Agent" -> params.getOrElse("userAgent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36").toString 24 | ), 25 | connectTimeout = params.getOrElse("connectTimeout", "3000").toString.toInt, 26 | readTimeout = params.getOrElse("readTimeout", "9000").toString.toInt, 27 | pollingPeriodInSeconds = params.getOrElse("pollingPeriodInSeconds", "3600").toString.toInt, 28 | ssc = ssc 29 | ) 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamfactories/RadioStreamFactory.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamfactories 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.sources.streamprovider.ConnectorConfig 4 | import com.microsoft.partnercatalyst.fortis.spark.sources.streamwrappers.radio.{RadioStreamUtils, RadioTranscription} 5 | import org.apache.spark.streaming.StreamingContext 6 | import org.apache.spark.streaming.dstream.DStream 7 | 8 | class RadioStreamFactory extends StreamFactoryBase[RadioTranscription]{ 9 | override protected def canHandle(connectorConfig: ConnectorConfig): Boolean = { 10 | "Radio".equalsIgnoreCase(connectorConfig.name) 11 | } 12 | 13 | override protected def buildStream(ssc: StreamingContext, connectorConfig: ConnectorConfig): DStream[RadioTranscription] = { 14 | import ParameterExtensions._ 15 | 16 | val params = connectorConfig.parameters 17 | 18 | RadioStreamUtils.createStream(ssc, 19 | params.getAs[String]("radioUrl"), 20 | params.getAs[String]("audioType"), 21 | params.getAs[String]("locale"), 22 | params.getAs[String]("subscriptionKey"), 23 | params.getAs[String]("speechType"), 24 | params.getAs[String]("outputFormat") 25 | ) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamfactories/RedditStreamFactory.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamfactories 2 | 3 | import com.github.catalystcode.fortis.spark.streaming.reddit.dto.RedditObject 4 | import com.github.catalystcode.fortis.spark.streaming.reddit.{RedditAuth, RedditUtils} 5 | import com.microsoft.partnercatalyst.fortis.spark.sources.streamprovider.ConnectorConfig 6 | import org.apache.spark.streaming.StreamingContext 7 | import org.apache.spark.streaming.dstream.DStream 8 | 9 | class RedditStreamFactory extends StreamFactoryBase[RedditObject] { 10 | override protected def canHandle(connectorConfig: ConnectorConfig): Boolean = { 11 | "RedditObject".equalsIgnoreCase(connectorConfig.name) 12 | } 13 | 14 | override protected def buildStream(ssc: StreamingContext, connectorConfig: ConnectorConfig): DStream[RedditObject] = { 15 | import ParameterExtensions._ 16 | 17 | val params = connectorConfig.parameters 18 | val auth = RedditAuth(params.getAs[String]("applicationId"), params.getAs[String]("applicationSecret")) 19 | val keywords = params.getAs[String]("keywords").split('|') 20 | val subreddit = params.get("subreddit").asInstanceOf[Option[String]] 21 | val searchLimit = params.getOrElse("searchLimit", "25").asInstanceOf[String].toInt 22 | val searchResultType = Some(params.getOrElse("searchResultType", "link").asInstanceOf[String]) 23 | 24 | RedditUtils.createPageStream( 25 | auth, 26 | keywords.toSeq, 27 | ssc, 28 | subredit = subreddit, 29 | searchLimit = searchLimit, 30 | searchResultType = searchResultType 31 | ) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamprovider/ConnectorConfig.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamprovider 2 | 3 | case class ConnectorConfig(name: String, parameters: Map[String, Any]) -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamprovider/StreamFactory.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamprovider 2 | 3 | import org.apache.spark.streaming.StreamingContext 4 | import org.apache.spark.streaming.dstream.DStream 5 | 6 | /** 7 | * Provides an interface for concrete stream factory definitions, which encapsulate the logic of creating a DStream 8 | * backed by a specific type of connector (i.e. Kafka, EventHub, Instagram), given a configuration bundle (config). 9 | * @tparam A The element type of the streams produced by this factory. 10 | */ 11 | trait StreamFactory[A] { 12 | /** 13 | * Creates a DStream for a given connector config iff the connector config is supported by the stream factory. 14 | * The param set allows the streaming context to be curried into the partial function which creates the stream. 15 | * @param streamingContext The Spark Streaming Context 16 | * @return A partial function for transforming a connector config 17 | */ 18 | def createStream(streamingContext: StreamingContext): PartialFunction[ConnectorConfig, DStream[A]] 19 | } -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamprovider/StreamProviderException.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamprovider 2 | 3 | sealed trait StreamProviderException { self: Throwable => 4 | // TODO 5 | } 6 | case class InvalidConnectorConfigException() extends Exception("Invalid connector config.") with StreamProviderException 7 | case class UnsupportedConnectorConfigException() extends Exception("Unsupported connector config.") with StreamProviderException -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamwrappers/customevents/CustomEvent.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamwrappers.customevents 2 | 3 | case class CustomEventFeature( 4 | `type`: String, 5 | coordinates: List[Float]) 6 | 7 | case class CustomEventFeatureCollection( 8 | `type`: String, 9 | features: List[CustomEventFeature]) 10 | 11 | case class CustomEvent( 12 | RowKey: String, 13 | created_at: String, 14 | featureCollection: CustomEventFeatureCollection, 15 | message: String, 16 | language: String, 17 | link: Option[String], 18 | source: Option[String], 19 | title: Option[String]) 20 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamwrappers/customevents/CustomEventsAdapter.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamwrappers.customevents 2 | 3 | import net.liftweb.json 4 | 5 | import scala.util.Try 6 | 7 | object CustomEventsAdapter { 8 | def apply(input: String): Try[CustomEvent] = { 9 | implicit val _ = json.DefaultFormats 10 | Try(json.parse(input)).flatMap(body => Try(body.extract[CustomEvent])) 11 | } 12 | } -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamwrappers/radio/RadioInputDStream.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamwrappers.radio 2 | 3 | import org.apache.spark.storage.StorageLevel 4 | import org.apache.spark.streaming.StreamingContext 5 | import org.apache.spark.streaming.dstream.ReceiverInputDStream 6 | import org.apache.spark.streaming.receiver.Receiver 7 | 8 | class RadioInputDStream( 9 | ssc: StreamingContext, 10 | radioUrl: String, 11 | audioType: String, 12 | locale: String, 13 | subscriptionKey: String, 14 | speechType: String, 15 | outputFormat: String, 16 | storageLevel: StorageLevel 17 | ) extends ReceiverInputDStream[RadioTranscription](ssc) { 18 | override def getReceiver(): Receiver[RadioTranscription] = { 19 | logDebug("Creating radio transcription receiver") 20 | new TranscriptionReceiver(radioUrl, audioType, locale, subscriptionKey, speechType, outputFormat, storageLevel) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamwrappers/radio/RadioStreamUtils.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamwrappers.radio 2 | 3 | import org.apache.spark.storage.StorageLevel 4 | import org.apache.spark.streaming.StreamingContext 5 | import org.apache.spark.streaming.dstream.DStream 6 | 7 | object RadioStreamUtils { 8 | def createStream( 9 | ssc: StreamingContext, 10 | radioUrl: String, 11 | audioType: String, 12 | locale: String, 13 | subscriptionKey: String, 14 | speechType: String, 15 | outputFormat: String, 16 | storageLevel: StorageLevel = StorageLevel.MEMORY_ONLY 17 | ): DStream[RadioTranscription] = { 18 | new RadioInputDStream(ssc, radioUrl, audioType, locale, subscriptionKey, speechType, outputFormat, storageLevel) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamwrappers/radio/RadioTranscription.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamwrappers.radio 2 | 3 | case class RadioTranscription( 4 | text: String, 5 | language: String, 6 | radioUrl: String) 7 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamwrappers/radio/TranscriptionReceiver.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamwrappers.radio 2 | 3 | import java.io.InputStream 4 | import java.net.URL 5 | import java.util.Locale 6 | import java.util.function.Consumer 7 | 8 | import com.github.catalystcode.fortis.speechtotext.Transcriber 9 | import com.github.catalystcode.fortis.speechtotext.config.{OutputFormat, SpeechServiceConfig, SpeechType} 10 | import org.apache.spark.storage.StorageLevel 11 | import org.apache.spark.streaming.receiver.Receiver 12 | 13 | class TranscriptionReceiver( 14 | radioUrl: String, 15 | audioType: String, 16 | locale: String, 17 | subscriptionKey: String, 18 | speechType: String, 19 | outputFormat: String, 20 | storageLevel: StorageLevel 21 | ) extends Receiver[RadioTranscription](storageLevel) { 22 | 23 | private val language = new Locale(locale).getLanguage 24 | private var audioStream: InputStream = _ 25 | private var transcriber: Transcriber = _ 26 | 27 | private val onTranscription = new Consumer[String] { 28 | override def accept(text: String): Unit = { 29 | val transcription = RadioTranscription(text = text, language = language, radioUrl = radioUrl) 30 | store(transcription) 31 | } 32 | } 33 | 34 | private val onHypothesis = new Consumer[String] { 35 | override def accept(hypothesis: String): Unit = { 36 | // do nothing 37 | } 38 | } 39 | 40 | override def onStart(): Unit = { 41 | val config = new SpeechServiceConfig( 42 | subscriptionKey, 43 | SpeechType.valueOf(speechType), 44 | OutputFormat.valueOf(outputFormat), 45 | new Locale(locale)) 46 | 47 | transcriber = Transcriber.create(audioType, config) 48 | audioStream = new URL(radioUrl).openConnection.getInputStream 49 | transcriber.transcribe(audioStream, onTranscription, onHypothesis) 50 | } 51 | 52 | override def onStop(): Unit = { 53 | if (audioStream != null) audioStream.close() 54 | if (transcriber != null) transcriber = null 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamwrappers/tadaweb/TadawebAdapter.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamwrappers.tadaweb 2 | 3 | import net.liftweb.json 4 | 5 | import scala.util.Try 6 | 7 | object TadawebAdapter { 8 | def apply(input: String): Try[TadawebEvent] = { 9 | implicit val _ = json.DefaultFormats 10 | Try(json.parse(input)).flatMap(body => Try(body.extract[TadawebEvent])) 11 | } 12 | } -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/sources/streamwrappers/tadaweb/TadawebEvent.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sources.streamwrappers.tadaweb 2 | 3 | case class TadawebEvent( 4 | language: String, 5 | text: String, 6 | cities: Seq[TadawebCity], 7 | sentiment: String, 8 | tada: TadawebTada, 9 | tags: Seq[String], 10 | title: String, 11 | link: String, 12 | published_at: String 13 | ) 14 | 15 | case class TadawebCity( 16 | city: String, 17 | coordinates: Seq[Double] 18 | ) 19 | 20 | case class TadawebTada( 21 | description: String, 22 | id: String, 23 | name: String 24 | ) -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transformcontext/TransformContext.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transformcontext 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.dto.{BlacklistedItem, SiteSettings} 4 | import com.microsoft.partnercatalyst.fortis.spark.transforms.image.ImageAnalyzer 5 | import com.microsoft.partnercatalyst.fortis.spark.transforms.language.LanguageDetector 6 | import com.microsoft.partnercatalyst.fortis.spark.transforms.locations.LocationsExtractorFactory 7 | import com.microsoft.partnercatalyst.fortis.spark.transforms.sentiment.SentimentDetectorAuth 8 | import com.microsoft.partnercatalyst.fortis.spark.transforms.topic.KeywordExtractor 9 | import org.apache.spark.broadcast.Broadcast 10 | 11 | case class TransformContext( 12 | siteSettings: SiteSettings = null, 13 | langToKeywordExtractor: Broadcast[Map[String, KeywordExtractor]] = null, 14 | blacklist: Broadcast[Seq[BlacklistedItem]] = null, 15 | locationsExtractorFactory: Broadcast[LocationsExtractorFactory] = null, 16 | 17 | // The following objects have a small serialized forms. Consequently, we don't bother to broadcast them 18 | // (instead, they're serialized into each task that uses them). 19 | imageAnalyzer: ImageAnalyzer = null, 20 | languageDetector: LanguageDetector = null, 21 | sentimentDetectorAuth: SentimentDetectorAuth = null 22 | ) -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transformcontext/TransformContextMessages.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transformcontext 2 | 3 | /** @note These have to be kept in sync with StreamingController.js in project-fortis-services **/ 4 | object TransformContextMessages { 5 | val ChangeRequired = "dirty" 6 | 7 | val SettingsChanged = "sitesettings" 8 | val WatchlistChanged = "watchlist" 9 | val BlacklistChanged = "blacklist" 10 | } 11 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/gender/GenderDetector.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.gender 2 | 3 | object GenderDetector extends Enumeration { 4 | val Male = "M" 5 | val Female = "F" 6 | } 7 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/image/dto/Json.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.image.dto 2 | 3 | case class JsonImageAnalysisResponse(categories: List[JsonImageCategory], tags: List[JsonImageTag], description: JsonImageDescription, faces: List[JsonImageFace]) 4 | case class JsonImageDescription(tags: List[String], captions: List[JsonImageCaption]) 5 | case class JsonImageFace(age: Double, gender: String, faceRectangle: JsonFaceRectangle) 6 | case class JsonFaceRectangle(left: Int, top: Int, width: Int, height: Int) 7 | case class JsonImageCaption(text: String, confidence: Double) 8 | case class JsonImageTag(name: String, confidence: Double) 9 | case class JsonImageCategory(name: String, score: Double, detail: Option[JsonImageCategoryDetail]) 10 | case class JsonImageCategoryDetail(celebrities: Option[List[JsonImageCelebrity]], landmarks: Option[List[JsonImageLandmark]]) 11 | case class JsonImageCelebrity(name: String, confidence: Double, faceRectangle: JsonFaceRectangle) 12 | case class JsonImageLandmark(name: String, confidence: Double) 13 | 14 | case class JsonImageAnalysisRequest(url: String) 15 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/language/LanguageDetector.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.language 2 | 3 | trait LanguageDetector extends Serializable { 4 | def detectLanguage(text: String): Option[String] 5 | } 6 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/language/LocalLanguageDetector.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.language 2 | 3 | import java.lang.System.currentTimeMillis 4 | 5 | import com.microsoft.partnercatalyst.fortis.spark.logging.FortisTelemetry 6 | import com.optimaize.langdetect.LanguageDetectorBuilder 7 | import com.optimaize.langdetect.ngram.NgramExtractors 8 | import com.optimaize.langdetect.profiles.LanguageProfileReader 9 | import com.optimaize.langdetect.text.{CommonTextObjectFactories, TextObjectFactory} 10 | 11 | @SerialVersionUID(100L) 12 | class LocalLanguageDetector extends LanguageDetector { 13 | @transient private lazy val languageProfiles = new LanguageProfileReader().readAllBuiltIn 14 | @transient private lazy val languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard()).withProfiles(languageProfiles).build() 15 | @transient private lazy val largeTextFactory = CommonTextObjectFactories.forDetectingOnLargeText() 16 | @transient private lazy val shortTextFactory = CommonTextObjectFactories.forDetectingShortCleanText() 17 | 18 | override def detectLanguage(text: String): Option[String] = { 19 | if (text.isEmpty) { 20 | return None 21 | } 22 | 23 | val startTime = currentTimeMillis() 24 | val language = detectWithFactory(text, if (text.length <= 200) shortTextFactory else largeTextFactory) 25 | val endTime = currentTimeMillis() 26 | 27 | FortisTelemetry.get.logDependency("transforms.localLanguageDetector", "detectLanguage", language.isDefined, endTime - startTime) 28 | language 29 | } 30 | 31 | private def detectWithFactory(text: String, factory: TextObjectFactory): Option[String] = { 32 | Option(languageDetector.detect(factory.forText(text)).orNull).map(_.getLanguage) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/language/TextNormalizer.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.language 2 | 3 | import java.util.Locale 4 | 5 | /** 6 | * Because misspellings are common for some stream types, this trait allows for correction/stemming in order to improve 7 | * lookup or processing of some text [fragments]. Think of this as a simpler version of a java.text.Collator. 8 | */ 9 | trait TextNormalizer { 10 | 11 | def normalizeText(text: String): String 12 | 13 | } 14 | 15 | object TextNormalizer { 16 | 17 | private val normalizersByLocale: Map[String, TextNormalizer] = Map( 18 | "es" -> SpanishNormalizer() 19 | ) 20 | 21 | private val defaultNormalizer = DefaultNormalizer() 22 | 23 | def apply(text: String, locale: String): String = { 24 | normalizersByLocale.getOrElse(locale, defaultNormalizer).normalizeText(text) 25 | } 26 | 27 | } 28 | 29 | case class DefaultNormalizer() extends TextNormalizer { 30 | override def normalizeText(text: String): String = text 31 | } 32 | 33 | /** 34 | * It is quite common, in informal writing, for people to leave out accent marks in Spanish words. So this normalizer 35 | * strips out diacritics and changes the incoming text to lower case. 36 | */ 37 | case class SpanishNormalizer() extends TextNormalizer { 38 | val locale = Locale.forLanguageTag("es") 39 | override def normalizeText(text: String): String = { 40 | org.apache.commons.lang3.StringUtils.stripAccents(text.toLowerCase(locale)) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/language/dto/Json.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.language.dto 2 | 3 | case class JsonLanguageDetectionResponse(documents: List[JsonLanguageDetectionResponseItem], errors: List[JsonLanguageDetectionResponseError] = List()) 4 | case class JsonLanguageDetectionResponseItem(id: String, detectedLanguages: List[JsonLanguageDetectionLanguage]) 5 | case class JsonLanguageDetectionLanguage(name: String, iso6391Name: String, score: Double) 6 | case class JsonLanguageDetectionResponseError(id: String, message: String) 7 | 8 | case class JsonLanguageDetectionRequest(documents: List[JsonLanguageDetectionRequestItem]) 9 | case class JsonLanguageDetectionRequestItem(id: String, text: String) 10 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/locations/LocationsExtractor.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.locations 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.dto.Location 4 | import com.microsoft.partnercatalyst.fortis.spark.logging.FortisTelemetry.{get => Log} 5 | import com.microsoft.partnercatalyst.fortis.spark.transforms.locations.client.FeatureServiceClient 6 | 7 | @SerialVersionUID(100L) 8 | class LocationsExtractor private[locations]( 9 | lookup: Map[String, Set[Location]], 10 | featureServiceClient: FeatureServiceClient, 11 | placeRecognizer: Option[PlaceRecognizer] = None, 12 | locationLimit: Int = Int.MaxValue, 13 | ngrams: Int = 3 14 | ) extends Serializable { 15 | 16 | def analyze(text: String): Iterable[Location] = { 17 | if (text.isEmpty) { 18 | return List() 19 | } 20 | 21 | val candidatePlaces = extractCandidatePlaces(text) 22 | val locationSetsInGeofence = candidatePlaces.flatMap(place => 23 | lookup.get(place._1.toLowerCase).map((_, place._2)) 24 | ) 25 | 26 | val locations = locationSetsInGeofence.flatMap(location => location._1.map((_, location._2))) 27 | val topLocations = locations.sortBy(_._2)(Ordering[Int].reverse).take(locationLimit) 28 | 29 | topLocations.map(_._1.copy(confidence = Some(0.5))) 30 | } 31 | 32 | private def extractCandidatePlaces(text: String): Seq[(String, Int)] = { 33 | // TODO: use max heap 34 | var candidatePlaces = Seq[(String, Int)]() 35 | 36 | if (placeRecognizer.isDefined) { 37 | candidatePlaces = placeRecognizer.get.extractPlacesAndOccurrence(text) 38 | } 39 | 40 | // TODO: ngrams will be very expensive on large text. Limit text or use summary only? 41 | if (candidatePlaces.isEmpty && (placeRecognizer.isEmpty || !placeRecognizer.get.isValid)) { 42 | Log.logDebug("Falling back to ngrams approach") 43 | candidatePlaces = StringUtils.ngrams(text, ngrams).groupBy(str => str).mapValues(_.length).toSeq 44 | } 45 | 46 | candidatePlaces 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/locations/LuceneLocationsExtractor.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.locations 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.dto.Location 4 | import com.microsoft.partnercatalyst.fortis.spark.transforms.locations.client.FeatureServiceClient 5 | import com.microsoft.partnercatalyst.fortis.spark.transforms.topic.LuceneKeyphraseExtractor 6 | 7 | class LuceneLocationsExtractor( 8 | lookup: Map[String, Set[Location]], 9 | featureServiceClient: FeatureServiceClient, 10 | locationLimit: Int = Int.MaxValue, 11 | ngrams: Int = 3 12 | ) extends LocationsExtractor(lookup, featureServiceClient, None, locationLimit, ngrams) { 13 | 14 | lazy private val keyphraseExtractor = new LuceneKeyphraseExtractor("UNKNOWN", lookup.keySet, locationLimit) 15 | 16 | override def analyze(text: String): Iterable[Location] = { 17 | if (text.isEmpty) { 18 | return List() 19 | } 20 | val matches = keyphraseExtractor.extractKeyphrases(text) 21 | matches.flatMap(m=>lookup(m.matchedPhrase)) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/locations/PlaceRecognizer.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.locations 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.transforms.ZipModelsProvider 4 | import com.microsoft.partnercatalyst.fortis.spark.transforms.entities.EntityRecognizer 5 | import com.microsoft.partnercatalyst.fortis.spark.transforms.language.TextNormalizer 6 | 7 | @SerialVersionUID(100L) 8 | class PlaceRecognizer( 9 | modelsProvider: ZipModelsProvider, 10 | language: Option[String] 11 | ) extends Serializable { 12 | 13 | @volatile private lazy val entityRecognizer = createEntityRecognizer() 14 | 15 | def extractPlacesAndOccurrence(text: String): Seq[(String, Int)] = { 16 | // See: https://github.com/opener-project/kaf/wiki/KAF-structure-overview 17 | entityRecognizer.extractTerms(TextNormalizer(text, language.getOrElse(""))) 18 | .filter(term => { 19 | val partOfSpeech = term.getPos 20 | "N".equals(partOfSpeech) || "R".equals(partOfSpeech) 21 | }) 22 | .groupBy(_.getStr) 23 | .map(place => (place._1, place._2.size)).toSeq 24 | } 25 | 26 | def isValid: Boolean = entityRecognizer.isValid 27 | 28 | protected def createEntityRecognizer(): EntityRecognizer = { 29 | new EntityRecognizer(modelsProvider, language) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/locations/StringUtils.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.locations 2 | 3 | object StringUtils { 4 | def ngrams(text: String, n: Int, sep: String = " "): Seq[String] = { 5 | val words = text.replaceAll("\\p{P}", sep).split(sep).filter(x => !x.isEmpty) 6 | val ngrams = Math.min(n, words.length) 7 | (1 to ngrams).flatMap(i => words.sliding(i).map(_.mkString(sep))) 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/locations/dto/Json.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.locations.dto 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.dto.Location 4 | 5 | case class FeatureServiceResponse(features: List[FeatureServiceFeature]) 6 | case class FeatureServiceFeature(id: String, name: String, layer: String, centroid: Option[List[Double]] = None) 7 | 8 | object FeatureServiceFeature { 9 | val DefaultLatitude = -1d 10 | val DefaultLongitude = -1d 11 | 12 | def toLocation(feature: FeatureServiceFeature): Location = { 13 | Location( 14 | wofId = feature.id, 15 | name = feature.name, 16 | layer = feature.layer, 17 | longitude = feature.centroid.map(_.head).getOrElse(DefaultLongitude), 18 | latitude = feature.centroid.map(_.tail.head).getOrElse(DefaultLatitude)) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/nlp/Tokenizer.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.nlp 2 | 3 | object Tokenizer { 4 | @transient private lazy val wordTokenizer = """\b""".r 5 | 6 | def apply(sentence: String): Seq[String] = { 7 | if (sentence.isEmpty) { 8 | return Seq() 9 | } 10 | 11 | wordTokenizer.split(sentence).toSeq 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/people/PeopleRecognizer.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.people 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.transforms.ZipModelsProvider 4 | import com.microsoft.partnercatalyst.fortis.spark.transforms.entities.EntityRecognizer 5 | import com.microsoft.partnercatalyst.fortis.spark.transforms.nlp.OpeNER.entityIsPerson 6 | 7 | @SerialVersionUID(100L) 8 | class PeopleRecognizer( 9 | modelsProvider: ZipModelsProvider, 10 | language: Option[String] 11 | ) extends Serializable { 12 | 13 | @volatile private lazy val entityRecognizer = createEntityRecognizer() 14 | 15 | def extractPeople(text: String): List[String] = { 16 | entityRecognizer.extractEntities(text).filter(entityIsPerson).map(_.getStr) 17 | } 18 | 19 | protected def createEntityRecognizer(): EntityRecognizer = { 20 | new EntityRecognizer(modelsProvider, language) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/sentiment/SentimentDetector.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.sentiment 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.logging.FortisTelemetry.{get => Log} 4 | import com.microsoft.partnercatalyst.fortis.spark.transforms.ZipModelsProvider 5 | 6 | import scala.util.{Failure, Success, Try} 7 | 8 | @SerialVersionUID(100L) 9 | class SentimentDetector( 10 | modelsProvider: ZipModelsProvider, 11 | language: Option[String], 12 | auth: SentimentDetectorAuth 13 | 14 | ) extends DetectsSentiment { 15 | 16 | private lazy val detectors = language.map(_ => initializeDetectors()) 17 | 18 | def detectSentiment(text: String): Option[Double] = { 19 | if (detectors.isEmpty) { 20 | return None 21 | } 22 | 23 | detectors.get.view.map(detector => { 24 | Try(detector.detectSentiment(text)) match { 25 | case Success(Some(sentimentScore)) => 26 | Log.logDebug(s"Computed sentiment via ${detector.getClass}") 27 | Some(sentimentScore) 28 | case Success(None) | Failure(_) => 29 | Log.logDebug(s"Unable to compute sentiment via ${detector.getClass}") 30 | None 31 | } 32 | }) 33 | .find(_.isDefined) 34 | .getOrElse(None) 35 | } 36 | 37 | protected def initializeDetectors(): Seq[DetectsSentiment] = { 38 | Seq(new CognitiveServicesSentimentDetector(language.get, auth), 39 | new WordListSentimentDetector(modelsProvider, language.get)) 40 | } 41 | } 42 | 43 | object SentimentDetector extends Enumeration { 44 | val Positive: Double = 1.0 45 | val Neutral: Double = 0.6 46 | val Negative: Double = 0.0 47 | } 48 | 49 | trait DetectsSentiment extends Serializable { 50 | def detectSentiment(text: String): Option[Double] 51 | } 52 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/sentiment/dto/Json.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.sentiment.dto 2 | 3 | case class JsonSentimentDetectionResponse(documents: List[JsonSentimentDetectionResponseItem], errors: List[JsonSentimentDetectionResponseError] = List()) 4 | case class JsonSentimentDetectionResponseItem(id: String, score: Double) 5 | case class JsonSentimentDetectionResponseError(id: String, message: String) 6 | 7 | case class JsonSentimentDetectionRequest(documents: List[JsonSentimentDetectionRequestItem]) 8 | case class JsonSentimentDetectionRequestItem(id: String, text: String, language: String) 9 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/topic/Blacklist.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.topic 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.dto.BlacklistedItem 4 | import com.microsoft.partnercatalyst.fortis.spark.transforms.nlp.Tokenizer 5 | 6 | @SerialVersionUID(100L) 7 | class Blacklist(blacklist: Seq[BlacklistedItem]) extends Serializable { 8 | def matches(text: String): Boolean = { 9 | if (text.isEmpty) { 10 | return false 11 | } 12 | 13 | val tokens = Tokenizer(text).toSet 14 | blacklist 15 | .filter(!_.isLocation) 16 | .exists(entry => entry.conjunctiveFilter.forall(tokens.contains)) 17 | } 18 | 19 | def matches(terms: Set[String]): Boolean = { 20 | blacklist 21 | .filter(!_.isLocation) 22 | .exists(entry => entry.conjunctiveFilter.forall(terms.contains)) 23 | } 24 | 25 | def matchesLocation(locations: Set[String]): Boolean = { 26 | // TODO: log if conjunctive filter has > 1 term and is of type location 27 | blacklist 28 | .filter(_.isLocation) 29 | .exists(entry => entry.conjunctiveFilter.forall(locations.contains)) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/topic/KeyphraseExtractor.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.topic 2 | 3 | trait KeyphraseExtractor extends Serializable { 4 | 5 | def extractKeyphrases(input: String): Set[Keyphrase] 6 | 7 | } 8 | 9 | case class Keyphrase(matchedPhrase: String, fragments:Set[String], count: Int) 10 | -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/topic/KeywordExtractor.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.topic 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.dto.Tag 4 | import com.microsoft.partnercatalyst.fortis.spark.transforms.language.TextNormalizer 5 | import com.microsoft.partnercatalyst.fortis.spark.transforms.nlp.Tokenizer 6 | import org.apache.commons.collections4.trie.PatriciaTrie 7 | 8 | import scala.collection.mutable.ListBuffer 9 | 10 | @SerialVersionUID(100L) 11 | class KeywordExtractor(language: String, keywords: Iterable[String], maxKeywords: Int = Int.MaxValue) extends Serializable { 12 | @transient private lazy val keywordTrie = initializeTrie(keywords) 13 | 14 | def extractKeywords(text: String): List[Tag] = { 15 | if (text.isEmpty) { 16 | return List() 17 | } 18 | 19 | def findMatches(segment: Seq[String]): Iterable[String] = { 20 | val sb = new StringBuilder() 21 | val result = ListBuffer[String]() 22 | 23 | val it = segment.iterator 24 | var prefix = "" 25 | while (it.hasNext && !keywordTrie.prefixMap(prefix).isEmpty) { 26 | prefix = sb.append(it.next()).mkString 27 | 28 | Option(keywordTrie.get(prefix)).foreach(result.append(_)) 29 | } 30 | 31 | result 32 | } 33 | 34 | val tokens = Tokenizer(TextNormalizer(text.toLowerCase, language)) 35 | val occurances = tokens.tails.flatMap(findMatches(_).map(Tag(_, confidence = None))).toIterable.groupBy(_.name.toLowerCase) 36 | occurances.toSeq.sortBy(_._2.size)(Ordering[Int].reverse).take(maxKeywords).map(_._2.head).toList 37 | } 38 | 39 | private def initializeTrie(keywords: Iterable[String]): PatriciaTrie[String] = { 40 | val trie = new PatriciaTrie[String]() 41 | keywords.foreach(k => { 42 | trie.put(k.toLowerCase, k) 43 | trie.put(TextNormalizer(k.toLowerCase, language), k) 44 | }) 45 | 46 | trie 47 | } 48 | } -------------------------------------------------------------------------------- /project-fortis-spark/src/main/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/topic/LuceneKeywordExtractor.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.topic 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.dto.Tag 4 | 5 | class LuceneKeywordExtractor(language: String, keywords: Iterable[String], maxKeywords: Int = Int.MaxValue) extends KeywordExtractor(language, keywords, maxKeywords) { 6 | 7 | lazy private val luceneKeyphraseExtractor = new LuceneKeyphraseExtractor(language, keywords.toSet, maxKeywords) 8 | 9 | override def extractKeywords(text: String): List[Tag] = { 10 | luceneKeyphraseExtractor 11 | .extractKeyphrases(text) 12 | .map(kp=>Tag(kp.matchedPhrase, Some(1))) 13 | .toList 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /project-fortis-spark/src/test/scala/com/microsoft/partnercatalyst/fortis/spark/dto/FortisEventSpec.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.dto 2 | 3 | import org.scalatest.FlatSpec 4 | 5 | class FortisEventSpec extends FlatSpec { 6 | "The Fortis event" should "have an ordering defined by layer" in { 7 | val country1 = Location(wofId = "id1", name = "country1", latitude = -1, longitude = -1, layer = "country") 8 | val country2 = Location(wofId = "id3", name = "country2", latitude = -1, longitude = -1, layer = "country") 9 | val neighbourhood = Location(wofId = "id2", name = "neighbourhood", latitude = -1, longitude = -1, layer = "neighbourhood") 10 | 11 | assert(country1 > neighbourhood) 12 | assert(country1.compare(country2) == 0) 13 | assert(neighbourhood < country2) 14 | } 15 | 16 | it should "have the ordering handle null and unknown values" in { 17 | val country = Location(wofId = "id1", name = "coutry", latitude = -1, longitude = -1, layer = "country") 18 | val nullLayer = Location(wofId = "id2", name = "null island", latitude = -1, longitude = -1, layer = null) 19 | val unknownLayer = Location(wofId = "id3", name = "unknown city", latitude = -1, longitude = -1, layer = "unknown layer type") 20 | 21 | assert(country < nullLayer) 22 | assert(country < unknownLayer) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /project-fortis-spark/src/test/scala/com/microsoft/partnercatalyst/fortis/spark/sinks/cassandra/CassandraConjunctiveTopicsTestSpec.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sinks.cassandra 2 | 3 | import java.util.{Date, UUID} 4 | 5 | import com.microsoft.partnercatalyst.fortis.spark.sinks.cassandra.dto._ 6 | import org.scalatest.FlatSpec 7 | 8 | class CassandraConjunctiveTopicsTestSpec extends FlatSpec { 9 | it should "flat map keywords" in { 10 | val event = Event( 11 | pipelinekey = "Twitter", 12 | computedfeatures_json = Features.asJson(Features( 13 | mentions = 1, 14 | sentiment = Sentiment(1.0), 15 | keywords = Seq("europe", "humanitarian"), 16 | places = Seq(Place("abc123", 10.0, 20.0)), 17 | entities = Seq() 18 | )), 19 | eventtime = Period("day-2017-08-11").startTime(), 20 | eventlangcode = "en", 21 | eventid = UUID.randomUUID().toString, 22 | sourceeventid = UUID.randomUUID().toString, 23 | insertiontime = new Date().getTime, 24 | body = "", 25 | imageurl = None, 26 | summary = "", 27 | batchid = UUID.randomUUID().toString, 28 | externalsourceid = "HamillHimself", 29 | sourceurl = "", 30 | title = "" 31 | ) 32 | 33 | val topics = CassandraConjunctiveTopics.flatMapKeywords(event) 34 | 35 | assert(topics == Seq( 36 | ("europe", ""), 37 | ("humanitarian", ""), 38 | ("europe", "humanitarian"), 39 | ("humanitarian", "europe") 40 | )) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /project-fortis-spark/src/test/scala/com/microsoft/partnercatalyst/fortis/spark/sinks/cassandra/CassandraIntegrationTestSpec.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.sinks.cassandra 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.sinks.cassandra._ 4 | import org.scalatest.FlatSpec 5 | 6 | class CassandraIntegrationTestSpec extends FlatSpec { 7 | it should "verify that we can produce conjunctive topic tuples from a list of topics" in { 8 | val conjunctiveTopics = Utils.getConjunctiveTopics(Option(Seq("sam", "erik", "tom"))).sorted 9 | val expectedTopics = Seq( 10 | ("erik","",""), 11 | ("erik","sam",""), 12 | ("erik","sam","tom"), 13 | ("erik","tom",""), 14 | ("sam","",""), 15 | ("sam","tom",""), 16 | ("tom","","") 17 | ) 18 | assert(conjunctiveTopics.length === 7) 19 | assert(conjunctiveTopics === expectedTopics) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /project-fortis-spark/src/test/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/language/CognitiveServicesLanguageDetectorSpec.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.language 2 | 3 | import org.scalatest.FlatSpec 4 | 5 | class TestCognitiveServicesLanguageDetector(cognitiveServicesResponse: String) extends CognitiveServicesLanguageDetector(CognitiveServicesLanguageDetectorAuth("key", "host")) { 6 | protected override def callCognitiveServices(request: String): String = cognitiveServicesResponse 7 | override def buildRequestBody(text: String, textId: String): String = super.buildRequestBody(text, textId) 8 | } 9 | 10 | class CognitiveServicesLanguageDetectorSpec extends FlatSpec { 11 | "The language detector" should "formulate correct request and parse response to domain types" in { 12 | val responseConfidence = 1.0 13 | val detector = new TestCognitiveServicesLanguageDetector(s"""{"documents":[{"id":"0","detectedLanguages":[{"name":"English","iso6391Name":"en","score":$responseConfidence}]}],"errors":[]}""") 14 | val language = detector.detectLanguage("some text") 15 | 16 | assert(language.contains("en")) 17 | } 18 | 19 | it should "ignore low confidence detections" in { 20 | val responseConfidence = 0.01 21 | val detector = new TestCognitiveServicesLanguageDetector(s"""{"documents":[{"id":"0","detectedLanguages":[{"name":"English","iso6391Name":"en","score":$responseConfidence}]}],"errors":[]}""") 22 | val language = detector.detectLanguage("some text") 23 | 24 | assert(language.isEmpty) 25 | } 26 | 27 | it should "build correct request body" in { 28 | val id = "0" 29 | val text = "some text" 30 | val requestBody = new TestCognitiveServicesLanguageDetector("").buildRequestBody(text, id) 31 | 32 | assert(requestBody == s"""{"documents":[{"id":"$id","text":"$text"}]}""") 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /project-fortis-spark/src/test/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/language/LocalLanguageDetectorSpec.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.language 2 | 3 | import org.scalatest.FlatSpec 4 | 5 | class LocalLanguageDetectorSpec extends FlatSpec { 6 | "The language detector" should "detect English" in { 7 | val detector = new LocalLanguageDetector() 8 | assert(detector.detectLanguage("And I in going, madam, weep o'er my father's death anew: but I must attend his majesty's command, to whom I am now in ward, evermore in subjection.").contains("en")) 9 | } 10 | 11 | it should "detect French" in { 12 | val detector = new LocalLanguageDetector() 13 | assert(detector.detectLanguage("Je l’avouerai franchement à mes lecteurs ; je n’étais jamais encore sorti de mon trou").contains("fr")) 14 | } 15 | 16 | it should "detect Spanish" in { 17 | val detector = new LocalLanguageDetector() 18 | assert(detector.detectLanguage("En un lugar de la Mancha, de cuyo nombre no quiero acordarme, no ha mucho tiempo que vivía un hidalgo de los de lanza en astillero").contains("es")) 19 | } 20 | 21 | it should "detect Chinese" in { 22 | val detector = new LocalLanguageDetector() 23 | assert(detector.detectLanguage("故经之以五事，校之以计，而索其情，一曰道，二曰天，三曰地，四曰将，五曰法。").contains("zh")) 24 | } 25 | 26 | it should "detect Urdu" in { 27 | val detector = new LocalLanguageDetector() 28 | assert(detector.detectLanguage("تازہ ترین خبروں، ویڈیوز اور آڈیوز کے لیے بی بی سی اردو پر آئیے۔ بی بی سی اردو دنیا بھر کی خبروں کے حصول کے لیے ایک قابلِ اعتماد ویب سائٹ ہے۔").contains("ur")) 29 | } 30 | 31 | it should "detect gibberish" in { 32 | val detector = new LocalLanguageDetector() 33 | assert(detector.detectLanguage("Heghlu'meH QaQ jajvam").isEmpty) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /project-fortis-spark/src/test/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/locations/PlaceRecognizerIntegrationSpec.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.locations 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.transforms.ZipModelsProvider 4 | 5 | import org.scalatest.FlatSpec 6 | 7 | class PlaceRecognizerIntegrationSpec extends FlatSpec { 8 | "The place recognizer" should "extract correct places" in { 9 | val modelsProvider = new ZipModelsProvider( 10 | language => s"https://fortiscentral.blob.core.windows.net/opener/opener-$language.zip") 11 | 12 | val testCases = List( 13 | ("I went to Paris last week. France was great!", "en", List(("France", 1), ("Paris", 1), ("week", 1))), 14 | ("A mi me piace Roma.", "it", List(("Roma", 1))), 15 | ("I love Rome.", "en", List(("Rome", 1))) 16 | ) 17 | 18 | testCases.foreach(test => { 19 | val recognizer = new PlaceRecognizer(modelsProvider, Some(test._2)) 20 | val places = recognizer.extractPlacesAndOccurrence(test._1) 21 | assert(places.toSet == test._3.toSet) 22 | }) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /project-fortis-spark/src/test/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/locations/StringUtilsSpec.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.locations 2 | 3 | import org.scalatest.FlatSpec 4 | 5 | class StringUtilsSpec extends FlatSpec { 6 | "The ngrams method" should "extract correct ngrams" in { 7 | assert(StringUtils.ngrams("the koala eats", n = 1) == List("the", "koala", "eats")) 8 | assert(StringUtils.ngrams("the koala eats", n = 2) == List("the", "koala", "eats", "the koala", "koala eats")) 9 | assert(StringUtils.ngrams("the koala eats", n = 3) == List("the", "koala", "eats", "the koala", "koala eats", "the koala eats")) 10 | assert(StringUtils.ngrams("the koala eats", n = 4) == List("the", "koala", "eats", "the koala", "koala eats", "the koala eats")) 11 | } 12 | 13 | it should "ignore extra whitespace" in { 14 | assert(StringUtils.ngrams("the koala eats ", n = 2) == List("the", "koala", "eats", "the koala", "koala eats")) 15 | } 16 | 17 | it should "ignore punctuation" in { 18 | assert(StringUtils.ngrams("the koala, eats!", n = 2) == List("the", "koala", "eats", "the koala", "koala eats")) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /project-fortis-spark/src/test/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/nlp/TokenizerSpec.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.nlp 2 | 3 | import org.scalatest.FlatSpec 4 | 5 | class TokenizerSpec extends FlatSpec { 6 | "The tokenizer" should "split sentences on spaces" in { 7 | assert(Tokenizer("foo bar baz") == Seq("foo", " ", "bar", " ", "baz")) 8 | } 9 | 10 | it should "handle non-space whitespace" in { 11 | assert(Tokenizer("\rfoo\tbar\nbaz") == Seq("\r", "foo", "\t", "bar", "\n", "baz")) 12 | } 13 | 14 | it should "handle non-standard whitespace" in { 15 | assert(Tokenizer("foo\u2000bar\u00a0baz") == Seq("foo", "\u2000", "bar", "\u00a0", "baz")) 16 | } 17 | 18 | it should "handle empty inputs" in { 19 | assert(Tokenizer("") == Seq()) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /project-fortis-spark/src/test/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/sentiment/CognitiveServicesSentimentDetectorSpec.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.sentiment 2 | 3 | import org.scalatest.FlatSpec 4 | 5 | class TestCognitiveServicesSentimentDetector(cognitiveServicesResponse: String, language: String) extends CognitiveServicesSentimentDetector(language, SentimentDetectorAuth("key", "host")) { 6 | protected override def callCognitiveServices(request: String): String = cognitiveServicesResponse 7 | override def buildRequestBody(text: String, textId: String): String = super.buildRequestBody(text, textId) 8 | } 9 | 10 | class CognitiveServicesSentimentDetectorSpec extends FlatSpec { 11 | "The sentiment detector" should "formulate correct request and parse response to domain types" in { 12 | val responseSentiment = 0.8 13 | val detector = new TestCognitiveServicesSentimentDetector(s"""{"documents":[{"id":"0","score":$responseSentiment}],"errors":[]}""", "en") 14 | val sentiment = detector.detectSentiment("some text") 15 | 16 | assert(sentiment.contains(responseSentiment)) 17 | } 18 | 19 | it should "ignore unsupported languages" in { 20 | val detector = new TestCognitiveServicesSentimentDetector("""{"documents":[],"errors":[{"id":"0","message":"Supplied language not supported. Pass in one of:en,es,pt,fr,de,it,nl,no,sv,pl,da,fi,ru,el,tr"}]}""", "zh") 21 | val sentiment = detector.detectSentiment("some text") 22 | 23 | assert(sentiment.isEmpty) 24 | } 25 | 26 | it should "build correct request body" in { 27 | val id = "0" 28 | val language = "en" 29 | val text = "some text" 30 | val requestBody = new TestCognitiveServicesSentimentDetector("", language).buildRequestBody(text, id) 31 | 32 | assert(requestBody == s"""{"documents":[{"id":"$id","text":"$text","language":"en"}]}""") 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /project-fortis-spark/src/test/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/sentiment/WordListSentimentDetectorIntegrationSpec.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.sentiment 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.transforms.ZipModelsProvider 4 | import com.microsoft.partnercatalyst.fortis.spark.transforms.sentiment.SentimentDetector.{Negative, Neutral, Positive} 5 | 6 | import org.scalatest.FlatSpec 7 | 8 | class WordListSentimentDetectorIntegrationSpec extends FlatSpec { 9 | "The word list sentiment detector" should "download models from blob" in { 10 | val modelsProvider = new ZipModelsProvider( 11 | language => s"https://fortiscentral.blob.core.windows.net/sentiment/sentiment-$language.zip") 12 | 13 | val testCases = List( 14 | ("victoire supérieure véritable siège tuer révolte révolte", "fr", Negative), 15 | ("erfolgreich unbeschränkt Pflege Zweifel tot angegriffen", "de", Neutral), 16 | ("libération du quai", "fr", Positive) 17 | ) 18 | 19 | testCases.foreach(test => { 20 | val detector = new WordListSentimentDetector(modelsProvider, test._2) 21 | val sentiment = detector.detectSentiment(test._1) 22 | assert(sentiment.contains(test._3)) 23 | }) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /project-fortis-spark/src/test/scala/com/microsoft/partnercatalyst/fortis/spark/transforms/topic/BlacklistSpec.scala: -------------------------------------------------------------------------------- 1 | package com.microsoft.partnercatalyst.fortis.spark.transforms.topic 2 | 3 | import com.microsoft.partnercatalyst.fortis.spark.dto.BlacklistedItem 4 | import org.scalatest.FlatSpec 5 | 6 | class BlacklistSpec extends FlatSpec { 7 | "The blacklist" should "match matching text" in { 8 | val blacklist = new Blacklist(Seq(BlacklistedItem(Set("foo"), isLocation = false))) 9 | assert(blacklist.matches("foo bar")) 10 | assert(!blacklist.matches("bar baz")) 11 | assert(blacklist.matches("foo bar".split(" ").toSet)) 12 | assert(!blacklist.matches("bar baz".split(" ").toSet)) 13 | } 14 | 15 | it should "match conjunctions" in { 16 | val blacklist = new Blacklist(Seq(BlacklistedItem(Set("foo", "bar"), isLocation = false))) 17 | assert(blacklist.matches("bar baz foo")) 18 | assert(!blacklist.matches("bar baz")) 19 | assert(blacklist.matches("bar baz foo".split(" ").toSet)) 20 | assert(!blacklist.matches("bar baz".split(" ").toSet)) 21 | } 22 | 23 | it should "match any conjunctions" in { 24 | val blacklist = new Blacklist( 25 | Seq(BlacklistedItem(Set("foo", "bar"), isLocation = false), BlacklistedItem(Set("pear"), isLocation = false)) 26 | ) 27 | assert(blacklist.matches("a b pear c")) 28 | assert(blacklist.matches("bar baz foo")) 29 | assert(blacklist.matches("a b pear c".split(" ").toSet)) 30 | assert(blacklist.matches("bar baz foo".split(" ").toSet)) 31 | } 32 | 33 | it should "handle the empty string" in { 34 | val blacklist = new Blacklist( 35 | Seq(BlacklistedItem(Set("foo", "bar"), isLocation = false), BlacklistedItem(Set("pear"), isLocation = false)) 36 | ) 37 | assert(!blacklist.matches("")) 38 | assert(!blacklist.matches(Set[String]())) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /project-fortis-spark/travis/ci.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | pushd "$(dirname $0)/.." 6 | 7 | sbt ++${TRAVIS_SCALA_VERSION} test 8 | 9 | popd 10 | -------------------------------------------------------------------------------- /project-fortis-spark/travis/publish.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | log() { 6 | echo "$@" >&2 7 | } 8 | 9 | check_preconditions() { 10 | if [ -z "${TRAVIS_TAG}" ]; then 11 | log "Build is not a tag, skipping publish" 12 | exit 0 13 | fi 14 | if [ -z "${DEPLOY_BLOB_ACCOUNT_NAME}" ] || [ -z "${DEPLOY_BLOB_ACCOUNT_KEY}" ] || [ -z "${DEPLOY_BLOB_CONTAINER}" ]; then 15 | log "Azure blob connection is not set, unable to publish builds" 16 | exit 1 17 | fi 18 | } 19 | 20 | install_azure_cli() { 21 | curl -sL 'https://deb.nodesource.com/setup_6.x' | sudo -E bash - 22 | sudo apt-get update 23 | sudo apt-get install -y -qq nodejs 24 | sudo npm install -g npm 25 | sudo npm install -g azure-cli 26 | } 27 | 28 | create_fat_jar() { 29 | sbt ++${TRAVIS_SCALA_VERSION} 'set test in assembly := {}' assembly 30 | } 31 | 32 | publish_fat_jar() { 33 | local fatjar="$(find target -name 'project-fortis-spark-assembly-*.jar' -print -quit)" 34 | 35 | if [ -z "${fatjar}" ] || [ ! -f "${fatjar}" ]; then 36 | log "Unable to locate fat jar" 37 | exit 1 38 | fi 39 | 40 | AZURE_NON_INTERACTIVE_MODE=1 \ 41 | azure storage blob upload \ 42 | --quiet \ 43 | --account-name "${DEPLOY_BLOB_ACCOUNT_NAME}" \ 44 | --account-key "${DEPLOY_BLOB_ACCOUNT_KEY}" \ 45 | --file "${fatjar}" \ 46 | --container "${DEPLOY_BLOB_CONTAINER}" \ 47 | --blob "fortis-${TRAVIS_TAG}.jar" 48 | } 49 | 50 | pushd "$(dirname $0)/.." 51 | 52 | check_preconditions 53 | install_azure_cli 54 | create_fat_jar 55 | publish_fat_jar 56 | 57 | popd 58 | -------------------------------------------------------------------------------- /project-fortis-spark/version.sbt: -------------------------------------------------------------------------------- 1 | version := "0.0.29" 2 | --------------------------------------------------------------------------------