├── images ├── sample-query-result.png └── tsignal-topology-small.png ├── configmaps ├── us-west1.yaml └── europe-west1.yaml ├── scripts ├── cleanup.sh ├── stocks.csv ├── config.sh ├── account.sh ├── setup.sh ├── store.ddl └── cluster.sh ├── Dockerfile ├── sql ├── counts.sql ├── temp.sql ├── price-per-hr.sql └── price-per-stock-hr.sql ├── .gitignore ├── processor.go ├── types.go ├── README.md ├── deployments └── tsignal.yaml ├── provider.go ├── prices.go ├── main.go ├── store.go └── LICENSE /images/sample-query-result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mchmarny/tsignal/HEAD/images/sample-query-result.png -------------------------------------------------------------------------------- /images/tsignal-topology-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mchmarny/tsignal/HEAD/images/tsignal-topology-small.png -------------------------------------------------------------------------------- /configmaps/us-west1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: cluster 5 | data: 6 | region: us-west1 7 | -------------------------------------------------------------------------------- /configmaps/europe-west1.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: cluster 5 | data: 6 | region: europe-west1 7 | -------------------------------------------------------------------------------- /scripts/cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR="$(dirname "$0")" 4 | . "${DIR}/config.sh" 5 | 6 | gcloud beta spanner instances delete ${GCLOUD_INSTANCE} 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang 2 | MAINTAINER Mark Chmarny 3 | 4 | RUN mkdir /app 5 | COPY ./tsignal /app/tsignal 6 | 7 | RUN mkdir /app/scripts 8 | COPY ./scripts/stocks.csv /app/scripts/stocks.csv 9 | 10 | WORKDIR /app 11 | CMD /app/tsignal 12 | -------------------------------------------------------------------------------- /scripts/stocks.csv: -------------------------------------------------------------------------------- 1 | AAPL,Apple,NASDAQ 2 | CSCO,Cisco,NASDAQ 3 | INTC,Intel,NASDAQ 4 | MSFT,Microsoft,NASDAQ 5 | EBAY,eBay,NASDAQ 6 | QCOM,QUALCOMM,NASDAQ 7 | MU,Micron,NASDAQ 8 | FB,Facebook,NASDAQ 9 | GRPN,Groupon,NASDAQ 10 | SBUX,Starbucks,NASDAQ 11 | -------------------------------------------------------------------------------- /sql/counts.sql: -------------------------------------------------------------------------------- 1 | -- Data summary 2 | SELECT * FROM 3 | (SELECT count(*) StockCount from Stocks), 4 | (SELECT count(*) PriceCount, MAX(SampleOn) LastPrice from Prices), 5 | (SELECT count(*) AuthorCount, MAX(UpdatedOn) LastAuthorUpUpdate from Authors), 6 | (SELECT count(*) PostCount, MAX(PostedOn) LastPost from Posts) 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | .DS_Store 27 | tsignal 28 | auth.json 29 | -------------------------------------------------------------------------------- /scripts/config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Assumes Twitter API envirnment variables are arleady defined 4 | # export T_CONSUMER_KEY="" 5 | # export T_CONSUMER_SECRET="" 6 | # export T_ACCESS_TOKEN="" 7 | # export T_ACCESS_SECRET="" 8 | 9 | # Google 10 | export APP_DIR="$(dirname "$0")" 11 | export APP_NAME="tsignal" 12 | export GCLOUD_PROJECT="cpe-data" 13 | export GOOGLE_APPLICATION_CREDENTIALS="./auth.json" 14 | export GCLOUD_INSTANCE="${APP_NAME}" 15 | export GCLOUD_DB="db" 16 | export GCLOUD_ZONE="us-west1-b" 17 | export GCLOUD_SA_NAME="${APP_NAME}-sa" 18 | export GCLOUD_SA_EMAIL="${GCLOUD_SA_NAME}@${GCLOUD_PROJECT}.iam.gserviceaccount.com" 19 | 20 | # set default procej 21 | gcloud config set project ${GCLOUD_PROJECT} 22 | -------------------------------------------------------------------------------- /scripts/account.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR="$(dirname "$0")" 4 | . "${DIR}/config.sh" 5 | 6 | 7 | echo "Checking if Service Account alredy created..." 8 | SA=$(gcloud iam service-accounts list --format='value(EMAIL)' --filter="EMAIL:${GCLOUD_SA_EMAIL}") 9 | if [ -z "${SA}" ]; then 10 | echo "Service Account not set, creating..." 11 | gcloud beta iam service-accounts create $GCLOUD_SA_NAME \ 12 | --display-name="${APP_NAME} service account" 13 | 14 | echo "Creating service account key..." 15 | gcloud iam service-accounts keys create --iam-account $GCLOUD_SA_EMAIL \ 16 | $GOOGLE_APPLICATION_CREDENTIALS 17 | fi 18 | 19 | echo "Creating service account bindings..." 20 | gcloud projects add-iam-policy-binding $GCLOUD_PROJECT \ 21 | --member "serviceAccount:${GCLOUD_SA_EMAIL}" \ 22 | --role='roles/spanner.databaseUser' 23 | -------------------------------------------------------------------------------- /scripts/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR="$(dirname "$0")" 4 | . "${DIR}/config.sh" 5 | 6 | 7 | gcloud beta spanner instances create ${GCLOUD_INSTANCE} \ 8 | --config=regional-us-central1 \ 9 | --description="${GCLOUD_INSTANCE} Instance" \ 10 | --nodes=1 11 | 12 | gcloud beta spanner databases create ${GCLOUD_DB} \ 13 | --instance=${GCLOUD_INSTANCE} 14 | 15 | # some gymnastics are required in order to parse a proper DDL in commandline 16 | echo 'Loading DDL...' 17 | echo 'NOTE: empty @type property warning on return protobuf message are OK' 18 | DDL=`cat ${DIR}/store.ddl | tr -d '\n' | tr -d '\r' | tr -d '\t'` 19 | IFS=';' read -ra LINES <<< "$DDL" 20 | for SQL in "${LINES[@]}"; do 21 | # echo $SQL 22 | if [ ${#SQL} -ge 5 ]; then 23 | gcloud beta spanner databases ddl update ${GCLOUD_DB} --instance=${GCLOUD_INSTANCE} --ddl="$SQL" 24 | fi 25 | done 26 | -------------------------------------------------------------------------------- /processor.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | lang "cloud.google.com/go/language/apiv1" 5 | langpb "google.golang.org/genproto/googleapis/cloud/language/v1" 6 | ) 7 | 8 | var ( 9 | langClient *lang.Client 10 | ) 11 | 12 | func initProcessor() error { 13 | var err error 14 | langClient, err = lang.NewClient(appContext) 15 | return err 16 | } 17 | 18 | func processSentiment(t Content, r chan<- Content) { 19 | logDebug.Printf("Processing [%v:%d]...", t.Post.Symbol, t.Post.PostID) 20 | result, err := langClient.AnalyzeSentiment(appContext, &langpb.AnalyzeSentimentRequest{ 21 | Document: &langpb.Document{ 22 | Source: &langpb.Document_Content{ 23 | Content: t.Post.Content, 24 | }, 25 | Type: langpb.Document_PLAIN_TEXT, 26 | }, 27 | EncodingType: langpb.EncodingType_UTF8, 28 | }) 29 | if err != nil { 30 | logDebug.Printf("Error envoking NLP API: %v", err) 31 | } else { 32 | t.Post.SentimentScore = float64(result.DocumentSentiment.Score) 33 | } 34 | r <- t 35 | 36 | } 37 | -------------------------------------------------------------------------------- /scripts/store.ddl: -------------------------------------------------------------------------------- 1 | CREATE TABLE Authors ( 2 | Username STRING(MAX) NOT NULL, 3 | FullName STRING(MAX) NOT NULL, 4 | FriendCount INT64, 5 | PostCount INT64 NOT NULL, 6 | Source STRING(MAX) NOT NULL, 7 | UpdatedOn TIMESTAMP NOT NULL 8 | ) PRIMARY KEY (Username); 9 | 10 | CREATE TABLE Posts ( 11 | Username STRING(MAX) NOT NULL, 12 | PostID INT64 NOT NULL, 13 | Symbol STRING(MAX) NOT NULL, 14 | PostedOn TIMESTAMP NOT NULL, 15 | Content STRING(MAX) NOT NULL, 16 | SentimentScore FLOAT64 NOT NULL 17 | ) PRIMARY KEY (Username, PostID), 18 | INTERLEAVE IN PARENT Authors ON DELETE CASCADE; 19 | 20 | CREATE INDEX PostSymbolIndex ON Posts (Symbol, PostedOn); 21 | 22 | CREATE TABLE Stocks ( 23 | Symbol STRING(MAX) NOT NULL, 24 | Company STRING(MAX) NOT NULL, 25 | Exchange STRING(MAX) NOT NULL 26 | ) PRIMARY KEY (Symbol); 27 | 28 | CREATE TABLE Prices ( 29 | Symbol STRING(MAX) NOT NULL, 30 | SampleOn TIMESTAMP NOT NULL, 31 | AskPrice FLOAT64 NOT NULL 32 | ) PRIMARY KEY (Symbol, SampleOn), 33 | INTERLEAVE IN PARENT Stocks ON DELETE CASCADE; 34 | -------------------------------------------------------------------------------- /sql/temp.sql: -------------------------------------------------------------------------------- 1 | -- Select hourly price 2 | SELECT p.Symbol, p.AskHour, ROUND(avg(p.AskPrice),2) as AvgAskPrice, count(*) RecordCount 3 | FROM ( 4 | SELECT Symbol, FORMAT_TIMESTAMP("%F-%k", SampleOn) AskHour, AskPrice 5 | FROM Prices 6 | ) p 7 | GROUP BY p.Symbol, p.AskHour 8 | 9 | -- Select posts sentiment by hour 10 | SELECT p.Symbol, p.PostHour, ROUND(AVG(p.SentimentScore),2) as AvgSentiment, count(*) RecordCount 11 | FROM ( 12 | SELECT Symbol, FORMAT_TIMESTAMP("%F-%k", PostedOn) PostHour, SentimentScore 13 | FROM Posts 14 | ) p 15 | GROUP BY p.Symbol, p.PostHour 16 | 17 | -- Most recent negative tweets 18 | select p.PostID, FORMAT_TIMESTAMP("%F-%k", p.PostedOn) PostHour, p.SentimentScore, p.Content 19 | FROM Posts p 20 | JOIN Authors a on p.Username = a.Username 21 | WHERE p.SentimentScore < 0 and a.Source = 'Twitter' 22 | ORDER BY p.PostedOn DESC 23 | 24 | -- One massive query for negative twitter posts by author's post count 25 | SELECT a.PostCount, p.SentimentScore, p.Content 26 | FROM Authors a 27 | JOIN Posts p ON a.Username = p.Username 28 | JOIN Prices r ON p.Symbol = r.Symbol 29 | WHERE p.Symbol = 'AAPL' 30 | AND a.Source = 'Twitter' 31 | AND p.SentimentScore < 0 32 | ORDER BY a.PostCount DESC 33 | -------------------------------------------------------------------------------- /scripts/cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # get global vars 4 | . scripts/config.sh 5 | 6 | # create cluster 7 | gcloud container clusters create "${APP_NAME}-cluster" \ 8 | --project ${GCLOUD_PROJECT} \ 9 | --machine-type "n1-standard-1" \ 10 | --image-type "COS" \ 11 | --disk-size "100" \ 12 | --scopes default,cloud-platform,logging-write,monitoring-write \ 13 | --num-nodes "1" \ 14 | --zone $GCLOUD_ZONE \ 15 | --network "default" \ 16 | --enable-cloud-logging \ 17 | --enable-cloud-monitoring 18 | 19 | # connect 20 | gcloud container clusters get-credentials "${APP_NAME}-cluster" \ 21 | --zone $GCLOUD_ZONE --project $GCLOUD_PROJECT 22 | 23 | # configs 24 | kubectl create configmap tsignal-config --from-file configmaps/us-west1.yaml 25 | 26 | 27 | # populate twitter secrets 28 | kubectl create secret generic tsignal-tw-key --from-literal=T_CONSUMER_KEY=$T_CONSUMER_KEY 29 | kubectl create secret generic tsignal-tw-secret --from-literal=T_CONSUMER_SECRET=$T_CONSUMER_SECRET 30 | kubectl create secret generic tsignal-tw-token --from-literal=T_ACCESS_TOKEN=$T_ACCESS_TOKEN 31 | kubectl create secret generic tsignal-tw-access --from-literal=T_ACCESS_SECRET=$T_ACCESS_SECRET 32 | 33 | # populate app secrets 34 | kubectl create secret generic tsignal-gcloud-project --from-literal=GCLOUD_PROJECT=$GCLOUD_PROJECT 35 | kubectl create secret generic tsignal-spanner-instance --from-literal=GCLOUD_INSTANCE=$GCLOUD_INSTANCE 36 | kubectl create secret generic tsignal-spanner-db --from-literal=GCLOUD_DB=$GCLOUD_DB 37 | kubectl create secret generic tsignal-sa --from-file auth.json 38 | 39 | # deploy 40 | # kubectl create -f deployments/tsignal.yaml 41 | -------------------------------------------------------------------------------- /types.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "time" 4 | 5 | // Price struct represents price in single point in time. 6 | type Price struct { 7 | Symbol string `json:"symbol"` 8 | AskPrice float64 `json:"ask_price"` 9 | SampleOn time.Time `json:"sample_on"` 10 | } 11 | 12 | // StockPrice is used as container for stock price data. 13 | type StockPrice struct { 14 | Stock Stock `json:"stock"` 15 | Price *Price `json:"price"` 16 | } 17 | 18 | // Content represents simple tweet content 19 | type Content struct { 20 | Post Post `json:"post"` 21 | Author Author `json:"author"` 22 | } 23 | 24 | // Author represents post author 25 | type Author struct { 26 | Username string `json:"username"` 27 | FullName string `json:"full_name"` 28 | FriendCount int64 `json:"friend_count"` 29 | PostCount int64 `json:"post_count"` 30 | Source string `json:"source"` 31 | UpdatedOn time.Time `json:"updated_on"` 32 | } 33 | 34 | // Post represents simple tweet content 35 | type Post struct { 36 | Username string `json:"username"` 37 | PostID int64 `json:"post_id"` 38 | PostedOn time.Time `json:"posted_on"` 39 | Symbol string `json:"symbol"` 40 | Content string `json:"content"` 41 | SentimentScore float64 `json:"sentiment_score"` 42 | } 43 | 44 | // Stock represents simple stcik item 45 | type Stock struct { 46 | Symbol string `json:"symbol"` 47 | Company string `json:"company"` 48 | Exchange string `json:"exchange"` 49 | } 50 | 51 | // ProviderRerun represents state of the provider job 52 | type ProviderRerun struct { 53 | Stock Stock 54 | Channel chan<- Content 55 | Error interface{} 56 | } 57 | -------------------------------------------------------------------------------- /sql/price-per-hr.sql: -------------------------------------------------------------------------------- 1 | -- Price per stock per hour 2 | SELECT p.Symbol, 3 | p.PostHour, 4 | ROUND(MIN(c.AskPrice),2) MinAskPrice, 5 | ROUND(AVG(c.AskPrice),2) AvgAskPrice, 6 | ROUND(MAX(c.AskPrice),2) MaxAskPrice, 7 | ROUND(AVG(p.SentimentScore),3) AvgSentiment, 8 | ROUND(AVG(p.WeightedScore),2) WeightedScore, 9 | SUM(p.PostCount) PostCount, 10 | SUM(p.PositiveCount) Positives, 11 | SUM(p.NegativeCount) Negatives, 12 | SUM(p.NoiseCount) Noise 13 | FROM ( 14 | SELECT r.Symbol, 15 | FORMAT_TIMESTAMP("%F-%k", r.PostedOn) 16 | PostHour, r.SentimentScore, 17 | -- Using Friend Count as a weight multiplier for significant sentiment, else 0 18 | (1 + a.FriendCount / 1000) * CASE 19 | -- if strong then full credit for score 20 | WHEN ABS(r.SentimentScore) > 0.6 THEN r.SentimentScore 21 | -- if poor give them 50% credit for the sentiment score 22 | WHEN ABS(r.SentimentScore) >= 0.3 AND ABS(r.SentimentScore) <= 0.6 THEN r.SentimentScore * 0.5 23 | -- else it is noise to ignore 24 | ELSE 0 END WeightedScore, 25 | 1 PostCount, 26 | -- for positic/negatives using 0.3 significant sentiment filter, else noise 27 | CASE WHEN r.SentimentScore >= 0.3 THEN 1 ELSE 0 END PositiveCount, 28 | CASE WHEN r.SentimentScore <= -0.3 THEN 1 ELSE 0 END NegativeCount, 29 | CASE WHEN r.SentimentScore > -0.3 AND r.SentimentScore < 0.3 THEN 1 ELSE 0 END NoiseCount 30 | FROM Posts r 31 | JOIN Authors a ON r.Username = a.Username 32 | ) p 33 | LEFT JOIN ( 34 | SELECT Symbol, FORMAT_TIMESTAMP("%F-%k", SampleOn) AskHour, AskPrice 35 | FROM Prices 36 | ) c ON p.Symbol = c.Symbol AND p.PostHour = c.AskHour 37 | GROUP BY p.Symbol, p.PostHour 38 | ORDER BY p.Symbol, p.PostHour DESC 39 | -------------------------------------------------------------------------------- /sql/price-per-stock-hr.sql: -------------------------------------------------------------------------------- 1 | -- Same as above but for a given stock 2 | SELECT p.Symbol, 3 | p.PostHour, 4 | ROUND(MIN(c.AskPrice),2) MinAskPrice, 5 | ROUND(AVG(c.AskPrice),2) AvgAskPrice, 6 | ROUND(MAX(c.AskPrice),2) MaxAskPrice, 7 | ROUND(AVG(p.SentimentScore),3) AvgSentiment, 8 | ROUND(AVG(p.WeightedScore),2) WeightedScore, 9 | SUM(p.PostCount) PostCount, 10 | SUM(p.PositiveCount) Positives, 11 | SUM(p.NegativeCount) Negatives, 12 | SUM(p.NoiseCount) Noise 13 | FROM ( 14 | SELECT r.Symbol, 15 | FORMAT_TIMESTAMP("%F-%k", r.PostedOn) 16 | PostHour, r.SentimentScore, 17 | -- Using Friend Count as a weight multiplier for significant sentiment, else 0 18 | (1 + a.FriendCount / 1000) * CASE 19 | -- if strong then full credit for score 20 | WHEN ABS(r.SentimentScore) > 0.6 THEN r.SentimentScore 21 | -- if poor give them 50% credit for the sentiment score 22 | WHEN ABS(r.SentimentScore) >= 0.3 AND ABS(r.SentimentScore) <= 0.6 THEN r.SentimentScore * 0.5 23 | -- else it is noise to ignore 24 | ELSE 0 END WeightedScore, 25 | 1 PostCount, 26 | -- for positic/negatives using 0.3 significant sentiment filter, else noise 27 | CASE WHEN r.SentimentScore >= 0.3 THEN 1 ELSE 0 END PositiveCount, 28 | CASE WHEN r.SentimentScore <= -0.3 THEN 1 ELSE 0 END NegativeCount, 29 | CASE WHEN r.SentimentScore > -0.3 AND r.SentimentScore < 0.3 THEN 1 ELSE 0 END NoiseCount 30 | FROM Posts r 31 | JOIN Authors a ON r.Username = a.Username 32 | WHERE r.Symbol = 'MSFT' 33 | ) p 34 | LEFT JOIN ( 35 | SELECT Symbol, FORMAT_TIMESTAMP("%F-%k", SampleOn) AskHour, AskPrice 36 | FROM Prices 37 | ) c ON p.Symbol = c.Symbol AND p.PostHour = c.AskHour 38 | GROUP BY p.Symbol, p.PostHour 39 | ORDER BY p.Symbol, p.PostHour DESC 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tsignal 2 | Analyzing social media sentiment and its impact on stock market 3 | 4 | > Personal project, does not represent Google 5 | 6 | ![tsignal topology](/../master/images/tsignal-topology-small.png?raw=true "tsignal topology") 7 | 8 | This Go app deploys into [GKE](https://cloud.google.com/container-engine/), subscribes to Twitter stream for all companies defined in the `Stocks` table in [Cloud Spanner](https://cloud.google.com/spanner/) and scores each event against the [Google NLP API](https://cloud.google.com/natural-language/) while comparing the user sentiment against the stock ask price against Yahoo API. 9 | 10 | Eventually there will be a UI, for now, there are sample SQL scripts you can use to execute against the Cloud Spanner DB to analyze the data. 11 | 12 | ![sample query](/../master/images/sample-query-result.png?raw=true "sample query") 13 | 14 | > All GCP services used in this example can be run under the GCP Free Tier plan. More more information see https://cloud.google.com/free/ 15 | 16 | ## Configuration 17 | 18 | Edit the `scripts/config.sh` file with your Twitter API info. Alternatively 19 | define the following environment variables. Instructions how to configure your Twitter API access codes are found [here](http://docs.inboundnow.com/guide/create-twitter-application/): 20 | 21 | ``` 22 | # export T_CONSUMER_KEY="" 23 | # export T_CONSUMER_SECRET="" 24 | # export T_ACCESS_TOKEN="" 25 | # export T_ACCESS_SECRET="" 26 | ``` 27 | 28 | ## Run 29 | 30 | Once all the necessary environment variables are defined you can execute the `tsignal` appl locally `go run *.go` or use the included `Dockerfile` to create an image which you can then publish to the GCP (if needed, there is a script `build-publish` that will do all that for you. 31 | 32 | ## Cleanup 33 | 34 | The cleanup of all the resources created in this application can be accomplished by executing the `scripts/cleanup.sh` script. 35 | 36 | ### TODO 37 | 38 | * Tests, yes please 39 | * UI for reports and config 40 | * Way to subscribe to non-NASDAQ stock prices 41 | -------------------------------------------------------------------------------- /deployments/tsignal.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | app: tsignal 6 | name: tsignal 7 | spec: 8 | replicas: 1 9 | template: 10 | metadata: 11 | labels: 12 | app: tsignal 13 | name: tsignal 14 | spec: 15 | containers: 16 | - name: tsignal 17 | image: gcr.io/cpe-data/tsignal:0.2.5 18 | imagePullPolicy: Always 19 | env: 20 | - name: T_CONSUMER_KEY 21 | valueFrom: 22 | secretKeyRef: 23 | name: tsignal-tw-key 24 | key: T_CONSUMER_KEY 25 | - name: T_CONSUMER_SECRET 26 | valueFrom: 27 | secretKeyRef: 28 | name: tsignal-tw-secret 29 | key: T_CONSUMER_SECRET 30 | - name: T_ACCESS_TOKEN 31 | valueFrom: 32 | secretKeyRef: 33 | name: tsignal-tw-token 34 | key: T_ACCESS_TOKEN 35 | - name: T_ACCESS_SECRET 36 | valueFrom: 37 | secretKeyRef: 38 | name: tsignal-tw-access 39 | key: T_ACCESS_SECRET 40 | - name: GCLOUD_PROJECT 41 | valueFrom: 42 | secretKeyRef: 43 | name: tsignal-gcloud-project 44 | key: GCLOUD_PROJECT 45 | - name: GCLOUD_INSTANCE 46 | valueFrom: 47 | secretKeyRef: 48 | name: tsignal-spanner-instance 49 | key: GCLOUD_INSTANCE 50 | - name: GCLOUD_DB 51 | valueFrom: 52 | secretKeyRef: 53 | name: tsignal-spanner-db 54 | key: GCLOUD_DB 55 | volumeMounts: 56 | - name: "service-account" 57 | mountPath: "/var/run/secret/cloud.google.com" 58 | - name: "certs" 59 | mountPath: "/etc/ssl/certs" 60 | volumes: 61 | - name: "service-account" 62 | secret: 63 | secretName: "tsignal-sa" 64 | - name: "certs" 65 | hostPath: 66 | path: "/etc/ssl/certs" 67 | -------------------------------------------------------------------------------- /provider.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | "time" 8 | 9 | "github.com/dghubble/go-twitter/twitter" 10 | "github.com/dghubble/oauth1" 11 | ) 12 | 13 | const ( 14 | dataSource = "Twitter" 15 | layoutTwitter = "Mon Jan 02 15:04:05 -0700 2006" 16 | layoutBigQuery = "2006-01-02 15:04:05" 17 | streamLimitPauseMin = 10 18 | ) 19 | 20 | // start initiates the Tweeter stream subscription and pumps all messages into 21 | // the passed in channel 22 | func subscribeToStream(stock Stock, ch chan<- Content) { 23 | 24 | logInfo.Printf("Subscribing to [%v:%v]...", stock.Symbol, stock.Company) 25 | 26 | consumerKey := os.Getenv("T_CONSUMER_KEY") 27 | consumerSecret := os.Getenv("T_CONSUMER_SECRET") 28 | accessToken := os.Getenv("T_ACCESS_TOKEN") 29 | accessSecret := os.Getenv("T_ACCESS_SECRET") 30 | 31 | if consumerKey == "" || consumerSecret == "" || accessToken == "" || accessSecret == "" { 32 | logErr.Fatal("Both, consumer key/secret and access token/secret are required") 33 | return 34 | } 35 | 36 | // init convif 37 | config := oauth1.NewConfig(consumerKey, consumerSecret) 38 | token := oauth1.NewToken(accessToken, accessSecret) 39 | 40 | // HTTP Client - will automatically authorize Requests 41 | httpClient := config.Client(appContext, token) 42 | httpClient.Timeout = time.Duration(30 * time.Second) 43 | client := twitter.NewClient(httpClient) 44 | demux := twitter.NewSwitchDemux() 45 | 46 | //Tweet processor 47 | demux.Tweet = func(tweet *twitter.Tweet) { 48 | // check if the tweet is a retweet 49 | if tweet.RetweetedStatus == nil { 50 | aquiredOn := time.Now() 51 | username := strings.ToLower(tweet.User.ScreenName) 52 | msg := Content{ 53 | Post: Post{ 54 | Symbol: stock.Symbol, 55 | PostID: tweet.ID, 56 | PostedOn: aquiredOn, 57 | Content: tweet.Text, 58 | Username: username, 59 | }, 60 | Author: Author{ 61 | Username: username, 62 | FullName: tweet.User.Name, 63 | FriendCount: int64(tweet.User.FollowersCount), 64 | PostCount: int64(tweet.User.StatusesCount), 65 | Source: dataSource, 66 | UpdatedOn: aquiredOn, 67 | }, 68 | } 69 | logDebug.Printf("Post [%v:%d]", stock.Symbol, msg.Post.PostID) 70 | ch <- msg 71 | } 72 | } 73 | 74 | // Tweet filter 75 | filterParams := &twitter.StreamFilterParams{ 76 | Track: []string{ 77 | "#" + stock.Symbol, // hashtag 78 | "$" + stock.Symbol, // stock sybmob search 79 | stock.Company, // just plain name of the company 80 | }, 81 | StallWarnings: twitter.Bool(true), 82 | Language: []string{"en"}, 83 | } 84 | 85 | // Start stream 86 | stream, err := client.Streams.Filter(filterParams) 87 | if err != nil { 88 | providerErrors <- ProviderRerun{ 89 | Error: fmt.Sprintf("Error while creating stream filter: %v", err), 90 | Channel: ch, 91 | Stock: stock, 92 | } 93 | return 94 | } 95 | 96 | demux.StreamLimit = func(limit *twitter.StreamLimit) { 97 | logErr.Printf("Reached stream limit %v - pausing: %d min", 98 | limit.Track, streamLimitPauseMin) 99 | time.Sleep(time.Duration(streamLimitPauseMin * time.Minute)) 100 | providerErrors <- ProviderRerun{ 101 | Error: fmt.Sprintf("Error while creating stream filter: %v", err), 102 | Channel: ch, 103 | Stock: stock, 104 | } 105 | return 106 | } 107 | 108 | // do the work 109 | go demux.HandleChan(stream.Messages) 110 | } 111 | -------------------------------------------------------------------------------- /prices.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/csv" 5 | "fmt" 6 | "net" 7 | "net/http" 8 | "strconv" 9 | "sync" 10 | "time" 11 | ) 12 | 13 | const ( 14 | // NOTE: At this time only NASDAQ traded stocks are supported 15 | // Yahoo has disabled NYSE for some reason while back 16 | yahooFinURL = "http://finance.yahoo.com/d/quotes.csv?s=%s&f=%s" 17 | 18 | // formatting for onlt the ask price [a] 19 | baseFormat = "a" 20 | ) 21 | 22 | func isStockMarketOpened() bool { 23 | est, _ := time.LoadLocation("EST") 24 | now := time.Now().In(est) 25 | day := now.Weekday() 26 | hr := now.Hour() 27 | 28 | // TODO: Market is open from 9:30 29 | return day != 6 && day != 7 && hr >= 9 && hr < 16 30 | } 31 | 32 | // updatePrices updates process for all stocks 33 | func updatePrices(stocks []Stock, r chan<- bool) { 34 | var numOfErrors int 35 | var mu sync.Mutex 36 | pauseDuration := time.Duration(stockRefreshMin) * time.Minute 37 | for { 38 | if isStockMarketOpened() { 39 | mu.Lock() 40 | numOfErrors = 0 41 | mu.Unlock() 42 | for _, stock := range stocks { 43 | price, err := getAskPrice(stock.Symbol) 44 | if err != nil { 45 | mu.Lock() 46 | numOfErrors++ 47 | mu.Unlock() 48 | logErr.Printf("Error getting prices for: %v - %v", stock.Symbol, err) 49 | continue 50 | } 51 | // save the current price 52 | inserErr := savePrice(price) 53 | if inserErr != nil { 54 | mu.Lock() 55 | numOfErrors++ 56 | mu.Unlock() 57 | logErr.Printf("Error on insert: %v - %v", stock.Symbol, inserErr) 58 | continue 59 | } 60 | } 61 | r <- numOfErrors == 0 62 | } else { 63 | logDebug.Print("Stock market closed") 64 | } 65 | time.Sleep(pauseDuration) 66 | } 67 | } 68 | 69 | // getAskPrice single stock price data 70 | func getAskPrice(symbol string) (*Price, error) { 71 | 72 | data, err := loadSingleStockPrice(symbol) 73 | if err != nil { 74 | return nil, err 75 | } 76 | 77 | val, fpErr := strconv.ParseFloat(data[0], 64) 78 | if fpErr != nil { 79 | logErr.Printf("Error while parsing prices: %v - %v", data[0], fpErr) 80 | return nil, fpErr 81 | } 82 | 83 | price := &Price{ 84 | AskPrice: val, 85 | Symbol: symbol, 86 | SampleOn: time.Now(), 87 | } 88 | 89 | return price, nil 90 | } 91 | 92 | func getHTTPClient(symbol string) (*http.Client, error) { 93 | tr := &http.Transport{ 94 | MaxIdleConns: 10, 95 | IdleConnTimeout: 30 * time.Second, 96 | DisableCompression: true, 97 | Dial: (&net.Dialer{ 98 | Timeout: 5 * time.Second, 99 | }).Dial, 100 | TLSHandshakeTimeout: 5 * time.Second, 101 | } 102 | c := &http.Client{ 103 | Transport: tr, 104 | Timeout: time.Duration(30 * time.Second), 105 | } 106 | return c, nil 107 | } 108 | 109 | // Single stock data request 110 | func loadSingleStockPrice(symbol string) ([]string, error) { 111 | 112 | c, cErr := getHTTPClient(symbol) 113 | if cErr != nil { 114 | logErr.Printf("Error creating yahoo client %v", cErr) 115 | return nil, cErr 116 | } 117 | 118 | url := fmt.Sprintf(yahooFinURL, symbol, baseFormat) 119 | logDebug.Printf("Stock URL: %v", url) 120 | 121 | resp, err := c.Get(url) 122 | if err != nil { 123 | logErr.Printf("Error creating yahoo client %v", err) 124 | return nil, err 125 | } 126 | 127 | defer resp.Body.Close() 128 | reader := csv.NewReader(resp.Body) 129 | data, err := reader.Read() 130 | if err != nil { 131 | return nil, err 132 | } 133 | logDebug.Printf("Stock Prices Data: %v", data) 134 | return data, err 135 | } 136 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "os/signal" 9 | "sync" 10 | "syscall" 11 | 12 | "golang.org/x/net/context" 13 | ) 14 | 15 | const ( 16 | defaultStockRefreshMin = 25 17 | ) 18 | 19 | var ( 20 | 21 | // logs 22 | logInfo *log.Logger 23 | logDebug *log.Logger 24 | logErr *log.Logger 25 | 26 | appContext context.Context 27 | projectID string 28 | instanceID string 29 | dbID string 30 | 31 | debug bool 32 | stockRefreshMin int 33 | 34 | providerErrors = make(chan ProviderRerun) 35 | maxProviderErrorCount = 10 36 | subErrors = make(map[string]int) 37 | subMu sync.Mutex 38 | ) 39 | 40 | func main() { 41 | 42 | logInfo = log.New(os.Stdout, "INFO: ", log.Ldate|log.Ltime|log.Lshortfile) 43 | logErr = log.New(os.Stderr, "ERROR: ", log.Ldate|log.Ltime|log.Lshortfile) 44 | 45 | // START CONFIG 46 | flag.StringVar(&projectID, "project", os.Getenv("GCLOUD_PROJECT"), 47 | "GCP Project ID") 48 | flag.StringVar(&instanceID, "instance", os.Getenv("GCLOUD_INSTANCE"), 49 | "GCP Spanner Instance ID") 50 | flag.StringVar(&dbID, "db", os.Getenv("GCLOUD_DB"), "GCP Spanner DB ID") 51 | flag.BoolVar(&debug, "debug", debug, "Use verbose logging") 52 | flag.IntVar(&stockRefreshMin, "refresh", defaultStockRefreshMin, 53 | "Frequency of stock refresh in minutes") 54 | flag.Parse() 55 | 56 | if projectID == "" || instanceID == "" || dbID == "" { 57 | logErr.Fatalf("Missing required argument: project:%v, instance:%v, db:%v", 58 | projectID, instanceID, dbID) 59 | } 60 | logInfo.Printf("CONF - project:%v, instance:%v, db:%v, debug:%v, refresh:%v", 61 | projectID, instanceID, dbID, debug, stockRefreshMin) 62 | // END CONFIG 63 | 64 | // configure loggers 65 | if debug { 66 | logDebug = log.New(os.Stderr, "DEBUG: ", log.Ldate|log.Ltime|log.Lshortfile) 67 | } else { 68 | logDebug = log.New(ioutil.Discard, "", log.Ldate) 69 | } 70 | 71 | // context 72 | ctx, cancel := context.WithCancel(context.Background()) 73 | appContext = ctx 74 | go func() { 75 | // Wait for SIGINT and SIGTERM (HIT CTRL-C) 76 | ch := make(chan os.Signal) 77 | signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) 78 | logInfo.Println(<-ch) 79 | cancel() 80 | os.Exit(0) 81 | }() 82 | 83 | // init data store 84 | storeErr := initStore() 85 | if storeErr != nil { 86 | logErr.Fatal(storeErr) 87 | return 88 | } 89 | 90 | // get stocks to process 91 | stocks, err := getStocks() 92 | if err != nil { 93 | logErr.Fatalf("Error getting stocks: %v", err) 94 | return 95 | } 96 | 97 | // channels for pipeline 98 | chanells := len(stocks) * 5 // number of channels per stock 99 | tweets := make(chan Content, chanells) 100 | results := make(chan Content, chanells) 101 | pricecheckResult := make(chan bool) 102 | 103 | // update pricess 104 | go updatePrices(stocks, pricecheckResult) 105 | 106 | // initialize processor 107 | processErr := initProcessor() 108 | if processErr != nil { 109 | logErr.Fatalf("Error from processor: %v", processErr) 110 | return 111 | } 112 | 113 | for _, s := range stocks { 114 | go subscribeToStream(s, tweets) 115 | } 116 | 117 | // report 118 | for { 119 | select { 120 | case <-appContext.Done(): 121 | break 122 | case t := <-tweets: 123 | go processSentiment(t, results) 124 | case p := <-pricecheckResult: 125 | logDebug.Printf("Price Check Succeeded: %v", p) 126 | case t := <-providerErrors: 127 | checkSubErrors(&t) 128 | go subscribeToStream(t.Stock, tweets) 129 | case r := <-results: 130 | savePost(&r) 131 | logDebug.Printf("Result [%d:%v]", r.Post.PostID, r.Post.SentimentScore) 132 | } 133 | } 134 | } 135 | 136 | func checkSubErrors(t *ProviderRerun) { 137 | logDebug.Printf("Provider rerun for %v", t.Stock.Symbol) 138 | if subErrors[t.Stock.Symbol] > maxProviderErrorCount { 139 | logErr.Fatalf("Max numbers of retries for %v reached: %d - %v", 140 | t.Stock.Symbol, 141 | subErrors[t.Stock.Symbol], 142 | t.Error, 143 | ) 144 | } 145 | subMu.Lock() 146 | subErrors[t.Stock.Symbol] = subErrors[t.Stock.Symbol] + 1 147 | subMu.Unlock() 148 | } 149 | -------------------------------------------------------------------------------- /store.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/csv" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | "cloud.google.com/go/spanner" 10 | "google.golang.org/api/iterator" 11 | ) 12 | 13 | const ( 14 | defaultStockData = "scripts/stocks.csv" 15 | stockTable = "Stocks" 16 | authorTable = "Authors" 17 | postTable = "Posts" 18 | postTableTimeIndex = "PostSymbolIndex" 19 | priceTable = "Prices" 20 | ) 21 | 22 | var ( 23 | client *spanner.Client 24 | postPostprocessingWindow = time.Duration(-1 * time.Hour) 25 | selectStocksStatement = spanner.NewStatement("SELECT * FROM Stocks ORDER BY Symbol") 26 | selectPostsSinceStatement = spanner.NewStatement( 27 | "SELECT * FROM Posts@{FORCE_INDEX=PostSymbolIndex} WHERE Symbol = @symbol AND PostedOn >= @since") 28 | ) 29 | 30 | func initStore() error { 31 | 32 | // projects/my-project/instances/my-instance/databases/example-db 33 | db := fmt.Sprintf("projects/%v/instances/%v/databases/%v", 34 | projectID, instanceID, dbID) 35 | 36 | logInfo.Printf("Initializing store [%v]...", db) 37 | c, err := spanner.NewClient(appContext, db) 38 | if err != nil { 39 | return err 40 | } 41 | client = c 42 | 43 | return loadDefaultData() 44 | } 45 | 46 | func savePrice(p *Price) error { 47 | 48 | mP, err := spanner.InsertOrUpdateStruct(priceTable, p) 49 | if err != nil { 50 | logErr.Fatal(err) 51 | return err 52 | } 53 | 54 | _, insertErr := client.Apply(appContext, []*spanner.Mutation{mP}) 55 | if insertErr != nil { 56 | logErr.Fatal(insertErr) 57 | } 58 | 59 | return insertErr 60 | 61 | } 62 | 63 | func savePost(p *Content) error { 64 | 65 | // insert author 66 | mAuth, maErr := spanner.InsertOrUpdateStruct(authorTable, p.Author) 67 | if maErr != nil { 68 | logErr.Fatal(maErr) 69 | return maErr 70 | } 71 | 72 | mPost, mpErr := spanner.InsertOrUpdateStruct(postTable, p.Post) 73 | if mpErr != nil { 74 | logErr.Fatal(mpErr) 75 | return mpErr 76 | } 77 | 78 | _, insertErr := client.Apply(appContext, []*spanner.Mutation{mAuth, mPost}) 79 | if insertErr != nil { 80 | logErr.Fatal(insertErr) 81 | } 82 | 83 | return insertErr 84 | 85 | } 86 | 87 | func loadDefaultData() error { 88 | 89 | f, err := os.Open(defaultStockData) 90 | if err != nil { 91 | logErr.Fatal(err) 92 | return err 93 | } 94 | defer f.Close() 95 | 96 | r := csv.NewReader(f) 97 | stocks, err := r.ReadAll() 98 | if err != nil { 99 | logErr.Fatal(err) 100 | return err 101 | } 102 | 103 | records := make([]*spanner.Mutation, len(stocks)) 104 | for i, stock := range stocks { 105 | record, mErr := spanner.InsertOrUpdateStruct(stockTable, Stock{ 106 | Symbol: stock[0], 107 | Company: stock[1], 108 | Exchange: stock[2], 109 | }) 110 | if mErr != nil { 111 | logErr.Print(mErr) 112 | } 113 | records[i] = record 114 | } 115 | 116 | _, insertErr := client.Apply(appContext, records) 117 | if insertErr != nil { 118 | logErr.Fatal(insertErr) 119 | return insertErr 120 | } 121 | return nil 122 | 123 | } 124 | 125 | func getPostsSince(symbol string, since time.Time) ([]*Post, error) { 126 | 127 | selectPostsSinceStatement.Params["symbol"] = symbol 128 | selectPostsSinceStatement.Params["since"] = since 129 | 130 | posts := []*Post{} 131 | iter := client.Single().Query(appContext, selectPostsSinceStatement) 132 | defer iter.Stop() 133 | for { 134 | row, err := iter.Next() 135 | if err == iterator.Done { 136 | logDebug.Print("Done post processing") 137 | return posts, nil 138 | } 139 | if err != nil { 140 | logInfo.Printf("Error while processing posts: %v", err) 141 | return posts, err 142 | } 143 | 144 | var p Post 145 | if err := row.ToStruct(&p); err != nil { 146 | logInfo.Printf("Error while processing post row: %v", err) 147 | return posts, err 148 | } 149 | logDebug.Printf("POST PROCESS: %v", p) 150 | posts = append(posts, &p) 151 | } 152 | } 153 | 154 | func getStocks() ([]Stock, error) { 155 | 156 | stocks := []Stock{} 157 | iter := client.Single().Query(appContext, selectStocksStatement) 158 | defer iter.Stop() 159 | for { 160 | row, err := iter.Next() 161 | if err == iterator.Done { 162 | return stocks, nil 163 | } 164 | if err != nil { 165 | return stocks, err 166 | } 167 | 168 | var colSymbol, colComp, colExchange string 169 | if err := row.Columns(&colSymbol, &colComp, &colExchange); err != nil { 170 | return stocks, err 171 | } 172 | stock := Stock{ 173 | Symbol: colSymbol, 174 | Company: colComp, 175 | Exchange: colExchange, 176 | } 177 | stocks = append(stocks, stock) 178 | } 179 | } 180 | 181 | func query(sql string) error { 182 | stmt := spanner.Statement{SQL: sql} 183 | iter := client.Single().Query(appContext, stmt) 184 | defer iter.Stop() 185 | for { 186 | row, err := iter.Next() 187 | if err == iterator.Done { 188 | return nil 189 | } 190 | if err != nil { 191 | return err 192 | } 193 | var singerID, albumID int64 194 | var albumTitle string 195 | if err := row.Columns(&singerID, &albumID, &albumTitle); err != nil { 196 | return err 197 | } 198 | logInfo.Printf("%d %d %s\n", singerID, albumID, albumTitle) 199 | } 200 | } 201 | 202 | func read(table string, args []string) error { 203 | iter := client.Single().Read(appContext, table, spanner.AllKeys(), args) 204 | defer iter.Stop() 205 | for { 206 | row, err := iter.Next() 207 | if err == iterator.Done { 208 | return nil 209 | } 210 | if err != nil { 211 | return err 212 | } 213 | var singerID, albumID int64 214 | var albumTitle string 215 | if err := row.Columns(&singerID, &albumID, &albumTitle); err != nil { 216 | return err 217 | } 218 | logDebug.Printf("%d %d %s\n", singerID, albumID, albumTitle) 219 | } 220 | } 221 | 222 | /* 223 | func update(ctx context.Context, w io.Writer, client *spanner.Client) error { 224 | cols := []string{"SingerId", "AlbumId", "MarketingBudget"} 225 | _, err := client.Apply(ctx, []*spanner.Mutation{ 226 | spanner.Update("Albums", cols, []interface{}{1, 1, 100000}), 227 | spanner.Update("Albums", cols, []interface{}{2, 2, 500000}), 228 | }) 229 | return err 230 | } 231 | */ 232 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------