├── looper.so ├── looper.c ├── webscrape_hansard.py ├── valence-shifter.R ├── early-hansard-parser.py ├── codebook.csv ├── ConLLSetup.java ├── lexicon-join.py ├── remove-decorum-words.sh ├── README.md ├── lexicon-generator.R ├── millbank-scraper.py ├── CoNLLOutputter.java ├── emotion-final-y.csv ├── movie-classifier.py ├── modern-hansard-parser.py ├── emotion-final-q.csv └── emotion-main-models.R /looper.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lrheault/emotion/HEAD/looper.so -------------------------------------------------------------------------------- /looper.c: -------------------------------------------------------------------------------- 1 | /* looper.c */ 2 | 3 | /* 4 | Description: C program to create a valence-shifting variable until next punctuation mark. 5 | Usage: To be called from an R wrapper. 6 | @Author: L. Rheault 7 | */ 8 | 9 | 10 | #include 11 | #include 12 | 13 | void looper(double *x, char *y[], double *z, int *m){ 14 | int i; 15 | for (i = 1; i < *m; i++){ 16 | if (x[i-1]==1 && z[i]==z[i-1] && ispunct(y[i][0])==0){ 17 | x[i]=1; 18 | } 19 | } 20 | } 21 | 22 | -------------------------------------------------------------------------------- /webscrape_hansard.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup, SoupStrainer 2 | import urllib.request 3 | import pandas as pd 4 | 5 | URL = "https://www.theyworkforyou.com/pwdata/scrapedxml/debates/" 6 | 7 | response = urllib.request.urlopen(URL) 8 | soup = BeautifulSoup(response.read()) 9 | 10 | # Retrieve the url links to each debate file. 11 | all_urls = [] 12 | for link in soup.find_all('a', href=True): 13 | if '.xml' in link['href']: 14 | all_urls.append(URL + link['href']) 15 | 16 | # To save the original xml files. 17 | for u in all_urls: 18 | filename = u.split('/')[-1] 19 | urllib.request.urlretrieve(u, filename) 20 | 21 | # Reading the files and extracting fields of interest. 22 | final_dataset = [] 23 | for u in all_urls: 24 | filename = u.split('/')[-1] 25 | with open(filename, 'r') as f: 26 | xml = f.read() 27 | soup = BeautifulSoup(xml , 'lxml') 28 | for speech in soup.find_all('speech'): 29 | if speech.get('speakername') is not None: 30 | name = speech.get('speakername') 31 | speechid = speech.get('id') 32 | pid = speech.get('person_id') 33 | text = speech.find('p').get_text() 34 | final_dataset.append((speechid, name, pid, text)) 35 | 36 | # Saving the dataset in a csv format. 37 | df = pd.DataFrame(final_dataset, columns=['speech_id','name', 'person_id', 'text']) 38 | df.to_csv('uk_hansard.csv', sep='\t', index=False) 39 | -------------------------------------------------------------------------------- /valence-shifter.R: -------------------------------------------------------------------------------- 1 | #!/usr/bin/Rscript 2 | 3 | #======================================================================# 4 | # 5 | # Description: An R wrapper to: 6 | # 1) Create a valence shifting variable from a CoNLL corpus. 7 | # 2) Save a lemmatized corpus for the creation of a vector space model. 8 | # 9 | # Usage: 10 | # Rscript valence-shifter.R [input-dir] [path/looper.so] [formatted-corpus-output] 11 | # 12 | # where 13 | # [input-dir] is the directory containing the input CoNLL files 14 | # [path/looper.so] is the path to the shared object looper.so. 15 | # [formatted-corpus-output] is the directory and file name for the lemmatized corpus. 16 | # 17 | # Author: L. Rheault 18 | # 19 | #======================================================================# 20 | 21 | library(data.table) 22 | 23 | #---------------------------------------------------------------------------------------------------------------------------------------------------------------# 24 | 25 | args <- commandArgs(trailingOnly = TRUE) 26 | lofiles <- list.files(args[1],full.names=TRUE) 27 | n <- length(lofiles) 28 | 29 | # Loading C program 30 | dyn.load(args[2]) 31 | looper <- function(x,y,z,m) { 32 | newx <- .C("looper", x=as.double(x),y=as.character(y),z=as.double(z),m=as.integer(m)) 33 | newx[["x"]] 34 | } 35 | # List of negation words 36 | negw <- c("never","no","nothing","nowhere","none","not","cannot") 37 | q <- length(negw) 38 | 39 | for (i in 1:n){ 40 | temp <- fread(lofiles[i]) 41 | m <- nrow(temp) 42 | ### Add valence shifting variable: 43 | temp$valence <- 0 44 | for (j in 1:q){ 45 | temp$valence[temp$lemma==negw[j]] <- 1 46 | } 47 | temp$valence <- looper(temp$valence, temp$lemma, temp$s, m) 48 | write.table(temp,lofiles[i],row.names=FALSE,col.names=TRUE,quote=FALSE,append=FALSE) 49 | ### Appending sentences in lemmatized form: 50 | ls <- temp[,list(speech = paste(lemma,pos1,sep="_",collapse=" ")),by=l]$speech 51 | write(ls,args[3],append=TRUE) 52 | 53 | rm(temp) 54 | rm(ls) 55 | cat(format(Sys.time(), "%X"),"\n") 56 | cat("File",i,"of",n,"has been processed.","\n") 57 | } 58 | -------------------------------------------------------------------------------- /early-hansard-parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | #=====================================================================# 5 | # 6 | # Description: A script to parse the British Hansard early files as stored in XML format on parliament.co.uk 7 | # 8 | # Usage: 9 | # python early-hansard-parser.py [path-to-xml-file] [output-directory] 10 | # 11 | # Authors: L. Rheault, K. Beelen 12 | # 13 | #=====================================================================# 14 | 15 | import os, sys, re 16 | import lxml.etree as etree 17 | 18 | #-------------------------------------------------------------------------------------------------------------------------------------------------------------# 19 | 20 | xmlfile = str(sys.argv[1]) 21 | 22 | xmlp = etree.parse(xmlfile) 23 | hoc = xmlp.xpath('.//housecommons') 24 | vol = xmlfile[25:29] 25 | index = 0 26 | 27 | for h in hoc: 28 | for s in h.xpath('.//date'): 29 | date = s.attrib["format"] 30 | nameout = str(sys.argv[2])+"/"+vol+"_"+str(index)+"_"+date 31 | with open(nameout,"w") as g: 32 | for c in h.xpath('.//membercontribution'): 33 | inner_tags = c.xpath('.//col') 34 | for i in inner_tags: 35 | etree.strip_elements(c, i.tag, with_tail=False) 36 | text = etree.tostring(c,method="text",encoding="utf-8",pretty_print=True) 37 | text = text.replace("-\n",'') 38 | text = text.replace("\n",' ') 39 | # Removes long sequences of spacing left during parsing and encodes. 40 | text = re.sub('\s{2,}', ' ',text) 41 | # Removes long dash characters. 42 | text = re.sub('\xe2\x80\x94',' ',text) 43 | # Corrects for punctuation not followed by a space. 44 | text = re.sub('([.,:;!?()])', r'\1 ', text) 45 | # Removes empty brackets used for external links. 46 | text = re.sub('\[\]','',text) 47 | # Removes redundant spacing that may have been introduced in previous steps. 48 | text = re.sub('\s{2,}', ' ', text) 49 | if text.startswith(":") or text.startswith(",") or text.startswith(";"): 50 | text = text[1:] 51 | g.write(text+"\t"+date+"\n") 52 | g.close() 53 | index = index+1 54 | -------------------------------------------------------------------------------- /codebook.csv: -------------------------------------------------------------------------------- 1 | Yearly Dataset , 2 | Variable,Description 3 | year,Year 4 | polar,Emotional Polarity 5 | dpolar,First-Difference (Emotional Polarity) 6 | recession,Recession 7 | election,Election 8 | wars,Wars 9 | conservative,Party Variable (Conservative = 1) 10 | ldisp,Labor Disputes (Log) 11 | unemp,Unemployment 12 | misery,Misery Index 13 | gdpg,GDP Growh 14 | polarg,Emotional Polarity (Government) 15 | polaro,Emotional Polarity (Opposition) 16 | polarg_graph,Emotional Polarity (Government) – Original Scale 17 | polaro_graph,Emotional Polarity (Opposition)– Original Scale 18 | ww1,World War 1 – Binary Variable 19 | ww2,World War 2 – Binary Variable 20 | angloirish,Irish War of Independence – Binary Variable 21 | korean,Korean War – Binary Variable 22 | falkland,Falklands War – Binary Variable 23 | irak,Irak War – Binary Variable 24 | afghan,Afghanistan War – Binary Variable 25 | troubles,Troubles – Binary Variable 26 | troubles69,Troubles – 1969 Violence Peak – Binary Variable 27 | troubles72,Troubles – 1972 Violence Peak – Binary Variable 28 | (Note: Continuous Variables Normalized), 29 | , 30 | Quarterly Dataset, 31 | Variable,Description 32 | year,Year 33 | quarter,Quarter 34 | date,Year-Quarter Indicator 35 | polar,Emotional Polarity 36 | dpolar,First-Difference (Emotional Polarity) 37 | recession,Recession 38 | election,Election 39 | wars,Wars 40 | conservatives,Party Variable (Conservative = 1) 41 | polarg,Emotional Polarity (Government) 42 | polaro,Emotional Polarity (Opposition) 43 | ldisp,Labor Disputes (Log) 44 | unemp,Unemployment 45 | misery,Misery Index 46 | gdpg,GDP Growh 47 | ww1,World War 1 – Binary Variable 48 | ww2,World War 2 – Binary Variable 49 | angloirish,Irish War of Independence – Binary Variable 50 | korean,Korean War – Binary Variable 51 | falkland,Falklands War – Binary Variable 52 | irak,Irak War – Binary Variable 53 | afghan,Afghanistan War – Binary Variable 54 | troubles,Troubles – Binary Variable 55 | troubles69,Troubles – 1969 Violence Peak – Binary Variable 56 | troubles72,Troubles – 1972 Violence Peak – Binary Variable 57 | nrcpol,NRC Polarity Lexicon 58 | ofpol,OpinionFinder Polarity Lexicon 59 | swnpol,SentiWordNet Polarity Lexicon 60 | (Note: Continuous Variables Normalized), 61 | -------------------------------------------------------------------------------- /ConLLSetup.java: -------------------------------------------------------------------------------- 1 | import java.io.*; 2 | import java.util.*; 3 | import edu.stanford.nlp.io.*; 4 | import edu.stanford.nlp.ling.*; 5 | import edu.stanford.nlp.pipeline.*; 6 | import edu.stanford.nlp.trees.*; 7 | import edu.stanford.nlp.trees.TreeCoreAnnotations.*; 8 | import edu.stanford.nlp.util.*; 9 | 10 | /* 11 | 12 | A Java wrapper for Stanford CoreNLP producing a CoNLL output with line and sentence numbering. 13 | 14 | Requirements: 15 | - openjdk 1.8 (Java v.8) 16 | - Stanford CoreNLP (v.2015-04-20) recompiled with our customized CoNLLOutputter.java file. 17 | 18 | Description: 19 | Pre-processes a corpus of text in which meaningful sections of text (e.g. speeches/Tweets/reviews) 20 | appear on separate lines and are possibly contained across multiple files. 21 | 22 | Output format is a CoNLL (one-word-per-line) tab-separated file with columns: 23 | lineid sentenceid wordid word lemma POS 24 | 1 1 1 The the DT 25 | ... 26 | 27 | where: 28 | lineid -> the original line number (e.g. the speech number) 29 | sentenceid -> the sentence numbering within each lineid 30 | wordid -> the word numbering within each sentenceid. 31 | 32 | Example usage: 33 | java -cp "*:." -Xmx16g CoNLLSetup [batch-file] [input-dir] [output-dir] 34 | 35 | where: 36 | [batch-file] -> a file naming all files to be processed, for instance as obtained by: 37 | cd input-dir ; ls > batch-file 38 | [input-dir] -> directory containing original corpus files 39 | [output-dir] -> output directory for CoNLL files. 40 | 41 | Author: L. Rheault 42 | 43 | */ 44 | 45 | 46 | public class CoNLLSetup { 47 | public static void main(String[] args) throws IOException { 48 | 49 | // Set the desired properties for CoreNLP. 50 | Properties props = new Properties(); 51 | props.setProperty("annotators", 52 | "tokenize, ssplit, pos, lemma"); 53 | // Set multi-threading to 6 (modify if needed). 54 | props.put("nthreads", "6"); 55 | StanfordCoreNLP pipeline = new StanfordCoreNLP(props); 56 | 57 | Iterable documents = IOUtils.readLines(args[0]); 58 | 59 | for (String document : documents) { 60 | Iterable lines = IOUtils.readLines(args[1]+"/"+document); 61 | int lineNumber = 1; 62 | for (String line : lines) { 63 | Annotation annotation = new Annotation(line); 64 | pipeline.annotate(annotation); 65 | for (CoreMap sentence : annotation.get(CoreAnnotations.SentencesAnnotation.class)) { 66 | sentence.set(CoreAnnotations.LineNumberAnnotation.class,lineNumber); 67 | } 68 | lineNumber += 1; 69 | PrintWriter out = new PrintWriter(new FileOutputStream(new File(args[2]+"/"+document+"-output"),true)); 70 | pipeline.conllPrint(annotation, out); 71 | out.close(); 72 | } 73 | } 74 | } 75 | } 76 | 77 | -------------------------------------------------------------------------------- /lexicon-join.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | #======================================================================# 5 | # 6 | # Description: A Python script to perform fast join operations on CoNLL formatted corpora. 7 | # In this case, joins a polarity lexicon by lemma/PoS pairs and computes aggregate polarity score. 8 | # 9 | # Usage: 10 | # python lexicon-join.py [input directory] [lexicon] 11 | # 12 | # Author: L. Rheault 13 | # 14 | #======================================================================# 15 | 16 | from __future__ import division 17 | import os, sys, re, time 18 | import pandas as pd 19 | from time import time 20 | import csv 21 | 22 | #---------------------------------------------------------------------------------------------------------------------------------------------------------------# 23 | 24 | filelist = sorted(os.listdir(sys.argv[1])) 25 | inpath = str(sys.argv[1]) 26 | 27 | lex1 = pd.read_table(str(sys.argv[2]),delimiter=',',dtype=object,header=0,encoding="utf-8") 28 | 29 | # To also join the general-purpose lexicons, uncomment the following lines: 30 | # --for nrc: 31 | #lex2 = pd.read_table(str(sys.argv[3]),delimiter='\t',dtype=object,header=0) 32 | # --for of: 33 | #lex3 = pd.read_table(str(sys.argv[4]),delimiter='\t',dtype=object,header=0) 34 | # --for swn: 35 | #lex4 = pd.read_table(str(sys.argv[5]),delimiter='\t',dtype=object,header=0) 36 | 37 | with open("emotion-dataset.csv","w") as g: 38 | writer = csv.writer(g) 39 | writer.writerow( ("date","polar","wc","sumpolar","lexcount") ) 40 | for f in filelist: 41 | t0 = time() 42 | temp = pd.read_table(inpath+'/'+f, delimiter='\t', dtype=object, header=0, encoding="utf-8") 43 | 44 | # Performs fast SQL-type join. 45 | temp = pd.merge(temp,lex1,how='left',on=['lemma','pos1']) 46 | #temp = pd.merge(temp,lex2,how='left',on=['lemma']) 47 | #temp = pd.merge(temp,lex3,how='left',on=['lemma','pos1']) 48 | #temp = pd.merge(temp,lex4,how='left',on=['lemma','pos1']) 49 | 50 | temp[['l','s','w','valence','polarity']] = temp[['l','s','w','valence','polarity']].apply(pd.to_numeric) 51 | temp.sort_values(['l','s','w'],ascending=[True, True, True],inplace=True) 52 | 53 | # Adjusting for valence. (1-temp.valence) is denoted \theta in the text. 54 | temp['polarity_val'] = (1 - temp.valence)*temp.polarity 55 | 56 | # Numerator and denominator from equation [3]. 57 | sumpolar = temp.polarity_val.sum(skipna=True) 58 | lexcount = temp.polarity.count() 59 | 60 | # The polarity measure y_t. 61 | polar = sumpolar/lexcount 62 | # A general word count. 63 | wc = temp.shape[0] 64 | 65 | # Writes the data by quarter. Polar is the quarterly variable y_t. 66 | # Variables sumpolar and lexcount can be summed by year to produce the yearly y_t. 67 | writer.writerow( (f[:6], polar, wc, sumpolar, lexcount) ) 68 | 69 | # Updates CoNLL file with joined polarity scores: 70 | temp.to_csv(inpath+'/'+f, sep='\t', index=False, encoding="utf-8") 71 | 72 | print "File %s processed in %0.3fs." % (f, (time()-t0) ) 73 | -------------------------------------------------------------------------------- /remove-decorum-words.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #============================================================================# 4 | # 5 | # Description: Perl-based Bash script to replace instances of formal addresses enforced by protocol in the British House of Commons. 6 | # 7 | # Usage: 8 | # ./remove-decorum-words.sh [directory] 9 | # 10 | # Detailed Description: 11 | # ---Parts 1 to 3 replace all instances of "Right Honourable X", "Honourable X", "Noble Lord" and other variants, accounting 12 | # for changes in reporting practices and abbreviations in the Hansards over the past hundred years. 13 | # ---Part 4 replaces orphan uses of capitalized (only) instances of Honourable or Right Honourable. 14 | # ---Part 5 replaces addresses to the Monarch. 15 | # ---Part 6 replaces references to the numerous "Lord High"-type positions, to reduce ambiguities with the adjective "high". 16 | # ---Part 7 replaces references to the Speech of the Throne, again to reduce ambiguities with adjectives. 17 | # ---Part 8 replaces references to the Prime Minister, to reduce ambiguities with other uses of the adjective "prime". 18 | # All addresses are replaced by the string FormalTitle. 19 | # 20 | # Authors: L. Rheault and K. Beelen 21 | # 22 | #============================================================================# 23 | 24 | DIRECTORY=$1/* 25 | 26 | #-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------# 27 | 28 | perl -i -pe 's/(?:\srt\s|\srt.\s|\sright\s){1}(?:hon\s|hon.\s|honourable\s){1}/ Right Honourable /gi' $DIRECTORY 29 | echo "Part 1 completed: Normalizing the spelling of Right Honourable." 30 | 31 | perl -i -pe 's/(?:\shon\s|\shon.\s){1}/ Honourable /gi' $DIRECTORY 32 | echo "Part 2 completed: Normalizing the spelling of Honourable." 33 | 34 | perl -i -pe 's/(?:(?:right\s){0,1}(?:honourable\s|noble\s){1}(?:(?:and\s)|(?:lord\s)|(?:honourable\s)|(?:learned\s)|(?:gallant\s)|(?:noble\s)|(?:respected\s)|(?:reverend\s)){0,4}(?:the\s){0,1}(?:(?:lord(?:s){0,1})|(?:justice(?:s){0,1}\b)|(?:gentle\sm[ea]n(?:t){0,1})|(?:gentlem[ea]n(?:t){0,1})|(?:friend(?:s){0,1}\b)|(?:member(?:s){0,1}\b)|(?:lad(?:y|(?:ies)){1})){1})/FormalTitle/gi' $DIRECTORY 35 | echo "Part 3 completed: Substituting main formal addresses." 36 | 37 | perl -i -pe 's/Right Honourable/FormalTitle/g' $DIRECTORY 38 | perl -i -pe 's/Honourable/FormalTitle/g' $DIRECTORY 39 | echo "Part 4 completed: Substituting orphan instances of Honourable." 40 | 41 | perl -i -pe 's/(?:his\s|her\s|your\s){1}(?:majesty|excellency|highness){1}(?:\sthe\sKing|\sthe\sQueen){0,1}/FormalTitle/gi' $DIRECTORY 42 | echo "Part 5 completed: Substituting formal addresses to the Queen/King." 43 | 44 | perl -i -pe 's/(?:lord\s){1}(?:high\s){1}(?:commissioner|admiral|chamberlain|chancellor|steward|treasurer){1}/FormalTitle/gi' $DIRECTORY 45 | perl -i -pe 's/(?:lord\s){1}(?:high){1}/FormalTitle/gi' $DIRECTORY 46 | echo "Part 6 completed: Substituting formal addresses to Lord High positions." 47 | 48 | perl -i -pe "s/(?:his\s|her\s){1}(?:majesty's most gracious speech){1}/FormalTitle/gi" $DIRECTORY 49 | perl -i -pe 's/Gracious Address/FormalTitle/gi' $DIRECTORY 50 | perl -i -pe 's/Gracious Speech/FormalTitle/gi' $DIRECTORY 51 | echo "Part 7 completed: Substituting references to the Gracious Address." 52 | 53 | perl -i -pe 's/\bprime\sminister\b/FormalTitle/gi' $DIRECTORY 54 | echo "Part 8 completed: Substituting references to Prime Minister." 55 | 56 | perl -i -pe "s/(?:FormalTitle(?:s(?:'){0,1}|t|l|a){1}){1}/FormalTitle/g" $DIRECTORY 57 | echo "Part 9 completed: Normalizing usage of FormalTitle placeholder." 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

Measuring Emotion in Parliamentary Debates with Automated Textual Analysis

2 | 3 |

Corpus Information

4 | 5 | Since the publication of this study, new resources have become available to access the British Hansard in digitized format. I would recommend using the data from the They Work for You project, which seems to be the most up-to-date. 6 | 7 | I've included a script in this repo (webscrape_hansard.py) showing how to fetch the data and save into a csv file. Feel free to edit and retrieve elements you want to access. The "They Work for You" dataset covers the period from 1919 to now. 8 | 9 | Historical files are available here: http://www.hansard-archive.parliament.uk/ 10 | 11 |

Supporting Scripts and Lexicon

12 | 13 | This page contains scripts, data files as well as the final lexicon used for a study of emotional polarity in the British House of Commons. The file lexicon-polarity.csv is an emotional polarity lexicon trained on the entire British Hansard for the period 1909-2013. It can be used as an off-the-shelf lexicon for studying sentiment in political texts. It has the benefit of being adapted to the domain under study and is robust to the evolution of language during the past century. 14 | 15 | Details regarding the methodology appear in the text. The raw corpus of parliamentary debates can be accessed here: http://search.politicalmashup.nl/ (link now dead). 16 | 17 | If using these materials, please cite the study as follows (click this link to access the full study): 18 | 19 | Rheault, Ludovic, Kaspar Beelen, Christopher Cochrane and Graeme Hirst. 2016. "Measuring Emotion in Parliamentary Debates with Automated Textual Analysis". PLoS ONE 11(12): e0168843. 20 | 21 | 
22 | @Article{RHE16,
23 |   Title = {{M}easuring {E}motion in {P}arliamentary {D}ebates with {A}utomated {T}extual {A}nalysis},
24 |   Author = {Ludovic Rheault and Kaspar Beelen and Christopher Cochrane and Graeme Hirst},
25 |   Journal = {PLoS ONE},
26 |   Year = {2016},
27 |   Volume = {11},
28 |   Number = {12},
29 |   Pages = {e0168843},
30 | }
31 |
32 | 33 | A previous version of the same manuscript was circulated online and presented in 2015 at the CPSA conference: Rheault, Ludovic, Kaspar Beelen, Christopher Cochrane and Graeme Hirst. 2015. "Measuring Emotion in Political Debates Using Natural Language Processing". Presented at the 87th Annual Conference of the Canadian Political Science Association, Ottawa, ON, Canada, June 2-4, 2015 (http://www.cpsaevents.ca/2015/CPSA_APSC_2015PROGRAMME.pdf). Use the link mentioned above for the most recent version. 34 | 35 | The following list describes the purpose of each script and data file. 36 | 37 |

Scripts

38 | 39 | early-hansard-parser.py - Python 2.7 - A script to parse XML files of the early Hansard volumes from the UK Parliament. 40 | 41 | millbank-scraper.py - Python 2.7 - A script to scrape the Millbank Systems website and retrieve Hansard volumes missing from the UK Parliament archives. 42 | 43 | modern-hansard-parser.py - Python 2.7 - A script to parse XML files of the modern Hansard (post 1936), in the Political Mashup format. 44 | 45 | CoNLLSetup.java - Java 8 - A custom class to use the Stanford CoreNLP library (requires CoNLLOutputter.java). 46 | 47 | remove-decorum-words.sh - Bash 4.3 - A Perl-based Shell script to remove expressions required by the decorum of the House (e.g. "The Right Honourable"). 48 | 49 | valence-shifter.R, looper.so - C, R 3.2 - An R wrapper to add a valence-shifting variable to the CoNLL corpus, using C for speed. 50 | 51 | lexicon-generator.R - R 3.2 - An R script to generate domain-specific lexicons based on the word vectors obtained using the Glove program. 52 | 53 | lexicon-join.py - Python 2.7 - A script to perform fast SQL-type join operations on the corpus and compute polarity scores by quarter and year. 54 | 55 | movie-classifier.py - Python 2.7 - A script to assess the accuracy of machine learning models based on the movie reviews dataset. 56 | 57 | emotion-main-models.R - R 3.2 - An R script to compute graphs and empirical models. 58 | 59 |

Datasets

60 | 61 | emotion-final-y.csv - Final dataset (yearly, normalized variables). 62 | 63 | emotion-final-q.csv - Final dataset (quarterly, normalized variables). 64 | 65 | codebook.csv - Description of variables in yearly and quarterly datasets. 66 | 67 | lexicon-polarity.csv - The domain-specific polarity lexicon (4200 words). 68 | -------------------------------------------------------------------------------- /lexicon-generator.R: -------------------------------------------------------------------------------- 1 | #!/usr/bin/Rscript 2 | 3 | #======================================================================# 4 | # 5 | # Description: An R script to create a polarity lexicon based on a vector space model and seed words. 6 | # 7 | # Usage: 8 | # Rscript lexicon-generator.R [vectorfile] [path-to-aux-files] [lower-bound] [lexicon-size] 9 | # 10 | # where 11 | # [vectorfile] = path and name of vector file 12 | # [path-to-aux-files] = the directory containing all other input files and dependencies, also the output directory. 13 | # [lower-bound] = only consider words occurring more than this bound (0 to use all words) 14 | # [lexicon-size] = how many words should the lexicon contain (e.g. 1000) 15 | # 16 | # Dependencies: 17 | # ---Vectors from vector space model; 18 | # ---Vocabulary file ("vocab-merged.txt"); 19 | # ---nationalities auxiliary file ("nationalities-unique.csv"); 20 | # ---seed files (positive and negative, "pos-100-final" and "neg-100-final"). 21 | # 22 | # Output: A positive and a negative lexicon. 23 | # 24 | # Author: L. Rheault 25 | # 26 | #======================================================================# 27 | 28 | library(data.table) 29 | 30 | #---------------------------------------------------------------------------------------------------------------------------------------------------------------# 31 | 32 | argv=commandArgs(TRUE) 33 | vectorfile <- argv[1] 34 | inpath <- argv[2] 35 | hh <- as.numeric(argv[3]) 36 | lexsize <- as.numeric(argv[4]) 37 | 38 | #---------------------------------------------------------------------------------------------------------------------------------------------------------------# 39 | # Loading and Cleaning 40 | #---------------------------------------------------------------------------------------------------------------------------------------------------------------# 41 | 42 | # loading vector space model (arg 1): 43 | vin <- fread(vectorfile) 44 | vecdim <- ncol(vin)-1 45 | # vocabulary file (arg 2): 46 | dd <- fread(paste(inpath,"/vocab-merged.txt",sep="")) 47 | ndd <- nrow(dd) 48 | # keeping only words with (arg 3) or more occurrences: 49 | x <- merge(vin,dd,by='V1') 50 | x <- x[x$V2.y>=hh,] 51 | 52 | # removing nationalities: 53 | nat <- fread(paste(inpath,"/nationalities-unique.csv",sep="")) 54 | nnat <- nrow(nat) 55 | for (jj in 1:nnat){x <- x[grepl(nat[jj],x$V1)==FALSE,]} 56 | # removing proper nouns and digits: 57 | x <- x[grepl('[[:upper:]]',x$V1,perl=T)==FALSE,] 58 | x <- x[grepl('[[:digit:]]',x$V1,perl=T)==FALSE,] 59 | # keeping only adj, adv, nouns, verbs, and excalamations: 60 | x <- x[grepl('_n|_a|_r|_v|_u',x$V1)==TRUE,] 61 | # removing incorrectly pos-tagged 'formaltitle' instances: 62 | x <- x[grepl('formaltitle',x$V1)==FALSE,] 63 | 64 | # creating matrix and preserving dictionary: 65 | y <- x[,2:vecdim+1,with=FALSE] 66 | y <- as.matrix(y) 67 | dictionary <- x[,1,with=FALSE] 68 | rm(x) 69 | rm(vin) 70 | 71 | # loading seeds: 72 | lexicon.pos <- readLines(paste(inpath,"/pos-100-final",sep="")) 73 | lexicon.neg <- readLines(paste(inpath,"/neg-100-final",sep="")) 74 | m <- length(lexicon.pos) 75 | pos.index <- vector(length=m) 76 | neg.index <- vector(length=m) 77 | for (i in 1:m){ 78 | pos.index[i] <- which(dictionary$V1==lexicon.pos[i]) 79 | neg.index[i] <- which(dictionary$V1==lexicon.neg[i]) 80 | } 81 | pos.matrix <- y[pos.index,] 82 | neg.matrix <- y[neg.index,] 83 | # removing seeds from main lexicon: 84 | seeds <- c(pos.index,neg.index) 85 | y.noseed <- y[-seeds,] 86 | dictionary.noseed <- dictionary[-seeds] 87 | 88 | cat("Pre-processing stage successful.","\n") 89 | cat(format(Sys.time(), "%X"),"\n") 90 | 91 | #---------------------------------------------------------------------------------------------------------------------------------------------------------------# 92 | # Creating Lexicon 93 | #---------------------------------------------------------------------------------------------------------------------------------------------------------------# 94 | 95 | # computing sentiment orientation: 96 | q <- nrow(y.noseed) 97 | vecpos <- vector(length=m) 98 | vecneg <- vector(length=m) 99 | results <- vector(length=q) 100 | sumvecpos <- "" 101 | sumvecneg <- "" 102 | cat("Computing the lexicons...","\n") 103 | for (i in 1:q){ 104 | for (j in 1:m){ 105 | vecpos[j] <- sum(y.noseed[i,]*pos.matrix[j,])/sqrt(sum(y.noseed[i,]^2)*sum(pos.matrix[j,]^2)) 106 | vecneg[j] <- sum(y.noseed[i,]*neg.matrix[j,])/sqrt(sum(y.noseed[i,]^2)*sum(neg.matrix[j,]^2)) 107 | } 108 | sumvecpos <- sum(vecpos) 109 | sumvecneg <- sum(vecneg) 110 | results[i] <- sumvecpos - sumvecneg 111 | 112 | if (i %% 1000 == 0) { 113 | prog <- round(i/q*100, digits = 2) 114 | cat("Progress:",prog,"%.","\n") 115 | cat(format(Sys.time(), "%X"),"\n") 116 | } 117 | } 118 | res <- cbind(dictionary.noseed,results) 119 | # rescaling between -1 and 1: 120 | rescale <- function(x) {2/(max(x) - min(x))*(x - min(x)) - 1} 121 | res$results <- rescale(res$results) 122 | 123 | # creating positive and negative lexicons: 124 | posres <- res[order(-results),] 125 | negres <- res[order(results),] 126 | exp.lex.pos <- posres[1:lexsize,] 127 | exp.lex.neg <- negres[1:lexsize,] 128 | write.table(exp.lex.pos,paste(inpath,"/positive-lexicon-",vecdim,"d-",lexsize,sep=""),sep=",",row.names=F,col.names=F) 129 | write.table(exp.lex.neg,paste(inpath,"/negative-lexicon-",vecdim,"d-",lexsize,sep=""),sep=",",row.names=F,col.names=F) 130 | # saving full lexicon (comment in to activate): 131 | # res <- res[order(results),] 132 | # write.table(res,paste(inpath,"/full-lexicon",vecdim,"d-",lexsize,sep=""),sep=",",row.names=F,col.names=F) 133 | 134 | cat("Process complete and successful.","\n") 135 | cat(format(Sys.time(), "%X"),"\n") 136 | -------------------------------------------------------------------------------- /millbank-scraper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | #======================================================================# 5 | # 6 | # Description: A script to scrape and parse volumes from the UK Hansards of the 5th Series, as stored 7 | # on the Millbank Systems website: http://hansard.millbanksystems.com 8 | # Requirement: Beautiful Soup v 4. 9 | # 10 | # Usage: 11 | # python millbank-scraper.py [path-to-volnumber-file] [path-to-tempdir]/ [path-to-outdir]/ 12 | # 13 | # [path-to-volnumber-file] points to a file indicating the volume numbers to scrape and parse, i.e. a file 14 | # with content arranged like so: 15 | # 2 16 | # 4 17 | # 184 18 | # ... 19 | # where the numbers on each line indicate the desired volumes. 20 | # [path-of-tempdir] points to a directory where temporary/archive files are saved. 21 | # [path-of-outdir] points to a directory where the parsed debates are saved. 22 | # 23 | # Author: L. Rheault 24 | # 25 | #======================================================================# 26 | 27 | import os, sys, re 28 | import urllib 29 | import urllib2 30 | from bs4 import BeautifulSoup, Tag 31 | 32 | #---------------------------------------------------------------------------------------------------------------------------------------------------------------# 33 | 34 | # The following function is taken from: http://www.peterbe.com/plog/uniqifiers-benchmark 35 | def unique(seq): 36 | seen = set() 37 | return [x for x in seq if x not in seen and not seen.add(x)] 38 | 39 | # This function retrieves the volume index from Millbank Systems, stores it, and 40 | # parses it to retrieve unique urls containing actual debates, filtering out 41 | # written answers to questions. 42 | 43 | def parseVolumeIndex (volume, path): 44 | # Downloads volume index: 45 | urlname = 'http://hansard.millbanksystems.com/volumes/5C/'+str(volume) 46 | text1 = urllib2.urlopen(urlname) 47 | 48 | # Parses the volume index with BeautifulSoup and returns date urls: 49 | soup = BeautifulSoup(text1, 'lxml') 50 | hlist=[] 51 | for a in soup.find_all('a'): 52 | if a['href'].startswith('/commons/'): 53 | hlink = 'http://hansard.millbanksystems.com'+a['href'] 54 | shortlink = hlink[:55] 55 | hlist.append(shortlink) 56 | # Removes duplicates and preserves as an ordered set: 57 | hlist = unique(hlist) 58 | 59 | # Downloads each date index within volume: 60 | jlist = [] 61 | for dates in hlist: 62 | text2 = urllib2.urlopen(dates) 63 | soup2 = BeautifulSoup(text2, 'lxml') 64 | # Parses each date index for specific URLs: 65 | for a in soup2.find_all('a'): 66 | if a['href'].startswith('/commons/'): 67 | jlist.append('http://hansard.millbanksystems.com'+a['href']) 68 | # Removes duplicates (if any) and preserves as an ordered set: 69 | jlist = unique(jlist) 70 | 71 | # Saves the parsed url index file in the temp folder for reference, 72 | # and returns the list for future usage. 73 | namevolindexparsed = str(path)+str(volume) 74 | with open(namevolindexparsed,'w') as f: 75 | for item in jlist: 76 | f.write('%s\n' % item) 77 | return jlist 78 | 79 | # The following function downloads each section of the volume, parses them and 80 | # appends them as text (one speech per line) in an output file for each day, named 81 | # after that day, and saved in the output folder. 82 | 83 | def scrapeAndParseVolume (outpath,jlist): 84 | # Creates a date index for days: 85 | dd=[] 86 | for f in jlist: 87 | dd.append(f[43:54].replace('/','')) 88 | dateset = unique(dd) 89 | 90 | # Parses files for each day: 91 | for d in dateset: 92 | finname = str(outpath)+str(d) 93 | with open(finname,'a') as h: 94 | for g in jlist: 95 | if g[43:54].replace('/','')==d: 96 | tempfile = urllib2.urlopen(g) 97 | text = tempfile.read().replace('\n','') 98 | soup = BeautifulSoup(text,'lxml') 99 | # Finds all div tags with class member_contribution: 100 | scriptTags = soup.find_all(class_='hentry member_contribution') 101 | for script in scriptTags: 102 | # Retrieves the Millbank Speech ID. 103 | if script.has_attr('id'): 104 | speechid = script['id'] 105 | else: 106 | speechid = '__NoID__' 107 | # Retrieves name of speaker: 108 | name = '' 109 | try: 110 | citetag = script.find('cite',{'class':True}) 111 | if 'unmatched-member' not in citetag['class']: 112 | name = script.find('a',{'href' : re.compile('/people/.*')})['title'] 113 | elif 'unmatched-member' in citetag['class']: 114 | subscript = script.find('cite') 115 | name = subscript.get_text().encode('UTF-8') 116 | else: 117 | name = '__NoName__' 118 | except: 119 | name = '__MalformedData__' 120 | for tag in script.find_all(['cite','span','a']): 121 | tag.replaceWith('') 122 | # Removes long sequences of spacing left during parsing and encodes. 123 | s = re.sub('\s{2,}',' ',script.get_text().encode('UTF-8')) 124 | # Removes long dash characters. 125 | s = re.sub('\xe2\x80\x94',' ',s) 126 | # Corrects for punctuation not followed by a space. 127 | s = re.sub('([.,:;!?()])', r'\1 ', s) 128 | # Removes empty brackets used for external links. 129 | s = re.sub('\[\]','',s) 130 | # Again removes redundant spacing that may have been introduced in previous steps. 131 | s = re.sub('\s{2,}', ' ', s) 132 | # Appends the speech to the current file. 133 | h.write(s+'\t'+speechid+'\t'+name+'\t'+str(d)+'\n') 134 | 135 | if __name__ == '__main__': 136 | # Reads the desired list of volumes as stored in the file specified by argv[1]. 137 | # This could be replaced by a sequence, for instance. 138 | with open(str(sys.argv[1]),'r') as x: 139 | volumefile = x.read().splitlines() 140 | # Processes each volume and echoes progress to the shell. 141 | for v in volumefile: 142 | jlist = parseVolumeIndex(v, str(sys.argv[2])) 143 | print('Index for Volume '+str(v)+' has been retrieved.') 144 | scrapeAndParseVolume(str(sys.argv[3]),jlist) 145 | print('Volume '+str(v)+' has been processed.') 146 | -------------------------------------------------------------------------------- /CoNLLOutputter.java: -------------------------------------------------------------------------------- 1 | package edu.stanford.nlp.pipeline; 2 | 3 | import edu.stanford.nlp.ling.CoreAnnotations; 4 | import edu.stanford.nlp.ling.CoreLabel; 5 | import edu.stanford.nlp.ling.IndexedWord; 6 | import edu.stanford.nlp.semgraph.SemanticGraph; 7 | import edu.stanford.nlp.semgraph.SemanticGraphCoreAnnotations; 8 | import edu.stanford.nlp.semgraph.SemanticGraphEdge; 9 | import edu.stanford.nlp.util.CoreMap; 10 | import edu.stanford.nlp.util.StringUtils; 11 | 12 | import java.io.IOException; 13 | import java.io.OutputStream; 14 | import java.io.PrintWriter; 15 | import java.util.ArrayList; 16 | import java.util.List; 17 | import java.util.Set; 18 | import java.util.stream.Collectors; 19 | 20 | /** 21 | *

Write a subset of our CoreNLP output in CoNLL format.

22 | * 23 | *

The fields currently output are:

24 | * 25 | * 26 | * 27 | * 28 | * 29 | * 30 | * 31 | * 32 | * 33 | * 34 | * 35 | * 36 | * 37 | * 38 | * 39 | * 40 | * 41 | * 42 | * 43 | * 44 | * 45 | * 46 | * 47 | * 48 | * 49 | * 50 | * 51 | * 52 | * 53 | * 54 | * 55 | * 56 | * 57 | * 58 | * 59 | * 61 | * 62 | * 63 | * 64 | * 65 | * 66 | * 67 | *
Field NumberField NameDescription
1IDToken Counter, starting at 1 for each new sentence.
2FORMWord form or punctuation symbol.
3LEMMALemma of word form, or an underscore if not available.
4POSTAGFine-grained part-of-speech tag, or underscore if not available.
5NERNamed Entity tag, or underscore if not available.
6HEADHead of the current token, which is either a value of ID or zero ('0'). 60 | * This is underscore if not available.
7DEPRELDependency relation to the HEAD, or underscore if not available.
68 | * 69 | * @author Gabor Angeli 70 | */ 71 | public class CoNLLOutputter extends AnnotationOutputter { 72 | 73 | private static final String NULL_PLACEHOLDER = "_"; 74 | 75 | public CoNLLOutputter() { } 76 | 77 | private static String orNull(String in) { 78 | if (in == null) { 79 | return NULL_PLACEHOLDER; 80 | } else { 81 | return in; 82 | } 83 | } 84 | 85 | /** 86 | * Produce a line of the CoNLL output. 87 | */ 88 | private static String line(int linenumber, int sentCount, int index, 89 | CoreLabel token, 90 | int head, String deprel) { 91 | ArrayList fields = new ArrayList<>(16); 92 | 93 | fields.add(Integer.toString(linenumber)); // 0 94 | fields.add(Integer.toString(sentCount)); // 1 95 | fields.add(Integer.toString(index)); // 2 96 | fields.add(orNull(token.word())); // 3 97 | fields.add(orNull(token.lemma())); // 4 98 | fields.add(orNull(token.tag())); // 5 99 | // fields.add(orNull(token.ner())); // 6 100 | // if (head >= 0) { 101 | // fields.add(Integer.toString(head)); // 7 102 | // fields.add(deprel); // 8 103 | // } else { 104 | // fields.add(NULL_PLACEHOLDER); 105 | // fields.add(NULL_PLACEHOLDER); 106 | // } 107 | 108 | return StringUtils.join(fields, "\t"); 109 | } 110 | 111 | @Override 112 | public void print(Annotation doc, OutputStream target, Options options) throws IOException { 113 | PrintWriter writer = new PrintWriter(target); 114 | 115 | // vv A bunch of nonsense to get tokens vv 116 | if (doc.get(CoreAnnotations.SentencesAnnotation.class) != null) { 117 | int sentCount = 1; 118 | for (CoreMap sentence : doc.get(CoreAnnotations.SentencesAnnotation.class)) { 119 | 120 | if (sentence.get(CoreAnnotations.TokensAnnotation.class) != null) { 121 | 122 | List tokens = sentence.get(CoreAnnotations.TokensAnnotation.class); 123 | Integer linenumber = sentence.get(CoreAnnotations.LineNumberAnnotation.class); 124 | // Integer sentnumber = sentence.get(CoreAnnotations.TokensAnnotation.class); 125 | SemanticGraph depTree = sentence.get(SemanticGraphCoreAnnotations.BasicDependenciesAnnotation.class); 126 | for (int i = 0; i < tokens.size(); ++i) { 127 | // ^^ end nonsense to get tokens ^^ 128 | 129 | // Newline if applicable 130 | if (i > 0) { 131 | writer.println(); 132 | } 133 | 134 | // Try to get the incoming dependency edge 135 | int head = -1; 136 | String deprel = null; 137 | if (depTree != null) { 138 | Set rootSet = depTree.getRoots().stream().map(IndexedWord::index).collect(Collectors.toSet()); 139 | IndexedWord node = depTree.getNodeByIndexSafe(i + 1); 140 | if (node != null) { 141 | List edgeList = depTree.getIncomingEdgesSorted(node); 142 | if (!edgeList.isEmpty()) { 143 | assert edgeList.size() == 1; 144 | head = edgeList.get(0).getGovernor().index(); 145 | deprel = edgeList.get(0).getRelation().toString(); 146 | } else if (rootSet.contains(i + 1)) { 147 | head = 0; 148 | deprel = "ROOT"; 149 | } 150 | } 151 | } 152 | 153 | // Write the token 154 | writer.print(line(linenumber, sentCount, i + 1, tokens.get(i), head, deprel)); 155 | 156 | } 157 | sentCount++; 158 | } 159 | 160 | writer.println(); 161 | // I believe this is the redundant extra line. 162 | // writer.println(); 163 | } 164 | 165 | } 166 | writer.flush(); 167 | } 168 | 169 | public static void conllPrint(Annotation annotation, OutputStream os) throws IOException { 170 | new CoNLLOutputter().print(annotation, os); 171 | } 172 | 173 | public static void conllPrint(Annotation annotation, OutputStream os, StanfordCoreNLP pipeline) throws IOException { 174 | new CoNLLOutputter().print(annotation, os, pipeline); 175 | } 176 | 177 | public static void conllPrint(Annotation annotation, OutputStream os, Options options) throws IOException { 178 | new CoNLLOutputter().print(annotation, os, options); 179 | } 180 | 181 | } 182 | -------------------------------------------------------------------------------- /emotion-final-y.csv: -------------------------------------------------------------------------------- 1 | year,polar,dpolar,recession,election,wars,conservative,ldisp,unemp,misery,gdpg,polarg,polaro,polarg_graph,polaro_graph,ww1,ww2,angloirish,korean,falkland,irak,afghan,troubles,troubles69,troubles72 2 | 1909,-1.3562726,,0,0,0,,-0.048297729,0.72008389,0.65293741,0.066390671,,,,,0,0,0,0,0,0,0,0,0,0 3 | 1910,-0.76349598,0.5927766,0,1,0,,0.93260974,-0.067082278,0.25106496,0.15190509,,,,,0,0,0,0,0,0,0,0,0,0 4 | 1911,-0.60709888,0.1563971,0,0,0,,0.95430535,-0.51105487,-0.15586248,0.36441478,,,,,0,0,0,0,0,0,0,0,0,0 5 | 1912,-0.91513097,-0.3080321,0,0,0,,2.0047019,-0.4266184,0.27633995,-0.21663652,,,,,0,0,0,0,0,0,0,0,0,0 6 | 1913,-0.95168549,-0.036554515,0,0,0,,0.92777944,-0.74529815,-0.38080996,0.62606335,,,,,0,0,0,0,0,0,0,0,0,0 7 | 1914,-1.401347,-0.44966155,0,0,1,,0.93344992,-0.45385596,-0.10783999,-0.052946612,,,,,1,0,0,0,0,0,0,0,0,0 8 | 1915,-0.69006854,0.7112785,0,0,1,,0.022886239,-1.0476353,-0.96466237,1.063527,,,,,1,0,0,0,0,0,0,0,0,0 9 | 1916,-1.1606189,-0.47055036,0,0,1,,-0.11916122,-1.2137846,-1.1643349,-0.32927305,,,,,1,0,0,0,0,0,0,0,0,0 10 | 1917,-1.0405366,0.12008226,1,0,1,,0.51176959,-1.1756519,-0.9191674,-0.89788008,,,,,1,0,0,0,0,0,0,0,0,0 11 | 1918,-0.70209372,0.33844292,0,1,1,,0.54161799,-1.1266243,-0.86356241,-0.080387801,,,,,1,0,0,0,0,0,0,0,0,0 12 | 1919,-0.59269869,0.10939503,1,0,1,,1.8867433,-0.48926485,0.17776747,-3.186029,,,,,0,0,1,0,0,0,0,0,0,0 13 | 1920,-1.3342326,-0.74153388,1,0,1,,1.6795518,-0.77798325,-0.15838999,-2.5650921,,,,,0,0,1,0,0,0,0,0,0,0 14 | 1921,-1.6016268,-0.26739419,1,0,1,,2.5642211,1.7414933,2.4752645,-3.7868633,,,,,0,0,1,0,0,0,0,0,0,0 15 | 1922,-1.2222549,0.37937188,0,1,0,,1.4597298,1.3710622,1.7599822,1.0057729,,,,,0,0,0,0,0,0,0,0,0,0 16 | 1923,-1.0468794,0.17537546,0,1,0,,0.99175179,0.90529948,1.1710749,0.25720268,,,,,0,0,0,0,0,0,0,0,0,0 17 | 1924,-0.32752031,0.7193591,0,1,0,,0.81337857,0.6656087,0.89052236,0.81783265,,,,,0,0,0,0,0,0,0,0,0,0 18 | 1925,-0.71008867,-0.38256836,0,0,0,,0.76989633,0.84265304,1.0396448,0.4346132,,,,,0,0,0,0,0,0,0,0,0,0 19 | 1926,-1.0394986,-0.3294099,1,0,0,,3.0439596,1.0932388,2.0329523,-1.6837831,,,,,0,0,0,0,0,0,0,0,0,0 20 | 1927,-1.6631844,-0.62368584,0,0,0,,-0.67269671,0.55393463,0.28897747,1.765511,,,,,0,0,0,0,0,0,0,0,0,0 21 | 1928,-0.72608489,0.93709952,0,0,0,,-0.54642528,0.75004524,0.51392496,-0.40330043,,,,,0,0,0,0,0,0,0,0,0,0 22 | 1929,-0.50839865,0.21768624,0,1,0,,0.80101383,0.68195128,0.90315986,0.22210349,,,,,0,0,0,0,0,0,0,0,0,0 23 | 1930,-0.76200819,-0.25360954,1,0,0,,0.32343593,1.7632834,1.7448173,-0.94223273,,,,,0,0,0,0,0,0,0,0,0,0 24 | 1931,-0.99877554,-0.23676735,1,1,0,,0.67190528,2.8037591,2.8265872,-2.1684711,,,,,0,0,0,0,0,0,0,0,0,0 25 | 1932,-0.89068699,0.10808855,1,0,0,,0.61646086,2.9426708,2.9377971,-0.66654444,,,,,0,0,0,0,0,0,0,0,0,0 26 | 1933,-0.79064363,0.10004336,0,0,0,,-0.7412371,2.5422783,2.1087773,0.32644382,,,,,0,0,0,0,0,0,0,0,0,0 27 | 1934,-1.0272709,-0.23662728,0,0,0,,-0.8252362,1.937604,1.5223973,1.21573,,,,,0,0,0,0,0,0,0,0,0,0 28 | 1935,-0.42593405,0.60133684,0,1,0,,-0.28812715,1.700637,1.4819573,0.49172917,,,,,0,0,0,0,0,0,0,0,0,0 29 | 1936,-0.62392253,-0.19798848,0,0,0,,-0.33836573,1.2512168,1.0497549,0.83346772,,,,,0,0,0,0,0,0,0,0,0,0 30 | 1937,-0.50359678,0.12032574,0,0,0,,0.13205695,0.8235867,0.80964237,0.4285506,,,,,0,0,0,0,0,0,0,0,0,0 31 | 1938,-0.65088683,-0.14729005,0,0,0,,-0.57634932,1.2403218,0.95623738,-0.44063324,,,,,0,0,0,0,0,0,0,0,0,0 32 | 1939,-0.45688429,0.19400254,0,0,1,,-0.56401438,0.27066377,0.06150249,0.7157259,,,,,0,1,0,0,0,0,0,0,0,0 33 | 1940,-0.45479542,0.0020888746,0,0,1,,-0.84032661,-0.47564605,-0.72202241,2.4780247,,,,,0,1,0,0,0,0,0,0,0,0 34 | 1941,-0.66431981,-0.20952439,0,0,1,,-0.73632896,-1.0149502,-1.1896099,2.090019,,,,,0,1,0,0,0,0,0,0,0,0 35 | 1942,-0.85102534,-0.18670553,0,0,1,,-0.47445339,-1.1783757,-1.2527974,-0.11612517,,,,,0,1,0,0,0,0,0,0,0,0 36 | 1943,-0.27413103,0.57689428,0,0,1,,-0.34707412,-1.2137846,-1.2426873,-0.14005645,,,,,0,1,0,0,0,0,0,0,0,0 37 | 1944,0.29847756,0.57260859,1,0,1,,0.19579157,-1.2219559,-1.0682899,-2.0922101,,,,,0,1,0,0,0,0,0,0,0,0 38 | 1945,0.38352489,0.085047334,1,1,1,,-0.0078655938,-1.186547,-1.1036749,-2.1474116,,,,,0,0,0,0,0,0,0,0,0,0 39 | 1946,0.091798052,-0.29172683,1,0,0,0,-0.21362862,-0.82701087,-0.83828741,-1.4725497,-0.92742103,0.70560461,0.29544944,0.28407547,0,0,0,0,0,0,0,0,0,0 40 | 1947,-0.33963278,-0.43143082,1,0,0,0,-0.12317979,-0.94413251,-0.9191674,-1.0953928,-1.117864,0.0038054173,0.29097447,0.27304479,0,0,0,0,0,0,0,0,0,0 41 | 1948,-0.24730906,0.09232372,0,0,0,0,-0.29238212,-0.94413251,-0.97477239,0.33761176,-1.0444821,0.1021467,0.29269877,0.27459049,0,0,0,0,0,0,0,0,0,0 42 | 1949,-0.32785377,-0.08054471,0,0,0,0,-0.34749132,-0.90872365,-0.9621349,0.39185598,-1.285028,0.24718054,0.28704646,0.2768701,0,0,0,0,0,0,0,0,0,0 43 | 1950,-0.063886695,0.26396707,0,1,0,0,-0.54588223,-0.90872365,-1.0253223,0.36664838,-0.99176717,0.44004935,0.29393744,0.27990153,0,0,0,0,0,0,0,0,0,0 44 | 1951,-0.48241961,-0.41853291,0,1,1,0,-0.39618745,-0.99316013,-1.0531249,0.51246959,-1.3600864,0.036991175,0.28528276,0.2735664,0,0,0,1,0,0,0,0,0,0 45 | 1952,-0.38282797,0.099591643,0,0,0,1,-0.35377726,-0.87331474,-0.92927736,-0.17707014,-0.94452631,-0.1639767,0.29504752,0.27040765,0,0,0,0,0,0,0,0,0,0 46 | 1953,-0.24560447,0.1372235,0,0,0,1,-0.2045974,-0.88693357,-0.89136487,1.0740567,-0.90039963,0.022677913,0.2960844,0.27334142,0,0,0,0,0,0,0,0,0,0 47 | 1954,0.02812564,0.2737301,0,0,0,1,-0.11577754,-0.95230377,-0.92422235,0.68509382,-0.48687485,0.1718449,0.3058013,0.275686,0,0,0,0,0,0,0,0,0,0 48 | 1955,-0.33014581,-0.35827145,0,1,0,1,0.20927413,-1.0231215,-0.87872738,0.5335291,-0.59529519,-0.5616762,0.30325368,0.2641567,0,0,0,0,0,0,0,0,0,0 49 | 1956,-0.61960965,-0.28946385,1,0,0,1,-0.24030307,-1.0095026,-1.0177399,-0.16622131,-0.91969347,-0.85108995,0.29563102,0.25960779,0,0,0,0,0,0,0,0,0,0 50 | 1957,-0.13612159,0.48348808,0,0,0,1,0.8123036,-0.93323743,-0.59564739,-0.076558799,-0.5723573,-0.25054854,0.30379266,0.26904693,0,0,0,0,0,0,0,0,0,0 51 | 1958,-0.18165012,-0.045528531,0,0,0,1,0.14280644,-0.79160202,-0.6866374,-0.28523949,-0.56678039,-0.38195273,0.3039237,0.26698154,0,0,0,0,0,0,0,0,0,0 52 | 1959,0.14062634,0.32227647,0,1,0,1,0.45966595,-0.76164067,-0.5526799,0.62255347,-0.36130685,0.076833747,0.30875188,0.27419263,0,0,0,0,0,0,0,0,0,0 53 | 1960,0.09386757,-0.046758771,0,0,0,1,0.040802725,-0.88148606,-0.80542988,1.3181558,-0.31707379,-0.048185587,0.30979127,0.27222762,0,0,0,0,0,0,0,0,0,0 54 | 1961,-0.33693904,-0.43080661,1,0,0,1,0.046269022,-0.91961867,-0.83828741,0.16626385,-0.62397885,-0.55571234,0.30257967,0.26425046,0,0,0,0,0,0,0,0,0,0 55 | 1962,-0.0008269104,0.33611211,0,0,0,1,0.5316692,-0.79432577,-0.55773491,-0.33693105,-0.28278065,-0.33017331,0.31059706,0.26779541,0,0,0,0,0,0,0,0,0,0 56 | 1963,0.33442745,0.33525437,0,0,0,1,-0.36951032,-0.69899422,-0.7725724,0.86888599,-0.004393341,-0.036040235,0.31713855,0.2724185,0,0,0,0,0,0,0,0,0,0 57 | 1964,0.28432092,-0.050106525,0,1,0,1,-0.173151,-0.86786723,-0.86356241,1.0871392,-0.41892377,0.53020108,0.30739802,0.28131852,0,0,0,0,0,0,0,0,0,0 58 | 1965,0.15582736,-0.12849356,0,0,0,0,0.015701862,-0.93051368,-0.85850739,-0.0015741459,-0.85974765,0.92479849,0.29703963,0.28752068,0,0,0,0,0,0,0,0,0,0 59 | 1966,-0.044033471,-0.19986083,0,1,0,0,-0.13410665,-0.91961867,-0.89641988,-0.19015257,-0.88012469,0.37994492,0.29656079,0.27895683,0,0,0,0,0,0,0,0,0,0 60 | 1967,-0.11934934,-0.075315863,0,0,0,0,-0.020742701,-0.70444173,-0.66136241,0.20263939,-0.92903316,0.2665863,0.29541156,0.2771751,0,0,0,0,0,0,0,0,0,0 61 | 1968,-0.28573951,-0.16639018,0,0,0,0,0.37173989,-0.67175663,-0.49960244,1.0657606,-1.0626507,-0.046625756,0.29227185,0.27225211,0,0,0,0,0,0,0,0,0,0 62 | 1969,-0.22701611,0.058723405,0,0,1,0,0.65696359,-0.67992795,-0.40861243,-0.067305379,-1.0047656,-0.010842443,0.29363203,0.27281457,0,0,0,0,0,0,0,1,1,0 63 | 1970,0.172855,0.39987111,0,1,0,1,1.0132073,-0.64451909,-0.25696248,0.17679361,-0.5872705,0.30350733,0.30344224,0.27775541,0,0,0,0,0,0,0,1,0,0 64 | 1971,-0.41881865,-0.59167367,0,0,0,1,1.171858,-0.59821516,-0.16344498,0.4228071,-0.55719531,-1.1308548,0.30414894,0.25521052,0,0,0,0,0,0,0,1,0,0 65 | 1972,-0.26786959,0.15094906,0,0,1,1,1.6000303,-0.51650244,0.056447491,0.66052443,-0.52248859,-0.84193707,0.30496445,0.25975165,0,0,0,0,0,0,0,1,0,1 66 | 1973,-0.32286966,-0.055000067,1,0,0,1,0.69466823,-0.73440307,-0.44905245,1.3979267,-0.54775816,-1.0416653,0.30437067,0.25661239,0,0,0,0,0,0,0,1,0,0 67 | 1974,-0.74110341,-0.41823375,1,1,0,0,1.2357924,-0.73440307,-0.26707247,-1.4923329,-1.6700516,-0.21505216,0.27799928,0.26960486,0,0,0,0,0,0,0,1,0,0 68 | 1975,-0.60631508,0.13478833,1,0,0,0,0.55900103,-0.46202725,-0.24179746,-1.1825027,-1.159252,-0.69608414,0.29000193,0.26204413,0,0,0,0,0,0,0,0,0,0 69 | 1976,-0.74996775,-0.14365268,0,0,0,0,0.10300197,-0.1896514,-0.14069748,0.27794313,-1.2948878,-0.79237884,0.28681478,0.26053059,0,0,0,0,0,0,0,0,0,0 70 | 1977,-0.53419632,0.21577144,0,0,0,0,0.95333934,-0.10793865,0.21820746,0.14009899,-1.1180226,-0.50076956,0.29097074,0.26511404,0,0,0,0,0,0,0,0,0,0 71 | 1978,-0.42600018,0.10819614,0,0,0,0,0.89644742,-0.13517624,0.17523998,0.62989235,-1.0804492,-0.36035454,0.29185364,0.26732102,0,0,0,0,0,0,0,0,0,0 72 | 1979,-0.30970472,0.11629546,0,1,0,1,1.7578279,-0.21688899,0.38754994,0.48502842,-0.64437735,-0.90001637,0.30210036,0.25883877,0,0,0,0,0,0,0,0,0,0 73 | 1980,-0.61353511,-0.30383039,1,0,0,1,1.077929,0.05548685,0.41282496,-1.3803345,-0.57467705,-1.8349644,0.30373815,0.24414355,0,0,0,0,0,0,0,0,0,0 74 | 1981,-0.46105897,0.15247613,1,0,0,1,0.30028468,0.8181392,0.86019236,-0.95850599,-0.39791387,-1.7978262,0.3078917,0.24472727,0,0,0,0,0,0,0,0,0,0 75 | 1982,-0.29033884,0.17072013,0,0,1,1,0.46579397,1.1994654,1.2696474,-0.025824502,-0.39094949,-1.5388892,0.30805534,0.24879716,0,0,0,0,1,0,0,0,0,0 76 | 1983,-0.067130812,0.22320804,0,1,0,1,0.20386982,1.4446037,1.4086598,0.6525473,0.056020729,-1.6479961,0.31855816,0.24708225,0,0,0,0,0,0,0,0,0,0 77 | 1984,-0.29248586,-0.22535506,0,0,0,1,1.6954759,1.4990788,1.9596547,0.032248721,-0.18895319,-1.6733624,0.31280181,0.24668355,0,0,0,0,0,0,0,0,0,0 78 | 1985,-0.23621055,0.056275308,0,0,0,1,0.60639822,1.5535539,1.6437173,0.44290936,-0.32209268,-1.4937073,0.30967331,0.24950732,0,0,0,0,0,0,0,0,0,0 79 | 1986,0.23753412,0.47374469,0,0,0,1,-0.30174991,1.6080291,1.3909674,0.32006216,0.14473189,-1.1047786,0.32064268,0.25562039,0,0,0,0,0,0,0,0,0,0 80 | 1987,0.379233,0.14169888,0,1,0,1,0.16088498,1.3084158,1.2671199,1.0813957,0.30730435,-0.90761286,0.32446277,0.25871938,0,0,0,0,0,0,0,0,0,0 81 | 1988,0.19151476,-0.18771824,0,0,0,1,0.19335113,0.7909016,0.79953241,1.2052002,0.33689281,-1.205856,0.32515803,0.25403169,0,0,0,0,0,0,0,0,0,0 82 | 1989,0.24086088,0.049346119,0,0,0,1,0.27548727,0.35510027,0.42293495,0.11521047,0.3395102,-1.1358067,0.32521951,0.2551327,0,0,0,0,0,0,0,0,0,0 83 | 1990,0.25036207,0.009501189,1,0,0,1,-0.30845654,0.24614994,0.12468998,-0.51689422,0.19880894,-0.91855115,0.32191336,0.25854746,0,0,0,0,0,0,0,0,0,0 84 | 1991,0.2184011,-0.031960964,1,0,0,1,-0.99962699,0.8181392,0.42293495,-1.0826296,0.25217438,-1.1411408,0.32316732,0.25504887,0,0,0,0,0,0,0,0,0,0 85 | 1992,0.85975409,0.64135301,0,1,0,1,-1.2752775,1.2539406,0.73634487,-0.54561174,0.91043162,-0.38984972,0.33863491,0.26685745,0,0,0,0,0,0,0,0,0,0 86 | 1993,0.74697995,-0.11277413,0,0,0,1,-1.1196798,1.3901285,0.91579735,0.15605317,0.70125014,-0.43613392,0.33371961,0.26612994,0,0,0,0,0,0,0,0,0,0 87 | 1994,1.0328327,0.28585279,0,0,0,1,-1.7590121,1.1449902,0.47348493,0.59606951,1.0255889,-0.15273765,0.34134087,0.27058429,0,0,0,0,0,0,0,0,0,0 88 | 1995,1.1674197,0.13458693,0,0,0,1,-1.4568775,0.8181392,0.27128497,0.11903947,1.0031501,0.12587453,0.34081361,0.27496344,0,0,0,0,0,0,0,0,0,0 89 | 1996,0.72605413,-0.44136554,0,0,0,1,-0.59408015,0.6274761,0.38249496,0.16275392,0.59831828,-0.43484291,0.33130094,0.26615024,0,0,0,0,0,0,0,0,0,0 90 | 1997,1.6525837,0.92652959,0,1,0,0,-1.8857273,0.19167477,-0.45410743,0.12637839,1.07399,1.3471634,0.34247819,0.29415929,0,0,0,0,0,0,0,0,0,0 91 | 1998,1.5716605,-0.0809232,0,0,0,0,-1.7482392,-0.026225902,-0.61081243,0.43206051,0.96797824,1.4310706,0.33998713,0.29547814,0,0,0,0,0,0,0,0,0,0 92 | 1999,1.4210109,-0.15064967,0,0,0,0,-1.8635929,-0.13517624,-0.74982488,0.31719041,1.018213,0.87659687,0.34116754,0.28676307,0,0,0,0,0,0,0,0,0,0 93 | 2000,1.4285079,0.0074970722,0,0,0,0,-1.3178766,-0.27136415,-0.69421989,0.51438409,1.0279744,1.028741,0.3413969,0.28915444,0,0,0,0,0,0,0,0,0,0 94 | 2001,1.1946708,-0.23383713,0,1,0,0,-1.2795744,-0.40755209,-0.80795741,0.16211577,0.75806081,0.82696658,0.33505455,0.285983,0,0,0,0,0,0,0,0,0,0 95 | 2002,1.7365783,0.54190755,0,0,0,0,-0.58259326,-0.40755209,-0.57289994,0.094150946,1.1547512,1.3951724,0.34437588,0.29491389,0,0,0,0,0,0,0,0,0,0 96 | 2003,1.5381299,-0.19844842,0,0,1,0,-1.3178766,-0.43478966,-0.8458699,0.68381751,1.0588363,1.1489598,0.34212211,0.291044,0,0,0,0,0,1,0,0,0,0 97 | 2004,1.6973416,0.15921164,0,0,0,0,-0.86894077,-0.51650244,-0.77004492,0.09478911,1.1849245,1.3118045,0.34508488,0.29360354,0,0,0,0,0,0,0,0,0,0 98 | 2005,1.7263188,0.028977275,0,1,0,0,-2.1898847,-0.51650244,-1.2123573,0.20742564,1.2737514,1.2508429,0.34717214,0.29264536,0,0,0,0,0,0,0,0,0,0 99 | 2006,2.0078304,0.28151155,0,0,1,0,-1.0055962,-0.43478966,-0.73971492,0.28241029,1.5150973,1.4944296,0.35284323,0.29647398,0,0,0,0,0,0,1,0,0,0 100 | 2007,2.3840928,0.37626243,0,0,1,0,-0.76336557,-0.51650244,-0.73465991,0.12733564,1.9231657,1.7211281,0.36243191,0.30003718,0,0,0,0,0,0,1,0,0,0 101 | 2008,2.3178947,-0.066198111,1,0,0,0,-1.0016115,-0.48926485,-0.7902649,-0.79417789,1.8942016,1.661122,0.36175135,0.29909402,0,0,0,0,0,0,0,0,0,0 102 | 2009,2.0522707,-0.26562405,1,0,0,0,-1.3874863,0.0010116817,-0.46421742,-2.0638118,1.6678151,1.4092635,0.35643175,0.29513538,0,0,0,0,0,0,0,0,0,0 103 | 2010,2.4997156,0.44744492,0,1,0,1,-1.5536894,-0.026225902,-0.54509741,-0.0784733,1.7170815,2.2743526,0.35758939,0.3087326,0,0,0,0,0,0,0,0,0,0 104 | 2011,2.4460173,-0.053698301,0,0,0,1,-0.54533947,0.028249266,-0.15586248,-0.16334955,1.7987131,1.3899277,0.35950756,0.29483145,0,0,0,0,0,0,0,0,0,0 105 | 2012,2.4600923,0.014075041,0,0,0,1,-1.8420897,0.028249266,-0.59059244,-0.47796601,1.8747853,1.324195,0.3612951,0.2937983,0,0,0,0,0,0,0,0,0,0 106 | 2013,2.2747557,-0.18533659,0,0,0,1,-1.4059412,-0.10793865,-0.5703724,-0.15696788,1.7301728,1.3002089,0.35789701,0.29342127,0,0,0,0,0,0,0,0,0,0 107 | -------------------------------------------------------------------------------- /movie-classifier.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | #=====================================================================# 5 | # 6 | # Description: 7 | # A script to evaluate polarity classifiers on the IMDb corpus. 8 | # 9 | # Usage: 10 | # python movie-classifier.py train-imdb.csv test-imdb.csv [full/extreme] [none/bow/all] [K] 11 | # 12 | # --option full or extreme will either use the full testing set or the one restricted to extreme reviews. 13 | # --option none, bow or all will use the polarity score only, add the bag-of-words models, or use all available features. 14 | # --option K, if provided, will restrict the number of features to the K best using a chi-square test. 15 | # Dependencies: train-imdb.csv and test-imdb.csv 16 | # 17 | # The script will display accuracy statistics via standard output. 18 | # 19 | # Results reported in text (Table SI3) are obtained with: 20 | # python movie-classifier.py train-imdb.csv test-imdb.csv full none 21 | # python movie-classifier.py train-imdb.csv test-imdb.csv full bow 20000 22 | # python movie-classifier.py train-imdb.csv test-imdb.csv full all 20000 23 | # python movie-classifier.py train-imdb.csv test-imdb.csv extreme none 24 | # python movie-classifier.py train-imdb.csv test-imdb.csv extreme bow 5000 25 | # python movie-classifier.py train-imdb.csv test-imdb.csv extreme all 5000 26 | # 27 | # Author: L. Rheault 28 | # 29 | #=====================================================================# 30 | 31 | from __future__ import division 32 | from operator import itemgetter 33 | import os, sys, codecs, re 34 | import pandas as pd 35 | import numpy as np 36 | from time import time 37 | from scipy import sparse 38 | from scipy.sparse import hstack 39 | from sklearn import metrics 40 | from sklearn.feature_extraction import DictVectorizer 41 | from sklearn.preprocessing import OneHotEncoder 42 | from sklearn.metrics import roc_auc_score 43 | from sklearn.datasets import load_svmlight_file 44 | from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, TfidfTransformer 45 | from sklearn.pipeline import Pipeline, FeatureUnion 46 | from sklearn.feature_selection import SelectKBest, chi2 47 | from sklearn.cross_validation import StratifiedKFold,StratifiedShuffleSplit,KFold,ShuffleSplit,train_test_split 48 | from sklearn.svm import LinearSVC, SVC 49 | from sklearn.linear_model import SGDClassifier 50 | from sklearn.linear_model import Perceptron 51 | from sklearn.neighbors import KNeighborsClassifier 52 | from sklearn.neighbors.nearest_centroid import NearestCentroid 53 | from sklearn.naive_bayes import MultinomialNB, BernoulliNB 54 | from sklearn.base import BaseEstimator, TransformerMixin 55 | import cPickle as pickle 56 | 57 | #=====================================================================# 58 | # Feature classes. 59 | 60 | class polarityTransformer(BaseEstimator, TransformerMixin): 61 | def fit(self, X, y=None): 62 | return self 63 | def transform(self, X): 64 | polarity = np.asarray(X['polar'].apply(pd.to_numeric)) 65 | return [{'polar': polar} for polar in polarity] 66 | 67 | class movieIdTransformer(BaseEstimator, TransformerMixin): 68 | def fit(self, X, y=None): 69 | return self 70 | def transform(self, X): 71 | movies = np.asarray(X['imdb'].astype('category')) 72 | return [{'movie': movie} for movie in movies] 73 | 74 | class movieGenreTransformer(BaseEstimator, TransformerMixin): 75 | def fit(self, X, y=None): 76 | return self 77 | def transform(self, X): 78 | genres = [ 'com','rom','cri' ,'war','dra','hor','fno','thr','ani','fam','adv','fan','sci','mys','bio','doc'] 79 | genrescol = X[genres].apply(pd.to_numeric) 80 | genreDict = genrescol.T.to_dict().values() 81 | return genreDict 82 | 83 | class movieRatingTransformer(BaseEstimator, TransformerMixin): 84 | def fit(self, X, y=None): 85 | return self 86 | def transform(self, X): 87 | ratings = np.asarray(X['rating'].apply(pd.to_numeric)) 88 | return [{'rating': rating} for rating in ratings] 89 | 90 | class textTransformer(BaseEstimator, TransformerMixin): 91 | def fit(self, X, y=None): 92 | return self 93 | def transform(self, X): 94 | text = X['text'].tolist() 95 | return text 96 | 97 | #=====================================================================# 98 | # For export. 99 | 100 | def exportModel(X, y, clf, features): 101 | clf.fit(X, y) 102 | print "Saving the models as a Pickle archive." 103 | with open('topic-classifier-models.pkl', 'wb') as p: 104 | pickle.dump(clf, p) 105 | print "Saving the features as a Pickle archive." 106 | with open('features.pkl', 'wb') as q: 107 | pickle.dump(features, q) 108 | 109 | # Full testing set or subset. 110 | def dataImport(trainData, testData, subsetOption): 111 | inpathTrain = trainData 112 | inpathTest = testData 113 | if subsetOption=="full": 114 | a = pd.read_table(inpathTrain,delimiter=",",header=0,dtype=object,encoding='utf-8') 115 | a['bscore'] = a.bscore.apply(pd.to_numeric) 116 | # Converting polarity score to positive range; required only for SelectKBest(chi2) script. 117 | a['polar'] = a.polar.apply(pd.to_numeric) 118 | a['polar'] = a.polar+1 119 | X_train = a[['polar','imdb','rating', 'com','rom','cri' ,'war','dra','hor','fno','thr','ani','fam','adv','fan','sci','mys','bio','doc','text']] 120 | y_train = a['bscore'].values.ravel() 121 | b = pd.read_table(inpathTest,delimiter=",",header=0,dtype=object,encoding='utf-8') 122 | b['bscore'] = b.bscore.apply(pd.to_numeric) 123 | # Converting polarity score to positive range; required only for SelectKBest(chi2) script. 124 | b['polar'] = b.polar.apply(pd.to_numeric) 125 | b['polar'] = b.polar+1 126 | X_test = b[['polar','imdb','rating','com','rom','cri' ,'war','dra','hor','fno','thr','ani','fam','adv','fan','sci','mys','bio','doc','text']] 127 | y_test = b['bscore'].values.ravel() 128 | return (X_train, y_train, X_test, y_test) 129 | elif subsetOption=="extreme": 130 | a = pd.read_table(inpathTrain,delimiter=",",header=0,dtype=object,encoding='utf-8') 131 | a = a[pd.notnull(a.bscore10)] 132 | a['bscore10'] = a.bscore10.apply(pd.to_numeric) 133 | # Converting polarity score to positive range; required only for SelectKBest(chi2) script. 134 | a['polar'] = a.polar.apply(pd.to_numeric) 135 | a['polar'] = a.polar+1 136 | X_train = a[['polar','imdb','rating', 'com','rom','cri' ,'war','dra','hor','fno','thr','ani','fam','adv','fan','sci','mys','bio','doc','text']] 137 | y_train = a['bscore10'].values.ravel() 138 | b = pd.read_table(inpathTest,delimiter=",",header=0,dtype=object,encoding='utf-8') 139 | b = b[pd.notnull(b.bscore10)] 140 | b['bscore10'] = b.bscore10.apply(pd.to_numeric) 141 | # Converting polarity score to positive range; required only for SelectKBest(chi2) script. 142 | b['polar'] = b.polar.apply(pd.to_numeric) 143 | b['polar'] = b.polar+1 144 | X_test = b[['polar','imdb','rating','com','rom','cri' ,'war','dra','hor','fno','thr','ani','fam','adv','fan','sci','mys','bio','doc','text']] 145 | y_test = b['bscore10'].values.ravel() 146 | return (X_train, y_train, X_test, y_test) 147 | else: 148 | sys.exit("Choose between the 'full' or 'extreme' review sets.") 149 | 150 | def featureSelect(X_train, y_train, X_test, y_test, subsetChoice, featChoice, K): 151 | 152 | moviePolarity = Pipeline([ 153 | ('polar', polarityTransformer()), 154 | ('dict-vect', DictVectorizer()) 155 | ]) 156 | 157 | movieId = Pipeline([ 158 | ('movie', movieIdTransformer()), 159 | ('dict-vect', DictVectorizer()) 160 | ]) 161 | 162 | movieGenre = Pipeline([ 163 | ('genre', movieGenreTransformer()), 164 | ('dict-vect', DictVectorizer()) 165 | ]) 166 | 167 | movieRating = Pipeline([ 168 | ('rating', movieRatingTransformer()), 169 | ('dict-vect', DictVectorizer()) 170 | ]) 171 | 172 | mainText = Pipeline([ 173 | ('text', textTransformer()), 174 | ('tfidf', TfidfVectorizer(stop_words='english',ngram_range=(1,3))) 175 | ]) 176 | 177 | if featChoice=="none": 178 | features = FeatureUnion([ 179 | ('moviePolar', moviePolarity) 180 | ]) 181 | X_train = features.fit_transform(X_train) 182 | X_test = features.transform(X_test) 183 | elif featChoice=="bow": 184 | features = FeatureUnion([ 185 | ('moviePolar', moviePolarity), 186 | ('text', mainText) 187 | ]) 188 | kbest = SelectKBest(chi2, k = K) 189 | X_train = features.fit_transform(X_train) 190 | X_train = kbest.fit_transform(X_train, y_train) 191 | X_test = features.transform(X_test) 192 | X_test = kbest.transform(X_test) 193 | elif featChoice=="all": 194 | features = FeatureUnion([ 195 | ('moviePolar', moviePolarity), 196 | ('movieGenre', movieGenre), 197 | ('movieRating', movieRating), 198 | ('text', mainText) 199 | ]) 200 | kbest = SelectKBest(chi2, k = K) 201 | X_train = features.fit_transform(X_train) 202 | X_train = kbest.fit_transform(X_train, y_train) 203 | X_test = features.transform(X_test) 204 | X_test = kbest.transform(X_test) 205 | else: 206 | sys.exit("Choose between 'none', 'bow' or 'all' additional features.") 207 | 208 | print "Training and evaluating the model(s)." 209 | models = ( 210 | (LinearSVC(), "Support Vector Machine (Default)"), 211 | (LinearSVC(penalty="l1", dual=False, tol=1e-3), "Support Vector Machine: l1, dual=False"), 212 | (LinearSVC(penalty="l2", dual=False, tol=1e-3), "Support Vector Machine: l2, dual=False") 213 | ) 214 | 215 | for clf, name in models: 216 | print('-' * 76) 217 | print(name) 218 | print('-' * 76) 219 | clf.fit(X_train,y_train) 220 | pred = clf.predict(X_test) 221 | pcp = metrics.accuracy_score(y_test,pred) 222 | f1 = metrics.f1_score(y_test,pred,average="binary") 223 | mc = metrics.classification_report(y_test, pred) 224 | cm = metrics.confusion_matrix(y_test, pred) 225 | if subsetChoice=="full": 226 | pre = (12500 - (int(cm[0][1]) + int(cm[1][0]))) / 12500 227 | elif subsetChoice=="extreme": 228 | pre = (4999 - (int(cm[0][1]) + int(cm[1][0]))) / 4999 229 | roc = roc_auc_score(y_test, pred) 230 | print "The percent correctly predicted is %0.3f" %pcp 231 | print "The F1-score is %0.3f" %f1 232 | print "The proportional reduction in errors is %0.3f" %pre 233 | print "The confusion matrix:" 234 | print cm 235 | print "The classification report:" 236 | print mc 237 | print "The area under the ROC curve is %0.3f" %roc 238 | print('-' * 76) 239 | 240 | def uFeatures(X_train, y_train, X_test, y_test, subsetChoice, featChoice): 241 | 242 | moviePolarity = Pipeline([ 243 | ('polar', polarityTransformer()), 244 | ('dict-vect', DictVectorizer()) 245 | ]) 246 | 247 | movieId = Pipeline([ 248 | ('movie', movieIdTransformer()), 249 | ('dict-vect', DictVectorizer()) 250 | ]) 251 | 252 | movieGenre = Pipeline([ 253 | ('genre', movieGenreTransformer()), 254 | ('dict-vect', DictVectorizer()) 255 | ]) 256 | 257 | movieRating = Pipeline([ 258 | ('rating', movieRatingTransformer()), 259 | ('dict-vect', DictVectorizer()) 260 | ]) 261 | 262 | mainText = Pipeline([ 263 | ('text', textTransformer()), 264 | ('tfidf', TfidfVectorizer(stop_words='english',ngram_range=(1,3))) 265 | ]) 266 | 267 | if featChoice=="none": 268 | features = FeatureUnion([ 269 | ('moviePolar', moviePolarity) 270 | ]) 271 | X_train = features.fit_transform(X_train) 272 | X_test = features.transform(X_test) 273 | elif featChoice=="bow": 274 | features = FeatureUnion([ 275 | ('moviePolar', moviePolarity), 276 | ('text', mainText) 277 | ]) 278 | X_train = features.fit_transform(X_train) 279 | X_test = features.transform(X_test) 280 | elif featChoice=="all": 281 | features = FeatureUnion([ 282 | ('moviePolar', moviePolarity), 283 | ('movieGenre', movieGenre), 284 | ('movieRating', movieRating), 285 | ('text', mainText) 286 | ]) 287 | X_train = features.fit_transform(X_train) 288 | X_test = features.transform(X_test) 289 | else: 290 | sys.exit("Choose between 'none', 'bow' or 'all' additional features.") 291 | 292 | print "Training and evaluating the model(s)." 293 | models = ( 294 | (LinearSVC(), "Support Vector Machine (Default)"), 295 | (LinearSVC(penalty="l1", dual=False, tol=1e-3), "Support Vector Machine: l1, dual=False"), 296 | (LinearSVC(penalty="l2", dual=False, tol=1e-3), "Support Vector Machine: l2, dual=False") 297 | ) 298 | 299 | for clf, name in models: 300 | print('-' * 76) 301 | print(name) 302 | print('-' * 76) 303 | clf.fit(X_train,y_train) 304 | pred = clf.predict(X_test) 305 | pcp = metrics.accuracy_score(y_test,pred) 306 | f1 = metrics.f1_score(y_test,pred,average="binary") 307 | mc = metrics.classification_report(y_test, pred) 308 | cm = metrics.confusion_matrix(y_test, pred) 309 | if subsetChoice=="full": 310 | pre = (12500 - (int(cm[0][1]) + int(cm[1][0]))) / 12500 311 | elif subsetChoice=="extreme": 312 | pre = (4999 - (int(cm[0][1]) + int(cm[1][0]))) / 4999 313 | roc = roc_auc_score(y_test, pred) 314 | print "The percent correctly predicted is %0.3f" %pcp 315 | print "The F1-score is %0.3f" %f1 316 | print "The proportional reduction in errors is %0.3f" %pre 317 | print "The confusion matrix:" 318 | print cm 319 | print "The classification report:" 320 | print mc 321 | print "The area under the ROC curve is %0.3f" %roc 322 | print('-' * 76) 323 | 324 | #=====================================================================# 325 | 326 | if __name__=="__main__": 327 | 328 | t0 = time() 329 | print "Opening and preparing the corpus." 330 | inpathTrain = str(sys.argv[1]) 331 | inpathTest = str(sys.argv[2]) 332 | subsetChoice = str(sys.argv[3]) 333 | featureChoice = str(sys.argv[4]) 334 | X_train, y_train, X_test, y_test = dataImport(inpathTrain, inpathTest, subsetChoice) 335 | if len(sys.argv)>5: 336 | K = int(sys.argv[5]) 337 | featureSelect(X_train, y_train, X_test, y_test, subsetChoice, featureChoice, K) 338 | else: 339 | uFeatures(X_train, y_train, X_test, y_test, subsetChoice, featureChoice) 340 | 341 | print("Done in %0.3fs" % (time() - t0)) 342 | 343 | -------------------------------------------------------------------------------- /modern-hansard-parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | #======================================================================# 5 | # 6 | # Description: A script to parse the British Hansard files in the Political Mashup XML format. 7 | # Will output speeches by quarter and save them in the output-directory. 8 | # 9 | # Usage: 10 | # python modern-hansard-parser.py [path-to-xml-file] [output-directory] 11 | # 12 | # Author: K. Beelen 13 | # 14 | #======================================================================# 15 | 16 | from lxml import etree as Etree 17 | from datetime import datetime as dt 18 | import os, codecs,re 19 | 20 | #---------------------------------------------------------------------------------------------------------------------------------------------------------------# 21 | 22 | class XML_to_TSV(object): 23 | def __init__(self,min_date="",max_date="",inpath,outpath): 24 | self.directory = inpath 25 | self.min_date = date(min_date) 26 | self.max_date = date(max_date) 27 | self.output_data = codecs.open(outpath+'/UKHansard-%sto%s.tsv'%(min_date,max_date),"w",encoding="utf-8") 28 | self.namespaces = {"pm":"http://www.politicalmashup.nl"} 29 | self.namespaces_mads = {"mads":"http://www.loc.gov/mads/v2"} 30 | self.gender_dict = {} 31 | 32 | def makeTable(self): 33 | years_all = [f for f in sorted(os.listdir(self.directory)) if f.startswith("uk")] 34 | for y in years_all: 35 | doc_date = date(y.split(".")[-2]) 36 | if doc_date >= self.min_date and doc_date <= self.max_date: 37 | print y 38 | self.SpeechXMLtoTSV(os.path.join(self.directory,y)) 39 | 40 | def SpeechXMLtoTSV(self,xml_file): 41 | date = xml_file.split("/")[-1].split(".")[-2] 42 | with open(xml_file,'rt') as xmlf: 43 | tree = Etree.parse(xmlf) 44 | topics =self.getELementsbyXpath(""".//pm:topic""",tree) 45 | for topic in topics: 46 | topic_title = self.getElementsAttrbyXpath("""@pm:title""",topic) 47 | scenes = self.getELementsbyXpath("""pm:scene""",topic) 48 | if len(scenes) > 0: 49 | for scene in scenes: 50 | scene_title = self.getElementsAttrbyXpath("""@pm:title""",scene) 51 | self.getSpeeches(scene,topic_title,scene_title,date) 52 | else: 53 | scene_title = "NA" 54 | self.getSpeeches(topic,topic_title,scene_title,date) 55 | 56 | def getSpeeches(self,parent_element,topic_title,scene_title,date): 57 | speeches = self.getELementsbyXpath(""".//pm:speech""",parent_element) 58 | for speech in speeches: 59 | speech_id = self.getElementsAttrbyXpath("""@pm:id""",speech) 60 | speaker_ref = self.getElementsAttrbyXpath("""@pm:member-ref""",speech) 61 | speaker_party = self.getElementsAttrbyXpath("""@pm:party""",speech) 62 | speaker_role = self.getElementsAttrbyXpath("""@pm:role""",speech) 63 | speaker_name = self.getElementsAttrbyXpath("""@pm:speaker""",speech) 64 | speaker_function = self.getElementsAttrbyXpath("""@pm:function""",speech) 65 | paragraphs = self.getELementsbyXpath(".//pm:p",speech) 66 | text = u" ".join([Etree.tostring(para,method="text",encoding="unicode") for para in paragraphs]) 67 | text = re.compile(ur"[\t\n\r\u2026\u2029]",re.DOTALL).sub(" ",text) 68 | self.output_data.write(u"{}\n".format(text,)) 69 | 70 | def getElementsAttrbyXpath(self,xpath,element): 71 | results = element.xpath(xpath,namespaces=self.namespaces) 72 | if len(results) > 0: return results[0] 73 | else: return "NA" 74 | 75 | def getELementsbyXpath(self,xpath,element): 76 | return element.xpath(xpath,namespaces=self.namespaces) 77 | 78 | class Table: 79 | def __init__(self,tsvfile,min_date="",max_date="",topic_term="",scene_term="",speech_term=""): 80 | self.data = tsvfile 81 | self.min_date = date(min_date) 82 | self.max_date = date(max_date) 83 | self.topic_term = topic_term 84 | self.scene_term = scene_term 85 | self.speech_term = speech_term 86 | self.output = codecs.open(os.path.join("/".join(tsvfile.split("/")[:-1]),"reducedUKHansard-ymin{}-ymax{}-top{}-sc{}-sp{}.tsv".format(min_date,max_date,self.topic_term,self.scene_term,self.speech_term)),"w",encoding='utf-8') 87 | 88 | def getRows(self): 89 | self.output.write("SPEECH\n") 90 | with codecs.open(self.data,encoding="utf-8") as data_file: 91 | for line in data_file: 92 | try: 93 | l = rowData(line) 94 | if date(l.date) < self.min_date: continue 95 | if date(l.date) > self.max_date: break 96 | 97 | self.selectRow_by_Content(l) 98 | except Exception as e: 99 | print e, line 100 | 101 | def selectRow_by_Content(self,rowObject): 102 | if self.speech_term in rowObject.speech.lower() and (self.scene_term in rowObject.scene or self.topic_term in rowObject.topic): 103 | self.output.write(u"{}".format(rowObject.data)) 104 | print rowObject.speech_id 105 | 106 | def selectRow_by_Metadata(self,rowObject): 107 | pass 108 | 109 | class rowData: 110 | def __init__(self,line): 111 | cells = line.split("\t") 112 | self.data = line 113 | self.speech = cells[0] 114 | #self.date = self.date 115 | 116 | def date(date=None): 117 | if date == "current": 118 | date = dt.date(dt.now()) 119 | yyyy,mm,dd = date.year,date.month,date.day 120 | elif len(date.split("-")) == 3: 121 | yyyy,mm,dd = date.split("-") 122 | else: 123 | yyyy,mm,dd = date.split("-")[0],"01","01" 124 | return dt(int(yyyy),int(mm),int(dd)) 125 | 126 | 127 | if __name__ == '__main__': 128 | listoffiles = [("1936-02-24","1936-03-31"), 129 | ("1936-04-01","1936-06-30"), 130 | ("1936-07-01","1936-07-31"), 131 | ("1936-10-29","1936-12-18"), 132 | ("1937-01-19","1937-03-25"), 133 | ("1937-04-06","1937-06-30"), 134 | ("1937-07-01","1937-07-30"), 135 | ("1937-10-21","1937-12-23"), 136 | ("1938-02-01","1938-03-31"), 137 | ("1938-04-01","1938-06-30"), 138 | ("1938-07-01","1938-09-28"), 139 | ("1938-10-03","1938-12-22"), 140 | ("1939-01-31","1939-03-31"), 141 | ("1939-04-03","1939-06-30"), 142 | ("1939-07-03","1939-09-29"), 143 | ("1939-10-02","1939-12-14"), 144 | ("1940-01-16","1940-03-21"), 145 | ("1940-04-02","1940-06-27"), 146 | ("1940-07-02","1940-09-19"), 147 | ("1940-10-08","1940-11-20"), 148 | ("1941-01-21","1941-03-27"), 149 | ("1941-04-01","1941-06-26"), 150 | ("1941-07-01","1941-09-30"), 151 | ("1941-10-01","1941-12-19"), 152 | ("1942-01-08","1942-03-26"), 153 | ("1942-05-19","1942-06-30"), 154 | ("1942-07-01","1942-09-30"), 155 | ("1942-10-01","1942-12-17"), 156 | ("1943-01-19","1943-03-31"), 157 | ("1943-04-01","1943-06-30"), 158 | ("1943-07-01","1943-09-24"), 159 | ("1943-10-12","1943-12-17"), 160 | ("1944-01-18","1944-03-31"), 161 | ("1944-04-04","1944-06-30"), 162 | ("1944-07-04","1944-09-29"), 163 | ("1944-10-03","1944-12-21"), 164 | ("1945-01-16","1945-03-29"), 165 | ("1945-04-10","1945-06-15"), 166 | ("1945-08-01","1945-08-24"), 167 | ("1945-10-09","1945-12-20"), 168 | ("1946-02-11","1946-03-22"), 169 | ("1946-04-30","1946-06-07"), 170 | ("1946-07-08","1946-08-02"), 171 | ("1946-10-08","1946-11-29"), 172 | ("1947-01-21","1947-03-31"), 173 | ("1947-04-01","1947-06-30"), 174 | ("1947-07-01","1947-08-13"), 175 | ("1947-10-20","1947-12-19"), 176 | ("1948-01-20","1948-03-25"), 177 | ("1948-04-06","1948-06-30"), 178 | ("1948-07-01","1948-09-24"), 179 | ("1948-10-25","1948-12-17"), 180 | ("1949-01-18","1949-03-31"), 181 | ("1949-04-01","1949-06-30"), 182 | ("1949-07-01","1949-09-29"), 183 | ("1949-10-18","1949-12-16"), 184 | ("1950-03-01","1950-03-31"), 185 | ("1950-04-03","1950-06-30"), 186 | ("1950-07-03","1950-09-19"), 187 | ("1950-10-17","1950-12-15"), 188 | ("1951-01-23","1951-03-22"), 189 | ("1951-04-03","1951-06-29"), 190 | ("1951-07-02","1951-08-02"), 191 | ("1951-10-04","1951-12-07"), 192 | ("1952-01-29","1952-03-31"), 193 | ("1952-04-01","1952-06-30"), 194 | ("1952-07-01","1952-08-01"), 195 | ("1952-10-14","1952-12-19"), 196 | ("1953-01-20","1953-03-31"), 197 | ("1953-04-01","1953-06-30"), 198 | ("1953-07-01","1953-07-31"), 199 | ("1953-10-20","1953-12-18"), 200 | ("1954-01-19","1954-03-11"), 201 | ("1954-04-05","1954-06-30"), 202 | ("1954-07-01","1954-07-30"), 203 | ("1954-10-19","1954-12-22"), 204 | ("1955-01-25","1955-03-31"), 205 | ("1955-04-01","1955-06-24"), 206 | ("1955-07-18","1955-07-28"), 207 | ("1955-10-25","1955-12-21"), 208 | ("1956-01-24","1956-03-29"), 209 | ("1956-04-10","1956-06-29"), 210 | ("1956-07-02","1956-09-14"), 211 | ("1956-10-23","1956-12-21"), 212 | ("1957-01-22","1957-03-29"), 213 | ("1957-04-01","1957-06-28"), 214 | ("1957-07-01","1957-08-02"), 215 | ("1957-10-29","1957-12-20"), 216 | ("1958-01-21","1958-03-31"), 217 | ("1958-04-01","1958-06-20"), 218 | ("1958-07-07","1958-08-01"), 219 | ("1958-10-23","1958-12-18"), 220 | ("1959-01-20","1959-03-26"), 221 | ("1959-04-07","1959-06-30"), 222 | ("1959-07-01","1959-09-18"), 223 | ("1959-10-20","1959-12-17"), 224 | ("1960-01-26","1960-03-31"), 225 | ("1960-04-01","1960-06-30"), 226 | ("1960-07-01","1960-07-29"), 227 | ("1960-10-25","1960-12-21"), 228 | ("1961-01-24","1961-03-30"), 229 | ("1961-04-11","1961-06-30"), 230 | ("1961-07-03","1961-08-04"), 231 | ("1961-10-17","1961-12-21"), 232 | ("1962-01-23","1962-03-30"), 233 | ("1962-04-02","1962-06-29"), 234 | ("1962-07-02","1962-08-03"), 235 | ("1962-10-25","1962-12-21"), 236 | ("1963-01-22","1963-03-29"), 237 | ("1963-04-01","1963-06-28"), 238 | ("1963-07-01","1963-08-02"), 239 | ("1963-10-24","1963-12-20"), 240 | ("1964-01-14","1964-03-26"), 241 | ("1964-04-07","1964-06-30"), 242 | ("1964-07-01","1964-07-31"), 243 | ("1964-10-27","1964-12-23"), 244 | ("1965-01-19","1965-03-31"), 245 | ("1965-04-01","1965-06-30"), 246 | ("1965-07-01","1965-08-05"), 247 | ("1965-10-26","1965-12-22"), 248 | ("1966-01-25","1966-03-10"), 249 | ("1966-04-18","1966-06-30"), 250 | ("1966-07-01","1966-08-12"), 251 | ("1966-10-18","1966-12-21"), 252 | ("1967-01-17","1967-03-23"), 253 | ("1967-04-04","1967-06-30"), 254 | ("1967-07-03","1967-07-28"), 255 | ("1967-10-23","1967-12-21"), 256 | ("1968-01-16","1968-03-29"), 257 | ("1968-04-01","1968-06-28"), 258 | ("1968-07-01","1968-08-27"), 259 | ("1968-10-14","1968-12-20"), 260 | ("1969-01-20","1969-03-31"), 261 | ("1969-04-01","1969-06-30"), 262 | ("1969-07-01","1969-07-25"), 263 | ("1969-10-13","1969-12-19"), 264 | ("1970-01-19","1970-03-26"), 265 | ("1970-04-06","1970-06-30"), 266 | ("1970-07-01","1970-07-24"), 267 | ("1970-10-27","1970-12-18"), 268 | ("1971-01-12","1971-03-31"), 269 | ("1971-04-01","1971-06-30"), 270 | ("1971-07-01","1971-09-23"), 271 | ("1971-10-18","1971-12-22"), 272 | ("1972-01-17","1972-03-29"), 273 | ("1972-04-10","1972-06-30"), 274 | ("1972-07-03","1972-08-09"), 275 | ("1972-10-17","1972-12-22"), 276 | ("1973-01-22","1973-03-30"), 277 | ("1973-04-02","1973-06-29"), 278 | ("1973-07-02","1973-07-25"), 279 | ("1973-10-16","1973-12-21"), 280 | ("1974-01-09","1974-03-29"), 281 | ("1974-04-01","1974-06-28"), 282 | ("1974-07-01","1974-07-31"), 283 | ("1974-10-22","1974-12-20"), 284 | ("1975-01-13","1975-03-27"), 285 | ("1975-04-07","1975-06-30"), 286 | ("1975-07-01","1975-08-07"), 287 | ("1975-10-13","1975-12-19"), 288 | ("1976-01-12","1976-03-31"), 289 | ("1976-04-01","1976-06-30"), 290 | ("1976-07-01","1976-08-06"), 291 | ("1976-10-11","1976-12-23"), 292 | ("1977-01-10","1977-03-31"), 293 | ("1977-04-01","1977-06-30"), 294 | ("1977-07-01","1977-07-28"), 295 | ("1977-10-26","1977-12-16"), 296 | ("1978-01-09","1978-03-23"), 297 | ("1978-04-03","1978-06-30"), 298 | ("1978-07-03","1978-08-03"), 299 | ("1978-10-24","1978-12-15"), 300 | ("1979-01-15","1979-03-30"), 301 | ("1979-04-02","1979-06-29"), 302 | ("1979-07-02","1979-07-27"), 303 | ("1979-10-22","1979-12-21"), 304 | ("1980-01-14","1980-03-31"), 305 | ("1980-04-01","1980-06-30"), 306 | ("1980-07-01","1980-08-08"), 307 | ("1980-10-27","1980-12-19"), 308 | ("1981-01-12","1981-03-31"), 309 | ("1981-04-01","1981-06-30"), 310 | ("1981-07-01","1981-07-31"), 311 | ("1981-10-19","1981-12-23"), 312 | ("1982-01-18","1982-03-31"), 313 | ("1982-04-01","1982-06-30"), 314 | ("1982-07-01","1982-07-30"), 315 | ("1982-10-18","1982-12-23"), 316 | ("1983-01-17","1983-03-31"), 317 | ("1983-04-11","1983-06-30"), 318 | ("1983-07-01","1983-07-29"), 319 | ("1983-10-24","1983-12-22"), 320 | ("1984-01-16","1984-03-30"), 321 | ("1984-04-02","1984-06-29"), 322 | ("1984-07-02","1984-08-01"), 323 | ("1984-10-22","1984-12-21"), 324 | ("1985-01-09","1985-03-29"), 325 | ("1985-04-01","1985-06-28"), 326 | ("1985-07-01","1985-07-26"), 327 | ("1985-10-21","1985-12-20"), 328 | ("1986-01-13","1986-03-27"), 329 | ("1986-04-08","1986-06-30"), 330 | ("1986-07-01","1986-07-25"), 331 | ("1986-10-21","1986-12-19"), 332 | ("1987-01-12","1987-03-31"), 333 | ("1987-04-01","1987-06-30"), 334 | ("1987-07-01","1987-07-24"), 335 | ("1987-10-21","1987-12-18"), 336 | ("1988-01-11","1988-03-31"), 337 | ("1988-04-12","1988-06-30"), 338 | ("1988-07-01","1988-07-29"), 339 | ("1988-10-19","1988-12-22"), 340 | ("1989-01-10","1989-03-23"), 341 | ("1989-04-04","1989-06-30"), 342 | ("1989-07-03","1989-07-28"), 343 | ("1989-10-17","1989-12-21"), 344 | ("1990-01-08","1990-03-30"), 345 | ("1990-04-02","1990-06-29"), 346 | ("1990-07-02","1990-09-07"), 347 | ("1990-10-15","1990-12-20"), 348 | ("1991-01-14","1991-03-28"), 349 | ("1991-04-15","1991-06-28"), 350 | ("1991-07-01","1991-08-17"), 351 | ("1991-10-14","1991-12-20"), 352 | ("1992-01-13","1992-03-16"), 353 | ("1992-04-27","1992-06-30"), 354 | ("1992-07-01","1992-09-25"), 355 | ("1992-10-19","1992-12-17"), 356 | ("1993-01-11","1993-03-31"), 357 | ("1993-04-01","1993-06-30"), 358 | ("1993-07-01","1993-07-27"), 359 | ("1993-10-18","1993-12-17"), 360 | ("1994-01-11","1994-03-31"), 361 | ("1994-04-12","1994-06-30"), 362 | ("1994-07-01","1994-07-21"), 363 | ("1994-10-17","1994-12-20"), 364 | ("1995-01-10","1995-03-31"), 365 | ("1995-04-03","1995-06-30"), 366 | ("1995-07-03","1995-07-19"), 367 | ("1995-10-16","1995-12-20"), 368 | ("1996-01-09","1996-03-29"), 369 | ("1996-04-01","1996-06-27"), 370 | ("1996-07-01","1996-07-24"), 371 | ("1996-10-14","1996-12-18"), 372 | ("1997-01-13","1997-03-21"), 373 | ("1997-05-07","1997-06-30"), 374 | ("1997-07-01","1997-07-31"), 375 | ("1997-10-27","1997-12-22"), 376 | ("1998-01-12","1998-03-31"), 377 | ("1998-04-01","1998-06-30"), 378 | ("1998-07-01","1998-09-03"), 379 | ("1998-10-19","1998-12-17"), 380 | ("1999-01-11","1999-03-31"), 381 | ("1999-04-13","1999-06-30"), 382 | ("1999-07-01","1999-07-27"), 383 | ("1999-10-19","1999-12-21"), 384 | ("2000-01-10","2000-03-30"), 385 | ("2000-04-03","2000-06-29"), 386 | ("2000-07-03","2000-07-28"), 387 | ("2000-10-23","2000-12-21"), 388 | ("2001-01-08","2001-03-30"), 389 | ("2001-04-02","2001-06-28"), 390 | ("2001-07-02","2001-09-14"), 391 | ("2001-10-04","2001-12-19"), 392 | ("2002-01-08","2002-03-26"), 393 | ("2002-04-03","2002-06-27"), 394 | ("2002-07-01","2002-09-24"), 395 | ("2002-10-15","2002-12-19"), 396 | ("2003-01-07","2003-03-31"), 397 | ("2003-04-01","2003-06-30"), 398 | ("2003-07-01","2003-09-18"), 399 | ("2003-10-14","2003-12-18"), 400 | ("2004-01-05","2004-03-31"), 401 | ("2004-04-01","2004-06-30"), 402 | ("2004-07-01","2004-09-16"), 403 | ("2004-10-11","2004-12-21"), 404 | ("2005-01-10","2005-03-24"), 405 | ("2005-04-04","2005-06-30"), 406 | ("2005-07-04","2005-07-21"), 407 | ("2005-10-10","2005-12-20"), 408 | ("2006-01-09","2006-03-30"), 409 | ("2006-04-18","2006-06-29"), 410 | ("2006-07-03","2006-07-25"), 411 | ("2006-10-09","2006-12-19"), 412 | ("2007-01-08","2007-03-29"), 413 | ("2007-04-16","2007-06-29"), 414 | ("2007-07-02","2007-07-26"), 415 | ("2007-10-08","2007-12-18"), 416 | ("2008-01-07","2008-03-31"), 417 | ("2008-04-01","2008-06-30"), 418 | ("2008-07-01","2008-07-22"), 419 | ("2008-10-06","2008-12-18"), 420 | ("2009-01-12","2009-03-31"), 421 | ("2009-04-01","2009-06-30"), 422 | ("2009-07-01","2009-07-21"), 423 | ("2009-10-12","2009-12-16"), 424 | ("2010-01-05","2010-03-30"), 425 | ("2010-04-06","2010-06-30"), 426 | ("2010-07-01","2010-09-16"), 427 | ("2010-10-11","2010-12-21"), 428 | ("2011-01-10","2011-03-31"), 429 | ("2011-04-01","2011-06-30"), 430 | ("2011-07-04","2011-09-15"), 431 | ("2011-10-10","2011-12-20"), 432 | ("2012-01-10","2012-03-27"), 433 | ("2012-04-16","2012-06-28"), 434 | ("2012-07-02","2012-09-18"), 435 | ("2012-10-15","2012-12-20"), 436 | ("2013-01-07","2013-03-26"), 437 | ("2013-04-10","2013-06-27"), 438 | ("2013-07-01","2013-09-13"), 439 | ("2013-10-08","2013-12-19")] 440 | for b,e in listoffiles: 441 | XML_to_TSV(b,e,str(sys.argv[1]),str(sys.argv[2])).makeTable() 442 | 443 | -------------------------------------------------------------------------------- /emotion-final-q.csv: -------------------------------------------------------------------------------- 1 | year,quarter,date,polar,dpolar,recession,election,wars,conservatives,polarg,polaro,ldisp,unemp,misery,gdpg,ww1,ww2,angloirish,korean,falkland,irak,afghan,troubles,troubles69,troubles72,nrcpol,ofpol,swnpol 2 | 1909,1,1909q1,-1.7899935,,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-1.7941715,-1.34937,-1.496798 3 | 1909,2,1909q2,-1.3339995,0.45599401,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-1.4601367,-0.81470871,0.2709021 4 | 1909,3,1909q3,-1.097386,0.23661351,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-1.3098654,-0.48125809,0.40986946 5 | 1909,4,1909q4,-1.3777519,-0.28036594,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-2.1553297,-1.0843973,0.2169802 6 | 1910,1,1910q1,-0.45878786,0.91896409,,1,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-1.0613828,-0.026695265,1.1101639 7 | 1910,2,1910q2,-0.94454336,-0.4857555,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-0.91390097,-0.59158027,0.51187307 8 | 1910,3,1910q3,-0.66739833,0.27714503,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-1.0483601,-0.32925415,1.0012857 9 | 1910,4,1910q4,-0.62635881,0.041039526,,1,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-0.8826946,-0.10503867,0.85749114 10 | 1911,1,1911q1,-0.58531934,0.041039467,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-0.71702904,0.11917679,0.7136966 11 | 1911,2,1911q2,-0.49273345,0.092585891,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-0.58437735,0.076288827,0.92135209 12 | 1911,3,1911q3,-0.471333,0.021400452,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-0.073570743,0.4370614,0.62469113 13 | 1911,4,1911q4,-0.74337345,-0.27204046,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-0.97067481,0.17036192,0.15210246 14 | 1912,1,1912q1,-0.60013789,0.14323556,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-0.48095655,-0.073513724,1.3423836 15 | 1912,2,1912q2,-0.94372088,-0.34358299,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-1.0262803,-0.63946545,0.3250066 16 | 1912,3,1912q3,-0.83424461,0.10947627,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-0.90014911,-0.24029499,0.37569034 17 | 1912,4,1912q4,-0.99533963,-0.16109502,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-2.0396531,-0.91564667,0.31772962 18 | 1913,1,1913q1,-0.85459673,0.1407429,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-0.64590085,-0.24970435,0.50677705 19 | 1913,2,1913q2,-1.0058836,-0.15128684,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-1.0386735,-0.618967,0.4860146 20 | 1913,3,1913q3,-0.81878871,0.18709487,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-0.20838909,-0.092016019,0.67229378 21 | 1913,4,1913q4,-1.4667537,-0.64796501,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-0.75985545,-1.0425844,-0.16525584 22 | 1914,1,1914q1,-2.1147187,-0.64796495,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-1.3113219,-1.9931529,-1.0028056 23 | 1914,2,1914q2,-0.8981548,1.2165639,,0,0,,,,,,,,0,0,0,0,0,0,0,0,0,0,-1.0123218,-0.70727026,0.1726888 24 | 1914,3,1914q3,-1.1767066,-0.27855176,,0,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-1.782757,-0.43469793,0.24874879 25 | 1914,4,1914q4,-1.5917035,-0.41499698,,0,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-2.1914229,-0.78727245,-0.35018966 26 | 1915,1,1915q1,-1.079069,0.51263452,,0,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-1.1749815,-0.85905069,0.90974694 27 | 1915,2,1915q2,-0.28407428,0.79499471,,0,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-1.2300086,-0.14870615,1.4406995 28 | 1915,3,1915q3,-0.5251236,-0.24104932,,0,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-1.7475681,-0.49970517,1.4984294 29 | 1915,4,1915q4,-1.0751742,-0.55005062,,0,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-2.9203479,-0.93641609,0.15437691 30 | 1916,1,1916q1,-1.2044917,-0.12931752,,0,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-2.5340102,-1.6973401,-0.78610587 31 | 1916,2,1916q2,-1.4251262,-0.22063446,,0,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-3.2066321,-1.2804822,-0.44915074 32 | 1916,3,1916q3,-0.71596575,0.70916045,,0,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-1.0468513,-0.39670727,0.43012604 33 | 1916,4,1916q4,-1.1499323,-0.43396652,,0,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-1.3274976,-1.041357,0.069448009 34 | 1917,1,1917q1,-1.0414318,0.10850048,,0,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-1.607986,-1.2089965,0.31164515 35 | 1917,2,1917q2,-0.86446571,0.17696607,,0,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-1.556161,-0.71879536,0.44722891 36 | 1917,3,1917q3,-1.5030117,-0.63854599,,0,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-2.1064861,-1.1168376,0.6543017 37 | 1917,4,1917q4,-0.7125634,0.79044831,,0,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-1.6051277,-0.61353213,0.43412513 38 | 1918,1,1918q1,-0.92838913,-0.21582574,,0,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-0.68052745,-0.9152689,0.86184174 39 | 1918,2,1918q2,-0.55256689,0.37582225,,0,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-1.0040933,-0.56113422,1.1244736 40 | 1918,3,1918q3,-0.51947582,0.033091068,,0,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-0.6580295,-0.51459986,1.0541469 41 | 1918,4,1918q4,-0.65681726,-0.13734144,,1,1,,,,,,,,1,0,0,0,0,0,0,0,0,0,-0.6730172,-0.93245459,-0.022924878 42 | 1919,1,1919q1,-0.30084676,0.3559705,,0,1,,,,,,,,0,0,1,0,0,0,0,0,0,0,-0.25295028,0.34581324,1.4278132 43 | 1919,2,1919q2,-0.66270506,-0.36185831,,0,1,,,,,,,,0,0,1,0,0,0,0,0,0,0,-0.21423745,-0.075059742,0.86882222 44 | 1919,3,1919q3,-0.48217186,0.1805332,,0,1,,,,,,,,0,0,1,0,0,0,0,0,0,0,-0.565422,0.053265758,1.3054256 45 | 1919,4,1919q4,-0.73879105,-0.25661919,,0,1,,,,,,,,0,0,1,0,0,0,0,0,0,0,-1.0706346,-0.30187008,0.40122423 46 | 1920,1,1920q1,-1.0067958,-0.26800472,,0,1,,,,,,,,0,0,1,0,0,0,0,0,0,0,-1.1333232,-0.67728168,0.25553089 47 | 1920,2,1920q2,-1.1115118,-0.10471606,0,0,1,,,,,,,-0.088850975,0,0,1,0,0,0,0,0,0,0,-1.4819576,-0.73972321,0.26715457 48 | 1920,3,1920q3,-0.9631021,0.14840972,1,0,1,,,,,,,-0.3807815,0,0,1,0,0,0,0,0,0,0,-1.7588946,-0.31506491,-0.095902845 49 | 1920,4,1920q4,-1.8016675,-0.83856535,1,0,1,,,,,,,-3.1512964,0,0,1,0,0,0,0,0,0,0,-1.9409994,-1.4581213,-1.2469261 50 | 1921,1,1921q1,-1.8817302,-0.080062747,1,0,1,,,,,,,-2.385999,0,0,1,0,0,0,0,0,0,0,-1.3007742,-0.97111964,-2.2536645 51 | 1921,2,1921q2,-1.3434745,0.53825569,1,0,1,,,,,,,-8.0481958,0,0,1,0,0,0,0,0,0,0,-1.5464946,-0.87319481,-0.66724837 52 | 1921,3,1921q3,-0.36860126,0.97487324,0,0,1,,,,,,,8.2365055,0,0,1,0,0,0,0,0,0,0,0.62860328,1.1571846,1.5766891 53 | 1921,4,1921q4,-1.8500729,-1.4814715,0,0,0,,,,,,,-1.1561239,0,0,0,0,0,0,0,0,0,0,-1.4636565,-1.2283438,-2.4916074 54 | 1922,1,1922q1,-1.3816466,0.46842623,0,0,0,,,,,,,0.054289155,0,0,0,0,0,0,0,0,0,0,-1.6693054,-0.74497378,-1.03969 55 | 1922,2,1922q2,-1.0060443,0.37560236,0,0,0,,,,,,,1.0612925,0,0,0,0,0,0,0,0,0,0,-1.2145556,-0.27410734,-0.41375792 56 | 1922,3,1922q3,-0.97739053,0.028653741,0,0,0,,,,,,,0.96147114,0,0,0,0,0,0,0,0,0,0,-1.0063597,-0.61397249,-0.40951037 57 | 1922,4,1922q4,-1.3046657,-0.32727516,0,1,0,,,,,,,-0.92885751,0,0,0,0,0,0,0,0,0,0,-1.4347379,-0.4623237,-1.7572778 58 | 1923,1,1923q1,-0.99543691,0.30922878,0,0,0,,,,,,,0.13150948,0,0,0,0,0,0,0,0,0,0,-1.1546397,-0.50813293,-1.7819847 59 | 1923,2,1923q2,-1.0870864,-0.091649532,0,0,0,,,,,,,0.35375339,0,0,0,0,0,0,0,0,0,0,-1.364777,-0.64153129,-0.34214461 60 | 1923,3,1923q3,-0.88370812,0.20337832,0,0,0,,,,,,,-0.22382961,0,0,0,0,0,0,0,0,0,0,-0.54213643,-0.12147678,-0.052398436 61 | 1923,4,1923q4,-0.56064761,0.32306051,0,1,0,,,,,,,0.43097371,0,0,0,0,0,0,0,0,0,0,-0.47533467,0.25302073,0.020002408 62 | 1924,1,1924q1,-0.23758709,0.32306051,0,0,0,,,,,,,0.21061325,0,0,0,0,0,0,0,0,0,0,-0.40853295,0.6275183,0.092403173 63 | 1924,2,1924q2,-0.29893002,-0.061342925,0,0,0,,,,,,,0.98972249,0,0,0,0,0,0,0,0,0,0,-0.46071026,0.28762054,0.20901185 64 | 1924,3,1924q3,-0.23173638,0.067193642,0,0,0,,,,,,,0.44352987,0,0,0,0,0,0,0,0,0,0,-0.21575701,0.44509578,0.40398884 65 | 1924,4,1924q4,-2.030164,-1.7984276,0,1,0,,,,,,,0.088190764,0,0,0,0,0,0,0,0,0,0,-2.4269342,-0.91801316,-2.1340439 66 | 1925,1,1925q1,-0.33426538,1.6958987,0,0,0,,,,,,,0.35626462,0,0,0,0,0,0,0,0,0,0,0.13572776,0.37626642,0.69339567 67 | 1925,2,1925q2,-0.68526655,-0.35100117,0,0,0,,,,,,,-0.083200708,0,0,0,0,0,0,0,0,0,0,-0.28801602,-0.26033312,-0.59029883 68 | 1925,3,1925q3,-0.86352205,-0.1782555,0,0,0,,,,,,,-0.32867348,0,0,0,0,0,0,0,0,0,0,-0.75314486,-0.31146258,-1.4318595 69 | 1925,4,1925q4,-0.74525791,0.11826414,0,0,0,,,,,,,0.44039083,0,0,0,0,0,0,0,0,0,0,-1.4052066,-0.22484598,-0.79220784 70 | 1926,1,1926q1,-0.3076227,0.43763521,0,0,0,,,,,,,0.31545711,0,0,0,0,0,0,0,0,0,0,0.058675226,0.2947588,0.13462742 71 | 1926,2,1926q2,-1.4375225,-1.1298999,1,0,0,,,,,,,-4.2292418,0,0,0,0,0,0,0,0,0,0,-1.2106569,-0.7401672,-0.26570344 72 | 1926,3,1926q3,-1.2479299,0.1895926,1,0,0,,,,,,,-2.1574769,0,0,0,0,0,0,0,0,0,0,-1.1987017,-0.37616298,0.21980236 73 | 1926,4,1926q4,-1.3664565,-0.11852658,0,0,0,,,,,,,1.9000435,0,0,0,0,0,0,0,0,0,0,-1.1069969,-1.1504244,-0.82505089 74 | 1927,1,1927q1,-0.92296028,0.44349623,0,0,0,,,,,,,4.1708236,0,0,0,0,0,0,0,0,0,0,-0.78728759,-0.37350374,0.069324605 75 | 1927,2,1927q2,-2.1597588,-1.2367985,0,0,0,,,,,,,0.46487531,0,0,0,0,0,0,0,0,0,0,-2.7895577,-1.8281969,-0.71146286 76 | 1927,3,1927q3,-1.3765501,0.78320873,0,0,0,,,,,,,-0.28598255,0,0,0,0,0,0,0,0,0,0,-1.4235173,-1.0796441,-1.4490242 77 | 1927,4,1927q4,-1.5114796,-0.13492954,0,0,0,,,,,,,-0.64320511,0,0,0,0,0,0,0,0,0,0,-1.2876405,-0.62912226,-3.195972 78 | 1928,1,1928q1,-0.62796301,0.88351661,0,0,0,,,,,,,-0.075039208,0,0,0,0,0,0,0,0,0,0,-0.75732464,-0.21831523,-0.34725922 79 | 1928,2,1928q2,-0.77318561,-0.1452226,0,0,0,,,,,,,-0.0084916037,0,0,0,0,0,0,0,0,0,0,-0.31068546,-0.20146984,0.21475929 80 | 1928,3,1928q3,-0.21944176,0.55374384,0,0,0,,,,,,,-0.45235157,0,0,0,0,0,0,0,0,0,0,0.43809012,0.59787041,0.76259154 81 | 1928,4,1928q4,-0.74851102,-0.52906924,0,0,0,,,,,,,-0.14723708,0,0,0,0,0,0,0,0,0,0,-0.25948551,-0.13306537,-1.2580466 82 | 1929,1,1929q1,-0.50796026,0.24055076,0,0,0,,,,,,,0.27025497,0,0,0,0,0,0,0,0,0,0,-0.018716443,-0.30949068,-0.76796091 83 | 1929,2,1929q2,-0.065789089,0.44217116,0,1,0,,,,,,,0.51635551,0,0,0,0,0,0,0,0,0,0,0.66748184,0.64253217,0.37050524 84 | 1929,3,1929q3,-0.017704956,0.048084132,0,0,0,,,,,,,0.23007528,0,0,0,0,0,0,0,0,0,0,-0.015209666,0.2053787,0.39221531 85 | 1929,4,1929q4,-0.82315373,-0.80544877,0,0,0,,,,,,,-0.3054446,0,0,0,0,0,0,0,0,0,0,-0.96925223,-0.3722192,-0.84036428 86 | 1930,1,1930q1,-0.16551414,0.65763962,1,0,0,,,,,,,-0.33746278,0,0,0,0,0,0,0,0,0,0,-0.42199463,0.1374272,0.097383641 87 | 1930,2,1930q2,-0.97305256,-0.80753839,1,0,0,,,,,,,-0.89683932,0,0,0,0,0,0,0,0,0,0,-0.81998861,-0.5463984,-0.80306852 88 | 1930,3,1930q3,-0.92713743,0.045915127,1,0,0,,,,,,,-1.1159441,0,0,0,0,0,0,0,0,0,0,-1.0574378,-0.59491593,-1.0407759 89 | 1930,4,1930q4,-1.0155019,-0.088364422,1,0,0,,,,,,,-1.4292202,0,0,0,0,0,0,0,0,0,0,-0.10173272,-0.38825154,-0.67467511 90 | 1931,1,1931q1,-0.26103896,0.7544629,1,0,0,,,,1.940177,,,-1.7016886,0,0,0,0,0,0,0,0,0,0,0.017566683,0.24453013,0.3480013 91 | 1931,2,1931q2,-0.72949821,-0.46845925,1,0,0,,,,0.012309061,,,-0.7279591,0,0,0,0,0,0,0,0,0,0,-0.66232342,-0.30203214,-0.57762617 92 | 1931,3,1931q3,-1.6501123,-0.92061406,1,0,0,,,,0.4615359,,,-0.57916868,0,0,0,0,0,0,0,0,0,0,-1.1199503,-0.91634411,-2.0698452 93 | 1931,4,1931q4,-1.4199282,0.23018408,0,1,0,,,,-0.78017962,,,0.10451376,0,0,0,0,0,0,0,0,0,0,-1.4128394,-0.29766226,-1.7493896 94 | 1932,1,1932q1,-0.31679311,1.1031351,0,0,0,,,,-0.0071026273,,,-0.238897,0,0,0,0,0,0,0,0,0,0,-1.3978034,0.07728973,-0.41053674 95 | 1932,2,1932q2,-1.0626247,-0.74583161,1,0,0,,,,-0.45923856,,,-0.76311636,0,0,0,0,0,0,0,0,0,0,-1.1185347,-0.48097107,-0.82867807 96 | 1932,3,1932q3,-0.99917644,0.06344825,1,0,0,,,,1.8274803,,,-0.66078371,0,0,0,0,0,0,0,0,0,0,-1.5402206,-0.31265405,-0.98939586 97 | 1932,4,1932q4,-0.92947346,0.069702983,0,0,0,,,,0.64649856,,,0.66577375,0,0,0,0,0,0,0,0,0,0,-1.2033617,-0.20753202,-1.5132879 98 | 1933,1,1933q1,-0.63371336,0.2957601,0,0,0,,,,-0.10024602,,,-0.37575904,0,0,0,0,0,0,0,0,0,0,-0.75350791,-0.1544642,-0.34155786 99 | 1933,2,1933q2,-0.85991281,-0.22619945,0,0,0,,,,-0.36718413,,,0.62810528,0,0,0,0,0,0,0,0,0,0,-1.5568694,-0.18973526,-0.84304816 100 | 1933,3,1933q3,-0.18773477,0.67217803,0,0,0,,,,-0.32780534,,,0.6852358,0,0,0,0,0,0,0,0,0,0,-0.013002083,0.55535513,0.50156164 101 | 1933,4,1933q4,-1.0820394,-0.89430457,0,0,0,,,,-0.69957078,,,0.57725286,0,0,0,0,0,0,0,0,0,0,-1.7612709,-0.6339097,-2.0551662 102 | 1934,1,1934q1,-0.74423587,0.33780348,0,0,0,,,,-0.59874862,,,0.84595454,0,0,0,0,0,0,0,0,0,0,-0.81350595,-0.28018042,-1.0292975 103 | 1934,2,1934q2,-1.2834848,-0.53924894,0,0,0,,,,-0.24732782,,,0.52075022,0,0,0,0,0,0,0,0,0,0,-1.5249697,-0.53790051,-1.3603292 104 | 1934,3,1934q3,-0.41286382,0.87062097,0,0,0,,,,-0.52397668,,,0.18926778,0,0,0,0,0,0,0,0,0,0,-0.87321764,-0.078716069,-0.37637311 105 | 1934,4,1934q4,-1.1208441,-0.70798028,0,0,0,,,,-0.39064971,,,-0.021047756,0,0,0,0,0,0,0,0,0,0,-1.8005102,-0.76293367,-1.3433057 106 | 1935,1,1935q1,-0.62743211,0.49341202,0,0,0,,,,-0.13123409,,,-0.099523708,0,0,0,0,0,0,0,0,0,0,-0.85173708,-0.26813737,-0.36057225 107 | 1935,2,1935q2,-0.14818339,0.4792487,0,0,0,,,,-0.10634217,,,0.71097589,0,0,0,0,0,0,0,0,0,0,-0.38747019,0.43917125,0.26923543 108 | 1935,3,1935q3,-0.58714181,-0.43895841,0,0,0,,,,-0.088200241,,,0.31922397,0,0,0,0,0,0,0,0,0,0,-0.43181649,-0.052913819,-0.89302069 109 | 1935,4,1935q4,-0.52898121,0.058160603,0,1,0,,,,0.52574915,,,0.29160044,0,0,0,0,0,0,0,0,0,0,-0.89380193,-0.20200568,-1.5722746 110 | 1936,1,1936q1,-0.51632935,0.012651861,0,0,0,,,,0.13867702,,,0.30415657,0,0,0,0,0,0,0,0,0,0,-0.39532825,-0.37735489,-0.70366591 111 | 1936,2,1936q2,-0.6268,-0.11047065,0,0,0,,,,0.12079908,,,0.68837482,0,0,0,0,0,0,0,0,0,0,-0.79248297,-0.28904408,-0.66931707 112 | 1936,3,1936q3,-0.55135316,0.075446844,0,0,0,,,,-0.18363938,,,0.44918013,0,0,0,0,0,0,0,0,0,0,-0.44026989,-0.21333753,-1.1941427 113 | 1936,4,1936q4,-0.63386917,-0.082516015,0,0,0,,,,0.10710561,,,-0.28409913,0,0,0,0,0,0,0,0,0,0,-0.17283657,-0.22053854,-0.86906374 114 | 1937,1,1937q1,-0.50022876,0.13364041,0,0,0,,,,0.27311316,,,0.13025388,0,0,0,0,0,0,0,0,0,0,-0.76088828,-0.31845939,-0.44002908 115 | 1937,2,1937q2,-0.37754971,0.12267905,0,0,0,,,,0.888255,,,0.56720793,0,0,0,0,0,0,0,0,0,0,-0.62908536,0.12742311,0.54355073 116 | 1937,3,1937q3,-0.44086727,-0.063317567,0,0,0,,,,0.63442457,,,0.22442502,0,0,0,0,0,0,0,0,0,0,-0.97071427,-0.099610545,0.35509735 117 | 1937,4,1937q4,-0.58970875,-0.14884147,0,0,0,,,,0.014049127,,,-0.36948097,0,0,0,0,0,0,0,0,0,0,-0.40595195,-0.45476878,-0.61297351 118 | 1938,1,1938q1,-0.32477865,0.2649301,0,0,0,,,,-0.13971822,,,-0.34436867,0,0,0,0,0,0,0,0,0,0,-0.32399029,-0.23771468,-0.13266763 119 | 1938,2,1938q2,-0.57155401,-0.24677536,0,0,0,,,,0.14454296,,,-0.60993129,0,0,0,0,0,0,0,0,0,0,-0.75212991,-0.21808633,-0.27343878 120 | 1938,3,1938q3,-0.57426339,-0.0027093887,0,0,0,,,,-0.37882525,,,-0.19871731,0,0,0,0,0,0,0,0,0,0,-0.088071711,-0.21198894,-0.38074455 121 | 1938,4,1938q4,-0.97454673,-0.40028334,0,0,0,,,,-0.52042663,,,0.0021811251,0,0,0,0,0,0,0,0,0,0,-1.2679725,-0.91036737,-1.4074051 122 | 1939,1,1939q1,-0.21535912,0.75918758,,0,0,,,,-0.26981714,,,,0,0,0,0,0,0,0,0,0,0,0.036750052,0.15578647,-0.49266157 123 | 1939,2,1939q2,-0.2342982,-0.018939078,,0,0,,,,-0.021542257,,,,0,0,0,0,0,0,0,0,0,0,-0.33401293,0.25662231,-0.13768262 124 | 1939,3,1939q3,-0.61681074,-0.38251254,,0,1,,,,-0.13123409,,,,0,1,0,0,0,0,0,0,0,0,-1.2074078,-0.58899873,-0.59930909 125 | 1939,4,1939q4,-0.78336775,-0.16655701,,0,1,,,,-0.28003278,,,,0,1,0,0,0,0,0,0,0,0,-1.298211,-1.0408945,-0.70852643 126 | 1940,1,1940q1,-0.20857058,0.57479715,,0,1,,,,0.043029431,,,,0,1,0,0,0,0,0,0,0,0,-0.67704535,-0.68008673,-0.10427163 127 | 1940,2,1940q2,-0.61862993,-0.41005933,,0,1,,,,-0.59874862,,,,0,1,0,0,0,0,0,0,0,0,-1.1631308,-0.98219663,0.30116946 128 | 1940,3,1940q3,-0.55447745,0.064152479,,0,1,,,,-1.0624242,,,,0,1,0,0,0,0,0,0,0,0,-1.2341228,-0.82138497,-0.15938032 129 | 1940,4,1940q4,-0.35608876,0.1983887,,0,1,,,,-0.57552499,,,,0,1,0,0,0,0,0,0,0,0,-0.2246196,-0.25447723,-0.092932791 130 | 1941,1,1941q1,-0.50037086,-0.1442821,,0,1,,,,-0.18590434,,,,0,1,0,0,0,0,0,0,0,0,-0.878573,-0.98764944,0.6376496 131 | 1941,2,1941q2,-1.0782226,-0.57785177,,0,1,,,,-0.12284476,,,,0,1,0,0,0,0,0,0,0,0,-1.5256679,-1.4576945,-0.28092939 132 | 1941,3,1941q3,-0.66170502,0.41651762,,0,1,,,,-0.7131815,,,,0,1,0,0,0,0,0,0,0,0,-0.84071738,-1.1535258,0.094946891 133 | 1941,4,1941q4,-0.21835613,0.44334888,,0,1,,,,-0.47908565,,,,0,1,0,0,0,0,0,0,0,0,-0.88478315,-0.80531031,0.47196802 134 | 1942,1,1942q1,-0.82262343,-0.6042673,,0,1,,,,-0.87049502,,,,0,1,0,0,0,0,0,0,0,0,-1.0012252,-1.5580314,-0.16753687 135 | 1942,2,1942q2,-0.8453685,-0.022745073,,0,1,,,,0.41613635,,,,0,1,0,0,0,0,0,0,0,0,-1.2952627,-1.42705,-0.090288669 136 | 1942,3,1942q3,-1.024008,-0.17863953,,0,1,,,,-0.85928828,,,,0,1,0,0,0,0,0,0,0,0,-1.2482971,-1.6249025,-0.40406498 137 | 1942,4,1942q4,-0.54720026,0.47680777,,0,1,,,,0.15469937,,,,0,1,0,0,0,0,0,0,0,0,-1.1271511,-1.0946555,0.010924092 138 | 1943,1,1943q1,-0.0002700121,0.54693025,,0,1,,,,-0.5275436,,,,0,1,0,0,0,0,0,0,0,0,-0.037639048,-0.19423418,1.0645483 139 | 1943,2,1943q2,-0.056942448,-0.056672435,,0,1,,,,-0.094198972,,,,0,1,0,0,0,0,0,0,0,0,-0.12395386,-0.31714508,0.69413042 140 | 1943,3,1943q3,-0.16630171,-0.10935926,,0,1,,,,0.22087099,,,,0,1,0,0,0,0,0,0,0,0,-0.46590641,-0.55113631,0.57214969 141 | 1943,4,1943q4,-0.86702925,-0.70072752,,0,1,,,,0.3254078,,,,0,1,0,0,0,0,0,0,0,0,-1.7411679,-0.97614598,-0.92177051 142 | 1944,1,1944q1,0.51667511,1.3837044,,0,1,,,,1.2048908,,,,0,1,0,0,0,0,0,0,0,0,0.71342391,0.48560259,1.4551076 143 | 1944,2,1944q2,0.52038872,0.0037136078,,0,1,,,,0.48231512,,,,0,1,0,0,0,0,0,0,0,0,0.21214426,0.35628906,1.5380912 144 | 1944,3,1944q3,0.17051311,-0.34987563,,0,1,,,,-0.090194501,,,,0,1,0,0,0,0,0,0,0,0,-0.18707679,0.081926756,0.64344686 145 | 1944,4,1944q4,-0.15380707,-0.3243202,,0,1,,,,-0.0035356963,,,,0,1,0,0,0,0,0,0,0,0,-0.91292053,-0.337899,-0.35772043 146 | 1945,1,1945q1,0.29015121,0.44395828,,0,1,,,,0.21690179,,,,0,1,0,0,0,0,0,0,0,0,0.1234429,0.036986496,0.69841838 147 | 1945,2,1945q2,0.33620274,0.046051532,,0,1,,,,0.010564955,,,,0,1,0,0,0,0,0,0,0,0,0.28490421,0.17488219,0.32945159 148 | 1945,3,1945q3,1.0316343,0.69543159,,1,0,0,-0.39829323,2.170763,-0.055084005,,,,0,0,0,0,0,0,0,0,0,0,0.65951937,1.5413812,2.4786327 149 | 1945,4,1945q4,0.30977041,-0.72186393,,0,0,0,-0.81551981,1.0912755,0.92167825,,,,0,0,0,0,0,0,0,0,0,0,-0.44032526,0.1130207,0.76378012 150 | 1946,1,1946q1,-0.059956796,-0.36972719,,0,0,0,-1.0431004,0.49060068,0.23395211,,,,0,0,0,0,0,0,0,0,0,0,-0.41279879,-0.2977756,0.82409137 151 | 1946,2,1946q2,0.46726424,0.52722102,,0,0,0,-0.5613879,1.098421,0.24680923,,,,0,0,0,0,0,0,0,0,0,0,0.86152858,0.68037182,1.3859235 152 | 1946,3,1946q3,0.070386924,-0.39687732,,0,0,0,-0.8265627,0.45165285,0.15036343,,,,0,0,0,0,0,0,0,0,0,0,0.28578106,0.070405364,0.6127224 153 | 1946,4,1946q4,-0.13106486,-0.20145178,,0,0,0,-1.0950959,0.35319689,0.083713144,,,,0,0,0,0,0,0,0,0,0,0,-0.1047628,-0.42345428,-0.14659737 154 | 1947,1,1947q1,-0.021655831,0.10940903,,0,0,0,-0.80760258,0.26836237,0.35798475,,,,0,0,0,0,0,0,0,0,0,0,0.044376392,-0.45545092,0.48703989 155 | 1947,2,1947q2,-0.16985682,-0.14820099,,0,0,0,-0.99030554,0.23235895,0.30327132,,,,0,0,0,0,0,0,0,0,0,0,-0.66156828,-0.39295328,0.77462655 156 | 1947,3,1947q3,-0.39797795,-0.22812113,,0,0,0,-1.1285669,-0.15991229,0.3150034,,,,0,0,0,0,0,0,0,0,0,0,-0.33851635,-0.59285921,0.46320289 157 | 1947,4,1947q4,-0.84245265,-0.4444747,,0,0,0,-1.5222842,-0.56406224,0.075750843,,,,0,0,0,0,0,0,0,0,0,0,-1.2399979,-1.1318398,-0.28494522 158 | 1948,1,1948q1,0.26280338,1.1052561,,0,0,0,-0.70012993,0.87773329,0.2594499,,,,0,0,0,0,0,0,0,0,0,0,0.16429798,-0.094890945,0.58193475 159 | 1948,2,1948q2,-0.62161231,-0.88441569,,0,0,0,-1.2673309,-0.50172102,0.63137543,,,,0,0,0,0,0,0,0,0,0,0,-1.2679368,-0.93074864,-0.24279578 160 | 1948,3,1948q3,-0.82266271,-0.2010504,,0,0,0,-1.2642668,-0.98820335,-0.60665226,,,,0,0,0,0,0,0,0,0,0,0,-1.0925566,-1.3107831,-1.1738243 161 | 1948,4,1948q4,-0.14831594,0.6743468,,0,0,0,-0.99643338,0.24035138,-0.66874027,,,,0,0,0,0,0,0,0,0,0,0,-0.22725059,-0.62886339,0.3921527 162 | 1949,1,1949q1,0.081000715,0.22931665,,0,0,0,-0.91540563,0.68915468,-0.42105216,,,,0,0,0,0,0,0,0,0,0,0,-0.1407996,-0.21284793,0.56083035 163 | 1949,2,1949q2,-0.21571769,-0.29671842,,0,0,0,-1.0719196,0.24557221,0.35688463,,,,0,0,0,0,0,0,0,0,0,0,-0.15678734,-0.43871179,0.7408393 164 | 1949,3,1949q3,-0.9212184,-0.70550072,,0,0,0,-1.7472472,-0.47365376,0.35798475,,,,0,0,0,0,0,0,0,0,0,0,-0.80906975,-1.4883364,-0.46532035 165 | 1949,4,1949q4,-0.53740549,0.3838129,,0,0,0,-1.5203784,-0.030213926,-0.5641771,,,,0,0,0,0,0,0,0,0,0,0,-0.63145709,-0.50585407,0.45985746 166 | 1950,1,1950q1,0.1128697,0.65027517,,1,0,0,-0.82500529,0.65758777,-0.30353919,,,,0,0,0,0,0,0,0,0,0,0,0.32530278,-0.25742012,0.57909453 167 | 1950,2,1950q2,-0.049532749,-0.16240245,,0,1,0,-0.78798616,0.20750946,-0.24732782,,,,0,0,0,1,0,0,0,0,0,0,-0.19796076,-0.27323329,0.90007895 168 | 1950,3,1950q3,-0.29001096,-0.24047822,,0,1,0,-1.2017361,0.20352565,0.056289602,,,,0,0,0,1,0,0,0,0,0,0,-0.1715115,-0.60293931,0.31217736 169 | 1950,4,1950q4,-0.025296817,0.26471415,,0,1,0,-1.0721943,0.54038149,-0.15916546,,,,0,0,0,1,0,0,0,0,0,0,0.27072912,-0.23052269,0.44363111 170 | 1951,1,1951q1,-0.19182467,-0.16652785,,0,1,0,-1.1551325,0.41036519,0.21023948,,,,0,0,0,1,0,0,0,0,0,0,-0.7681514,-0.73517352,0.22997341 171 | 1951,2,1951q2,-0.70522058,-0.51339591,,0,1,0,-1.6300087,-0.15089257,0.33226481,,,,0,0,0,1,0,0,0,0,0,0,-1.0782889,-0.91862208,-0.15751982 172 | 1951,3,1951q3,-0.85868043,-0.15345985,,0,1,0,-1.5535829,-0.6938023,-0.46579614,,,,0,0,0,1,0,0,0,0,0,0,-0.50661701,-0.99926889,0.23773268 173 | 1951,4,1951q4,-0.15983175,0.69884866,,1,1,1,-0.75179893,0.11159553,-0.40874308,,,,0,0,0,1,0,0,0,0,0,0,0.19525279,-0.3103306,0.12173963 174 | 1952,1,1952q1,-0.35249421,-0.19266246,,0,1,1,-0.86499727,-0.25170106,0.034621306,,,,0,0,0,1,0,0,0,0,0,0,0.34511444,-0.56824553,0.055060398 175 | 1952,2,1952q2,-0.55507803,-0.20258382,,0,1,1,-1.1070092,-0.32679334,0.04968879,,,,0,0,0,1,0,0,0,0,0,0,-1.1556851,-0.74323815,-0.10910045 176 | 1952,3,1952q3,-0.18963513,0.3654429,,0,1,1,-0.82220304,0.1093156,0.18575348,,,,0,0,0,1,0,0,0,0,0,0,0.47673038,-0.15327188,0.62301296 177 | 1952,4,1952q4,-0.2337117,-0.044076577,,0,1,1,-0.78493488,-0.035980318,-0.13971822,,,,0,0,0,1,0,0,0,0,0,0,-0.24312064,-0.39504299,-0.15052509 178 | 1953,1,1953q1,-0.12972362,0.10398808,,0,1,1,-0.8061173,0.11294039,0.093157768,,,,0,0,0,1,0,0,0,0,0,0,0.117565,-0.35715163,0.62768412 179 | 1953,2,1953q2,-0.26041797,-0.13069434,,0,1,1,-1.0222226,0.18116271,-0.70408034,,,,0,0,0,1,0,0,0,0,0,0,-0.33447415,-0.22628181,0.39412454 180 | 1953,3,1953q3,-0.071382858,0.18903512,,0,1,1,-0.73532009,0.15496561,-0.5909273,,,,0,0,0,1,0,0,0,0,0,0,0.11971729,-0.088760361,0.33757523 181 | 1953,4,1953q4,-0.43776971,-0.36638686,,0,0,1,-0.88542014,-0.46729705,0.86623478,,,,0,0,0,0,0,0,0,0,0,0,0.064108223,-0.2974982,-0.0065230117 182 | 1954,1,1954q1,-0.22553188,0.21223783,,0,0,1,-0.64182717,-0.15113756,0.020969339,,,,0,0,0,0,0,0,0,0,0,0,-0.13307254,-0.43383631,0.060399666 183 | 1954,2,1954q2,0.058851633,0.28438351,,0,0,1,-0.7694065,0.50945711,-0.375898,,,,0,0,0,0,0,0,0,0,0,0,-0.12495685,0.032561574,0.52285719 184 | 1954,3,1954q3,0.31188452,0.25303289,,0,0,1,-0.24556418,0.44526196,-0.33884728,,,,0,0,0,0,0,0,0,0,0,0,0.59704572,0.3142204,0.54228175 185 | 1954,4,1954q4,0.053275455,-0.25860906,,0,0,1,-0.21672095,-0.063304052,0.94618094,,,,0,0,0,0,0,0,0,0,0,0,0.44740301,-0.12709528,-0.017738834 186 | 1955,1,1955q1,-0.25048554,-0.30376101,,0,0,1,-0.59447414,-0.41120395,-0.062745683,-1.1520337,,,0,0,0,0,0,0,0,0,0,0,-0.3243137,-0.971376,0.10135221 187 | 1955,2,1955q2,-0.022897268,0.22758827,0,1,0,1,-0.29273009,-0.25072744,1.3071946,-1.187798,-1.14433,-0.27844888,0,0,0,0,0,0,0,0,0,0,0.23932776,0.11322551,0.50316918 188 | 1955,3,1955q3,0.44085142,0.46374869,0,0,0,1,0.24701703,0.21120274,-0.066606037,-1.187798,-1.0615081,0.93635881,0,0,0,0,0,0,0,0,0,0,1.0292732,0.3290028,1.05065 189 | 1955,4,1955q4,-0.75092667,-1.1917781,0,0,0,1,-0.96373242,-0.96442115,0.24297516,-1.2235622,-0.77990407,-0.69091845,0,0,0,0,0,0,0,0,0,0,-0.64536428,-1.1145995,-0.5762912 190 | 1956,1,1956q1,-0.5981968,0.15272987,0,0,0,1,-0.88348073,-0.79723001,0.32770044,-1.187798,-1.3225874,0.33240792,0,0,0,0,0,0,0,0,0,0,-0.31381795,-0.92155039,-0.69075698 191 | 1956,2,1956q2,-0.39207992,0.20611688,1,0,0,1,-0.89886504,-0.4173215,0.061202407,-1.187798,-0.82673156,-0.48562539,0,0,0,0,0,0,0,0,0,0,-0.67464501,-0.54965746,0.58857429 192 | 1956,3,1956q3,-0.26538152,0.1266984,1,0,0,1,-0.7412315,-0.22478698,0.34579378,-1.1520337,-1.6049261,-0.42284462,0,0,0,0,0,0,0,0,0,0,-0.11100276,-0.68595159,-0.12930545 193 | 1956,4,1956q4,-1.041737,-0.77635545,0,0,0,1,-1.0013906,-1.5521362,-0.27235806,-1.1520337,-1.1678636,-0.078806058,0,0,0,0,0,0,0,0,0,0,-1.6269741,-1.297766,-1.0355123 194 | 1957,1,1957q1,-0.64120269,0.40053427,0,0,0,1,-0.91303033,-0.99101144,1.7215861,-1.0447409,-1.0097476,0.8748337,0,0,0,0,0,0,0,0,0,0,-0.81119418,-0.96108717,-0.35492903 195 | 1957,2,1957q2,0.21142457,0.85262728,0,0,0,1,-0.36351109,0.22834192,1.4057063,-1.0447409,-1.1813022,-0.34060183,0,0,0,0,0,0,0,0,0,0,0.2681255,-0.087125011,1.0229729 196 | 1957,3,1957q3,-0.16574147,-0.37716603,0,0,0,1,-0.66622776,-0.12059291,0.78239369,-1.0805051,-0.96563083,-0.76123291,0,0,0,0,0,0,0,0,0,0,-0.23945716,-0.25902081,0.59105629 197 | 1957,4,1957q4,0.095942728,0.26168421,0,0,0,1,-0.28057998,0.024556497,-0.38176396,-1.0805051,-1.0126153,-0.23010768,0,0,0,0,0,0,0,0,0,0,0.72011554,0.012744137,0.26138383 198 | 1958,1,1958q1,0.059883256,-0.036059473,0,0,0,1,-0.3232837,-0.14034727,-0.028867561,-0.97321236,-1.202708,0.94640374,0,0,0,0,0,0,0,0,0,0,0.54400718,-0.012614804,0.0042616306 199 | 1958,2,1958q2,-0.3149024,-0.37478566,0,0,0,1,-0.83213836,-0.28098229,1.3382177,-0.86591959,-0.73273391,-1.7914652,0,0,0,0,0,0,0,0,0,0,-0.52090693,-0.37623346,-0.14843665 200 | 1958,3,1958q3,-0.093851827,0.22105056,0,0,0,1,-0.53760529,-0.25385574,-0.32780534,-0.75862676,-1.5062999,1.033669,0,0,0,0,0,0,0,0,0,0,0.19251041,0.18994541,0.56352341 201 | 1958,4,1958q4,-0.37842274,-0.2845709,0,0,0,1,-0.58025837,-0.77462256,-0.36143056,-0.79439104,-0.7176016,-0.3067002,0,0,0,0,0,0,0,0,0,0,-0.27303076,-0.66764098,-0.344284 202 | 1959,1,1959q1,0.027830418,0.40625316,0,0,0,1,-0.45372233,-0.10493288,-0.078307636,-0.79439104,-1.0027416,-0.014141873,0,0,0,0,0,0,0,0,0,0,0.67108351,-0.19685335,-0.2352532 203 | 1959,2,1959q2,0.15868902,0.1308586,0,0,0,1,-0.34477186,0.13491869,0.86735123,-0.79439104,-1.4675841,0.66702938,0,0,0,0,0,0,0,0,0,0,-0.23128234,0.083786972,0.057443466 204 | 1959,3,1959q3,-0.14443792,-0.30312693,0,0,0,1,-0.58379644,-0.30395252,1.5064771,-0.83015531,-1.2838804,0.98156095,0,0,0,0,0,0,0,0,0,0,0.089344673,-0.30797029,0.15382092 205 | 1959,4,1959q4,0.35973698,0.50417489,0,1,0,1,-0.11809152,0.31931651,-0.062745683,-0.90168387,-0.99787837,1.2546573,0,0,0,0,0,0,0,0,0,0,1.0447241,0.30560353,0.5107128 206 | 1960,1,1960q1,0.5590803,0.19934332,0,0,0,1,0.068598025,0.52322203,0.3557829,-0.97321236,-1.2931952,1.1887375,0,0,0,0,0,0,0,0,0,0,1.5830029,0.25931039,0.95187712 207 | 1960,2,1960q2,-0.12334576,-0.6824261,0,0,0,1,-0.58069819,-0.25007156,0.60731554,-1.0089766,-1.1374221,-0.84787035,0,0,0,0,0,0,0,0,0,0,-0.11005219,-0.0005473043,0.29644322 208 | 1960,3,1960q3,-0.21739577,-0.094050005,0,0,0,1,-0.35448045,-0.62038016,0.28894648,-1.0089766,-1.3082932,0.63877803,0,0,0,0,0,0,0,0,0,0,0.056473818,-0.095794506,0.1109902 209 | 1960,4,1960q4,-0.075937897,0.14145787,0,0,0,1,-0.44205359,-0.22160324,0.44605416,-1.0447409,-1.0239357,0.027921237,0,0,0,0,0,0,0,0,0,0,0.28728655,-0.19711548,0.42052609 210 | 1961,1,1961q1,-0.071300611,0.0046372861,0,0,0,1,-0.22755155,-0.35824206,0.69153327,-1.0805051,-1.2367126,0.66640157,0,0,0,0,0,0,0,0,0,0,0.52197087,-0.16102184,0.33497727 211 | 1961,2,1961q2,-0.1525913,-0.081290692,0,0,0,1,-0.60078269,-0.14921142,0.54997182,-1.0805051,-1.0159942,-0.10580178,0,0,0,0,0,0,0,0,0,0,-0.22865866,-0.13104953,0.27541399 212 | 1961,3,1961q3,-0.44555151,-0.29296023,1,0,0,1,-0.68319154,-0.7687704,0.097835898,-1.0447409,-0.97874618,-0.67647886,0,0,0,0,0,0,0,0,0,0,0.17236003,-0.5709995,-0.11525989 213 | 1961,4,1961q4,-0.79098576,-0.34543425,1,0,0,1,-1.0398442,-1.0454421,0.28046235,-1.0089766,-0.97809184,-0.46679115,0,0,0,0,0,0,0,0,0,0,-0.28398392,-0.68133307,-0.96221983 214 | 1962,1,1962q1,-0.18868835,0.60229743,0,0,0,1,-0.45509961,-0.46411759,1.6987107,-1.0089766,-1.0082248,-0.051810328,0,0,0,0,0,0,0,0,0,0,0.20706774,-0.37718707,0.32062644 215 | 1962,2,1962q2,0.055349842,0.24403819,0,0,0,1,-0.38313746,-0.10079575,0.21157669,-0.90168387,-0.56370002,0.240748,0,0,0,0,0,0,0,0,0,0,0.7055434,0.34857625,0.58608502 216 | 1962,3,1962q3,-0.060451534,-0.11580138,0,0,0,1,-0.16184561,-0.57951242,-0.15046,-0.83015531,-1.3432165,0.026665621,0,0,0,0,0,0,0,0,0,0,0.58706534,-0.045508105,-0.073040806 217 | 1962,4,1962q4,0.1896352,0.25008672,0,0,0,1,-0.0098989084,-0.18698913,0.492493,-0.79439104,-1.1503981,-0.60679221,0,0,0,0,0,0,0,0,0,0,1.0394982,0.1644669,0.5335148 218 | 1963,1,1963q1,0.16327363,-0.02636157,0,0,0,1,-0.053761613,-0.33654106,-0.52397668,-0.68709826,-0.58202851,-0.010375027,0,0,0,0,0,0,0,0,0,0,0.4947989,-0.069287464,-0.04715151 219 | 1963,2,1963q2,0.39314047,0.22986683,0,0,0,1,-0.017355781,0.13253853,-0.001758565,-0.72286248,-0.90635574,2.3382533,0,0,0,0,0,0,0,0,0,0,0.79717231,0.41126424,0.91512698 220 | 1963,3,1963q3,0.048150204,-0.34499025,0,0,0,1,-0.15948309,-0.38663852,0.29971567,-0.79439104,-1.3703066,-0.058088403,0,0,0,0,0,0,0,0,0,0,0.83097577,-0.48218188,0.24313682 221 | 1963,4,1963q4,0.63189906,0.58374888,0,0,0,1,0.18411998,0.39951473,0.1086395,-0.86591959,-0.9622708,1.0933107,0,0,0,0,0,0,0,0,0,0,1.8300138,0.45739964,1.2617229 222 | 1964,1,1964q1,0.66110575,0.029206693,0,0,0,1,0.12733217,0.66947526,0.41613635,-0.93744808,-0.98792797,0.0059479708,0,0,0,0,0,0,0,0,0,0,1.329015,0.33856243,0.90904123 223 | 1964,2,1964q2,-0.1566568,-0.81776255,0,0,0,1,-0.74670589,-0.17580485,0.31383842,-0.97321236,-0.71166027,0.7467609,0,0,0,0,0,0,0,0,0,0,0.096025161,-0.50570303,0.52676719 224 | 1964,3,1964q3,0.26655608,0.42321289,0,0,0,1,-0.10339256,0.054837629,0.13572676,-0.97321236,-1.0699701,-0.11961355,0,0,0,0,0,0,0,0,0,0,1.3736868,0.17146286,0.70194465 225 | 1964,4,1964q4,0.23427691,-0.032279179,0,1,0,0,-0.90008152,1.0647004,-0.066606037,-1.0447409,-1.0859405,0.61806035,0,0,0,0,0,0,0,0,0,0,0.67616856,0.17622189,0.69171709 226 | 1965,1,1965q1,0.63900667,0.40472978,0,0,0,0,-0.6004312,1.6585072,0.5775516,-1.0805051,-1.1112059,-0.52831632,0,0,0,0,0,0,0,0,0,0,1.45662,0.35001272,1.5027878 227 | 1965,2,1965q2,-0.096799955,-0.73580664,0,0,0,0,-0.8603,0.39945513,0.71170521,-1.0805051,-0.59722692,-0.30042213,0,0,0,0,0,0,0,0,0,0,-0.40315041,0.20785257,1.0408963 228 | 1965,3,1965q3,0.037008993,0.13380894,0,0,0,0,-0.82211518,0.54238141,0.12530863,-1.0447409,-1.233948,0.17796725,0,0,0,0,0,0,0,0,0,0,0.0086142747,0.21712281,0.42017952 229 | 1965,4,1965q4,-0.061314952,-0.098323941,0,0,0,0,-1.1100767,0.62152576,-0.017906234,-1.0805051,-1.210603,0.037966158,0,0,0,0,0,0,0,0,0,0,0.35732245,-0.45216069,-0.035645131 230 | 1966,1,1966q1,0.21790491,0.27921987,0,1,0,0,-0.81912363,0.85805953,0.10248506,-1.1162695,-1.2158865,-0.26087025,0,0,0,0,0,0,0,0,0,0,1.1913549,-0.096694782,0.59158933 231 | 1966,2,1966q2,0.1083229,-0.10958201,0,0,0,0,-0.70878196,0.52941543,0.84239662,-1.1162695,-0.80551696,-0.0022135281,0,0,0,0,0,0,0,0,0,0,-0.047740154,-0.0237207,0.59885174 232 | 1966,3,1966q3,-0.24057896,-0.34890187,0,0,0,0,-0.8024655,-0.089497544,-0.375898,-1.0447409,-1.2875246,-0.12651943,0,0,0,0,0,0,0,0,0,0,-0.61357653,-0.56142908,0.089790195 233 | 1966,4,1966q4,-0.24681988,-0.0062409192,0,0,0,0,-1.0330197,0.015841996,-0.13334616,-0.90168387,-0.99882317,-0.61934841,0,0,0,0,0,0,0,0,0,0,0.32641098,-0.61301696,0.26180303 234 | 1967,1,1967q1,0.33006659,0.57688648,0,0,0,0,-0.44759333,0.71135807,0.059568372,-0.79439104,-0.98597693,0.50882185,0,0,0,0,0,0,0,0,0,0,0.8510673,-0.12721899,0.76895899 235 | 1967,2,1967q2,-0.20293483,-0.53300142,0,0,0,0,-0.91362113,0.082478933,0.26817343,-0.75862676,-0.83307803,0.51196086,0,0,0,0,0,0,0,0,0,0,-0.35519788,-0.50657284,-0.23146954 236 | 1967,3,1967q3,-0.5497179,-0.34678307,0,0,0,0,-1.215765,-0.42950556,0.3601802,-0.72286248,-1.2691957,-0.068761133,0,0,0,0,0,0,0,0,0,0,-0.12082853,-0.69597346,-0.67414665 237 | 1967,4,1967q4,-0.34181064,0.20790726,0,0,0,0,-1.2491724,0.10701286,0.67080444,-0.72286248,-0.68230683,-0.027325831,0,0,0,0,0,0,0,0,0,0,0.6933347,-0.53041404,-0.4803552 238 | 1968,1,1968q1,-0.40203422,-0.060223579,0,0,0,0,-1.0232868,-0.37153715,0.39131302,-0.72286248,-0.62389266,1.9352007,0,0,0,0,0,0,0,0,0,0,0.50643802,-0.51881856,-0.42556846 239 | 1968,2,1968q2,-0.038417142,0.36361709,0,0,0,0,-0.8926779,0.35399643,1.3000184,-0.68709826,-0.28740698,-0.59674734,0,0,0,0,0,0,0,0,0,0,0.46668711,-0.25336444,0.37956893 240 | 1968,3,1968q3,-0.44687149,-0.40845436,0,0,0,0,-1.3280851,-0.0050957832,0.47576573,-0.72286248,-0.91968381,0.90496844,0,0,0,0,0,0,0,0,0,0,0.028226914,-0.72127622,-0.50951278 241 | 1968,4,1968q4,-0.27415287,0.17271861,0,0,0,0,-1.0220094,-0.1629298,0.45865729,-0.72286248,-0.69511992,-0.080061667,0,0,0,0,0,0,0,0,0,0,0.37560219,-0.49309447,-0.22613913 242 | 1969,1,1969q1,0.071889259,0.34604213,0,0,0,0,-0.78885853,0.37744522,0.97379196,-0.72286248,-0.42558053,-0.51073766,0,0,0,0,0,0,0,0,0,0,0.85581475,-0.33544615,0.0025013869 243 | 1969,2,1969q2,-0.28431764,-0.35620689,0,0,0,0,-0.98872215,-0.12238088,0.72732699,-0.72286248,-0.57298988,0.095096648,0,0,0,0,0,0,0,0,0,0,0.38411993,-0.43139097,-0.020778373 244 | 1969,3,1969q3,-0.33277139,-0.048453748,0,0,1,0,-1.1165013,-0.19685863,0.89527476,-0.72286248,-1.0190896,0.017876314,0,0,0,0,0,0,0,1,1,0,0.29928467,-0.33036,-0.04608522 245 | 1969,4,1969q4,-0.44239727,-0.10962588,0,0,0,0,-1.1302633,-0.3030563,1.412214,-0.72286248,-0.66178125,-0.028581446,0,0,0,0,0,0,0,1,0,0,-0.036477704,-0.96850383,-0.823479 246 | 1970,1,1970q1,0.21926153,0.66165876,0,0,0,0,-0.7220881,0.60766721,1.2365941,-0.68709826,-0.43328035,-0.78634518,0,0,0,0,0,0,0,1,0,0,1.3710517,-0.30742207,0.21827829 247 | 1970,2,1970q2,-0.43240124,-0.65166277,0,1,0,1,-1.0651004,-0.41828075,1.4175946,-0.68709826,-0.2579731,1.1447909,0,0,0,0,0,0,0,1,0,0,0.068915151,-0.77618444,-0.14041634 248 | 1970,3,1970q3,0.5944559,1.0268571,0,0,0,1,-0.087474167,0.65965176,1.3040829,-0.65133399,-0.65282041,0.27213839,0,0,0,0,0,0,0,1,0,0,1.7362545,0.38042453,0.22120506 249 | 1970,4,1970q4,0.11009404,-0.48436186,0,0,0,1,-0.38654035,-0.13501462,1.5995291,-0.65133399,-0.32936913,0.22630844,0,0,0,0,0,0,0,1,0,0,0.67639762,-0.13375962,0.304196 250 | 1971,1,1971q1,-0.6448161,-0.75491011,0,0,0,1,-0.90343368,-1.021655,2.3343668,-0.83015531,-0.29055738,-0.74993235,0,0,0,0,0,0,0,1,0,0,-0.070147887,-0.5799638,-0.19642423 251 | 1971,2,1971q2,-0.39213192,0.25268418,0,0,0,1,-0.6256181,-1.0115399,0.93300796,-0.72286248,0.074796513,0.74801654,0,0,0,0,0,0,0,1,0,0,0.56413192,-0.21815193,-0.73278368 252 | 1971,3,1971q3,-0.51236868,-0.12023675,0,0,0,1,-0.42398706,-1.338406,0.83077353,-0.68709826,-0.59375167,0.76371175,0,0,0,0,0,0,0,1,0,0,0.36573526,-0.47687507,-1.3762355 253 | 1971,4,1971q4,-0.10797611,0.40439257,0,0,0,1,-0.22228585,-0.8055703,0.84354913,-0.57980543,-0.55861461,-0.11898574,0,0,0,0,0,0,0,1,0,0,0.95791072,-0.2792365,-0.74307305 254 | 1972,1,1972q1,-0.31112477,-0.20314866,0,0,0,1,-0.56192607,-0.76254922,2.5419817,-0.54404116,-0.42702386,-0.3142339,0,0,0,0,0,0,0,1,0,0,0.35470775,-0.50551391,-1.1190846 255 | 1972,2,1972q2,-0.21415879,0.096965984,0,0,0,1,-0.48723716,-0.67741567,1.467124,-0.57980543,-0.34054866,1.314299,0,0,0,0,0,0,0,1,0,0,0.026330631,-0.64038265,-0.084947765 256 | 1972,3,1972q3,-0.13585371,0.078305081,0,0,1,1,-0.44432822,-0.67875594,2.087182,-0.65133399,-0.47329959,-0.20311196,0,0,0,0,0,0,0,1,0,1,0.41882548,-0.44071934,-0.23640095 257 | 1972,4,1972q4,-0.34384629,-0.20799258,0,0,0,1,-0.54147792,-0.95425195,0.97909826,-0.68709826,-0.26630366,0.63940585,0,0,0,0,0,0,0,0,0,0,0.24454869,-0.73866946,-0.78720725 258 | 1973,1,1973q1,-0.20563893,0.13820736,0,0,0,1,-0.46247771,-0.86831117,1.2551259,-0.83015531,-0.60002017,2.7971804,0,0,0,0,0,0,0,0,0,0,0.83566135,-0.66623175,-0.19200172 259 | 1973,2,1973q2,-0.17716126,0.028477669,0,0,0,1,-0.43399689,-0.70632142,1.127362,-0.86591959,-0.17309864,-0.26212588,0,0,0,0,0,0,0,0,0,0,0.34185013,-0.74284929,-0.6330359 260 | 1973,3,1973q3,-0.34054387,-0.1633826,1,0,0,1,-0.69492096,-0.86274624,0.87124592,-0.93744808,-0.78520757,-0.98975486,0,0,0,0,0,0,0,0,0,0,-0.13921644,-0.81762183,-0.88723254 261 | 1973,4,1973q4,-0.57000828,-0.22946441,1,0,0,1,-0.66818041,-1.3556464,1.0364561,-1.0089766,-0.20756049,-0.62060398,0,0,0,0,0,0,0,0,0,0,0.10678216,-1.0930097,-1.1787525 262 | 1974,1,1974q1,-0.18529168,0.3847166,1,1,0,0,-2.1647029,1.3582515,2.0489755,-0.93744808,0.063574366,-2.0645614,0,0,0,0,0,0,0,0,0,0,1.0939639,-0.31513855,-0.25603205 263 | 1974,2,1974q2,-0.71982187,-0.53453016,0,0,0,0,-1.4900237,-0.43285143,1.2892829,-0.93744808,0.6735605,0.54711813,0,0,0,0,0,0,0,0,0,0,0.024276217,-0.86664659,-1.6321779 264 | 1974,3,1974q3,-0.9584316,-0.23860973,0,0,0,0,-1.5019935,-0.92730123,1.1714177,-0.86591959,-0.40573394,0.039849579,0,0,0,0,0,0,0,0,0,0,-0.56575668,-1.1501244,-2.0617924 265 | 1974,4,1974q4,-0.91616517,0.042266428,0,1,0,0,-1.4536492,-1.034416,1.6620679,-0.86591959,0.25848508,-1.3005196,0,0,0,0,0,0,0,0,0,0,-0.094298117,-1.1901082,-1.5271776 266 | 1975,1,1975q1,-0.25522342,0.66094172,0,0,0,0,-0.99085885,-0.078412734,0.91699362,-0.79439104,0.81812823,-0.31235048,0,0,0,0,0,0,0,0,0,0,0.074535213,-0.35651511,-0.61847293 267 | 1975,2,1975q2,-0.83846468,-0.58324122,1,0,0,0,-1.2768475,-1.0855548,1.3222579,-0.61556971,2.0907578,-1.4292202,0,0,0,0,0,0,0,0,0,0,-0.23570649,-1.058962,-1.6484162 268 | 1975,3,1975q3,-0.50566399,0.33280069,1,0,0,0,-0.93419373,-0.65198678,0.89688545,-0.47251263,0.57907552,-0.55845106,0,0,0,0,0,0,0,0,0,0,0.11843238,-0.36986068,-0.92389321 269 | 1975,4,1975q4,-0.70466465,-0.19900066,0,0,0,0,-1.1888345,-0.82682353,0.38391554,-0.32945558,0.39938316,0.40900046,0,0,0,0,0,0,0,0,0,0,0.33983529,-1.0624663,-1.8568716 270 | 1976,1,1976q1,-0.58229202,0.12237263,0,0,0,0,-1.2050583,-0.58054167,0.5379582,-0.22216277,0.53973281,0.6852358,0,0,0,0,0,0,0,0,0,0,0.18717994,-0.61579216,-1.169426 271 | 1976,2,1976q2,-0.56244403,0.019847989,0,0,0,0,-1.2020986,-0.45251811,0.39967898,-0.18639851,0.60063016,-0.48248634,0,0,0,0,0,0,0,0,0,0,0.13379095,-0.66953039,-1.3295227 272 | 1976,3,1976q3,-1.0779694,-0.5155254,0,0,0,0,-1.430055,-1.2652569,0.58571297,-0.15063424,0.19023171,0.45169136,0,0,0,0,0,0,0,0,0,0,-0.70077062,-1.3384564,-2.0970843 273 | 1976,4,1976q4,-0.77085632,0.30711311,0,0,0,0,-1.2554713,-0.79786253,0.4470312,-0.15063424,0.94327033,1.0236241,0,0,0,0,0,0,0,0,0,0,-0.21292333,-1.0228418,-1.667383 274 | 1977,1,1977q1,-0.58863938,0.18221694,0,0,0,0,-1.2357146,-0.44580716,1.2554586,-0.15063424,1.0807011,-0.31988418,0,0,0,0,0,0,0,0,0,0,-0.035846677,-0.80823779,-1.2098178 275 | 1977,2,1977q2,-0.25346518,0.3351742,0,0,0,0,-0.77653909,-0.2660518,1.0901562,-0.11486997,0.91746968,-0.74114305,0,0,0,0,0,0,0,0,0,0,0.3861675,-0.53607529,-0.60782772 276 | 1977,3,1977q3,-1.0531212,-0.79965603,0,0,0,0,-1.6059066,-1.0429119,1.315225,-0.079105705,0.029111948,0.078145847,0,0,0,0,0,0,0,0,0,0,-1.049931,-1.2411613,-1.9113717 277 | 1977,4,1977q4,-0.3391352,0.71398604,0,0,0,0,-0.90558147,-0.35157746,1.6122538,-0.043341443,0.013419836,0.71599835,0,0,0,0,0,0,0,0,0,0,0.60496557,-0.56118971,-0.69165003 278 | 1978,1,1978q1,-0.66139662,-0.32226142,0,0,0,0,-1.2449801,-0.69471562,1.0788778,-0.11486997,0.039252542,0.13025388,0,0,0,0,0,0,0,0,0,0,0.23483917,-0.9755919,-1.4635911 279 | 1978,2,1978q2,-0.098567791,0.56282884,0,0,0,0,-0.78175896,0.039561585,0.98484451,-0.11486997,0.36187783,0.42469564,0,0,0,0,0,0,0,0,0,0,0.39132684,-0.39398479,-0.71788633 280 | 1978,3,1978q3,-0.47350371,-0.37493593,0,0,0,0,-1.0972785,-0.41587856,1.0553654,-0.11486997,0.037802469,0.46801436,0,0,0,0,0,0,0,0,0,0,0.074172854,-0.80433279,-1.0760174 281 | 1978,4,1978q4,-0.45517346,0.018330246,0,0,0,0,-1.1152054,-0.374962,1.742395,-0.18639851,-0.031820253,0.26334909,0,0,0,0,0,0,0,0,0,0,0.60565871,-0.88298523,-1.2696203 282 | 1979,1,1979q1,-0.64109176,-0.1859183,0,0,0,0,-1.1761137,-0.854828,2.0751081,-0.18639851,0.42130351,-0.66203928,0,0,0,0,0,0,0,0,0,0,0.31311461,-0.95530182,-1.3378085 283 | 1979,2,1979q2,0.69677079,1.3378625,0,1,0,1,0.11530983,0.40899289,1.1514344,-0.22216277,0.57178861,2.4016616,0,0,0,0,0,0,0,0,0,0,1.8181649,1.0179143,0.57684761 284 | 1979,3,1979q3,-0.71338362,-1.4101543,0,0,0,1,-0.82798696,-1.4167259,2.7482529,-0.25792703,1.5278832,-1.6872491,0,0,0,0,0,0,0,0,0,0,-0.3325707,-0.72571105,-1.2851206 285 | 1979,4,1979q4,-0.23573561,0.47764802,0,0,0,1,-0.33839267,-1.0916419,1.7401305,-0.25792703,0.25902241,0.30101755,0,0,0,0,0,0,0,0,0,0,0.6291489,-0.13835582,-0.51106852 286 | 1980,1,1980q1,-0.46344388,-0.22770827,1,0,0,1,-0.66124737,-1.2179888,2.3179011,-0.18639851,0.93493158,-1.0462575,0,0,0,0,0,0,0,0,0,0,0.53411001,-0.70283073,-1.0325022 287 | 1980,2,1980q2,-0.59432966,-0.13088578,1,0,0,1,-0.56979305,-1.658847,1.0618513,-0.043341443,1.4363577,-1.5805218,0,0,0,0,0,0,0,0,0,0,-0.013495084,-0.91515839,-1.6185871 288 | 1980,3,1980q3,-0.44326934,0.15106031,1,0,0,1,-0.29073974,-1.5995957,0.1177776,0.20700842,0.46394721,-0.50571519,0,0,0,0,0,0,0,0,0,0,0.0006523872,-0.51196396,-1.8296758 289 | 1980,4,1980q4,-0.86632353,-0.42305419,1,0,0,1,-0.61412293,-2.3521364,0.015785169,0.56465107,0.68823951,-0.99352169,0,0,0,0,0,0,0,0,0,0,-0.36991018,-1.2356162,-2.2028856 290 | 1981,1,1981q1,-0.35936549,0.50695801,1,0,0,1,-0.32492042,-1.5814477,0.87952423,0.85076523,1.1176122,-0.52517724,0,0,0,0,0,0,0,0,0,0,0.2645613,-0.78195655,-1.0463713 291 | 1981,2,1981q2,-0.71356994,-0.35420445,0,0,1,1,-0.64890867,-1.8080399,0.85893685,1.1011151,2.1747305,-0.18302211,0,0,0,0,1,0,0,0,0,0,-0.17935202,-0.92320228,-2.0431483 292 | 1981,3,1981q3,-0.27558041,0.43798953,0,0,0,1,-0.27724761,-1.257988,0.21690179,1.2799364,1.3009884,0.37509882,0,0,0,0,0,0,0,0,0,0,-0.29209286,-0.54572821,-2.1640472 293 | 1981,4,1981q4,-0.27449533,0.0010850728,0,0,0,1,-0.2229186,-1.6239303,0.64574963,1.3872292,1.6551701,-0.28221571,0,0,0,0,0,0,0,0,0,0,0.089268588,-0.84546489,-1.9854592 294 | 1982,1,1982q1,-0.27790317,-0.003407836,0,0,0,1,-0.30857161,-1.5251842,1.1324738,1.494522,1.4849808,-0.26275367,0,0,0,0,0,0,0,0,0,0,0.62251431,-0.49280035,-1.4738984 295 | 1982,2,1982q2,-0.29052293,-0.012619764,0,0,0,1,-0.62042332,-1.078567,0.78426528,1.5660505,2.0519476,0.41402289,0,0,0,0,0,0,0,0,0,0,-0.28512251,-0.39084083,-1.24143 296 | 1982,3,1982q3,-0.036308642,0.25421429,0,0,0,1,-0.20371705,-0.92048812,0.9054178,1.6733433,1.2535932,-0.23450233,0,0,0,0,0,0,0,0,0,0,0.64654899,0.048989415,-1.4762136 297 | 1982,4,1982q4,-0.4134852,-0.37717655,0,0,0,1,-0.27127266,-1.902582,0.4557676,1.7806362,1.4420309,0.078773655,0,0,0,0,0,0,0,0,0,0,0.46456629,-0.62658143,-2.2920165 298 | 1983,1,1983q1,-0.29159498,0.12189022,0,0,0,1,-0.1685545,-1.5799038,0.99714583,1.8521646,1.4462345,0.72792673,0,0,0,0,0,0,0,0,0,0,0.26201418,-0.6044696,-1.5925552 299 | 1983,2,1983q2,0.24559838,0.53719336,0,1,0,1,0.23007715,-1.1176683,0.3126716,1.9236932,2.010432,0.20307955,0,0,0,0,0,0,0,0,0,0,0.75234342,0.28576386,-1.0056535 300 | 1983,3,1983q3,-0.12804414,-0.3736425,0,0,0,1,0.19877037,-1.9233966,0.36564085,1.9594575,1.7980119,0.38074911,0,0,0,0,0,0,0,0,0,0,0.46485078,-0.36716035,-0.98093903 301 | 1983,4,1983q4,0.0091563622,0.1372005,0,0,0,1,0.069425248,-1.3742872,0.49707407,1.9952217,1.7769897,0.26962715,0,0,0,0,0,0,0,0,0,0,0.50746614,-0.28735417,-0.90408403 302 | 1984,1,1984q1,-0.062467806,-0.071624167,0,0,0,1,0.072804689,-1.3416393,1.4701293,1.9594575,1.5667745,0.17482822,0,0,0,0,0,0,0,0,0,0,0.62372136,-0.61014104,-0.84309506 303 | 1984,2,1984q2,-0.27932748,-0.21685967,0,0,0,1,-0.29110572,-1.2574573,2.2434561,1.9952217,2.0498433,-0.93325222,0,0,0,0,0,0,0,0,0,0,0.039206475,-0.8561427,-1.7845373 304 | 1984,3,1984q3,-0.42360857,-0.14428109,0,0,0,1,-0.32336548,-1.7518446,2.1565135,2.0309861,1.7278891,-0.12149697,0,0,0,0,0,0,0,0,0,0,-0.30192071,-0.6840862,-1.2639594 305 | 1984,4,1984q4,-0.4505327,-0.026924133,0,0,0,1,-0.29546401,-1.8938922,2.2262156,2.1025145,1.892113,0.57097483,0,0,0,0,0,0,0,0,0,0,0.21988085,-0.93917078,-1.7001069 306 | 1985,1,1985q1,-0.47289142,-0.022358716,0,0,0,1,-0.62885141,-1.4976599,1.7863063,2.0667503,1.8736628,0.36882076,0,0,0,0,0,0,0,0,0,0,0.27070048,-0.76100177,-1.7379017 307 | 1985,2,1985q2,-0.090786442,0.38210499,0,0,0,1,-0.14832701,-1.2433565,0.25693873,2.1025145,2.6220617,0.67079622,0,0,0,0,0,0,0,0,0,0,0.18748015,-0.33264071,-1.0424612 308 | 1985,3,1985q3,-0.08231809,0.0084683523,0,0,0,1,-0.23065418,-0.98583245,0.12079908,2.1025145,1.5768654,-0.40777722,0,0,0,0,0,0,0,0,0,0,1.1481956,-0.17483512,-0.79889274 309 | 1985,4,1985q4,-0.16263881,-0.080320723,0,0,0,1,-0.1838735,-1.4860642,0.40589285,2.1025145,1.6675283,-0.058088403,0,0,0,0,0,0,0,0,0,0,0.8483867,-0.59163916,-1.1910334 310 | 1986,1,1986q1,0.17504689,0.3376857,0,0,0,1,0.16360253,-1.1946721,0.31964538,2.1740432,1.7785339,0.1158143,0,0,0,0,0,0,0,0,0,0,0.93844998,-0.26452917,-0.75186378 311 | 1986,2,1986q2,0.25797078,0.082923889,0,0,0,1,0.064394169,-0.81806344,0.26444718,2.2098074,2.0128548,0.35689241,0,0,0,0,0,0,0,0,0,0,0.49069837,-0.13339229,-0.68249661 312 | 1986,3,1986q3,0.49601486,0.23804408,0,0,0,1,0.40871567,-0.74180251,-0.29038933,2.2098074,1.6172879,0.06872873,0,0,0,0,0,0,0,0,0,0,0.46623814,-0.053789642,-0.65501368 313 | 1986,4,1986q4,0.074447669,-0.4215672,0,0,0,1,0.016725289,-1.1119823,-0.080275759,2.1382787,1.9414985,0.82963151,0,0,0,0,0,0,0,0,0,0,0.74711764,-0.4872835,-1.0250061 314 | 1987,1,1987q1,0.42403424,0.34958658,0,0,0,1,0.19667165,-0.56013429,1.1897945,2.0309861,1.8160664,0.1735726,0,0,0,0,0,0,0,0,0,0,0.48891273,-0.18814357,-0.025845313 315 | 1987,2,1987q2,-0.1218477,-0.54588193,0,1,0,1,-0.15013218,-1.5673289,0.56763941,1.887929,1.8000785,0.51698333,0,0,0,0,0,0,0,0,0,0,-0.19149707,-0.73629498,-1.3393015 316 | 1987,3,1987q3,1.061355,1.1832027,0,0,0,1,0.79046386,0.057497781,-0.22788757,1.7091076,1.2229509,1.1818316,0,0,0,0,0,0,0,0,0,0,1.6683731,0.89233983,1.0989916 317 | 1987,4,1987q4,0.21746746,-0.84388757,0,0,0,1,0.43923935,-1.0934424,-0.35857022,1.5302863,1.3367907,0.38577157,0,0,0,0,0,0,0,0,0,0,0.7808907,-0.13948146,-0.88161039 318 | 1988,1,1988q1,0.360396,0.14292854,0,0,0,1,0.415043,-0.88393795,0.659118,1.351465,0.94305283,0.83151495,0,0,0,0,0,0,0,0,0,0,0.7031337,-0.062960498,-0.078956015 319 | 1988,2,1988q2,0.0107294,-0.3496666,0,0,0,1,0.097812064,-0.94793773,0.14161572,1.2084079,1.319703,0.058683809,0,0,0,0,0,0,0,0,0,0,0.015381436,-0.19382596,-0.92621452 320 | 1988,3,1988q3,0.29131091,0.2805815,0,0,0,1,0.45370325,-1.0370688,1.124199,1.0295866,0.78032517,0.62119943,0,0,0,0,0,0,0,0,0,0,0.7627387,0.0092344033,-0.71751559 321 | 1988,4,1988q4,0.020284683,-0.27102622,0,0,0,1,0.31453183,-1.6138499,-0.32780534,0.88652951,0.75663793,0.18361752,0,0,0,0,0,0,0,0,0,0,0.4283731,-0.35456643,-0.88137931 322 | 1989,1,1989q1,-0.32137248,-0.34165716,0,0,0,1,-0.082882158,-1.7732219,-0.61866587,0.70770818,0.51360297,-0.09512905,0,0,0,0,0,0,0,0,0,0,0.15610816,-0.82579547,-1.1823997 323 | 1989,2,1989q2,0.33947906,0.66085154,0,0,0,1,0.36529687,-0.72940326,0.19400452,0.56465107,0.84045988,0.046127658,0,0,0,0,0,0,0,0,0,0,0.51355547,-0.20249714,-0.31170204 324 | 1989,3,1989q3,0.7379905,0.39851144,0,0,0,1,0.75976813,-0.67175746,1.3599486,0.4573583,0.22345847,-0.32113978,0,0,0,0,0,0,0,0,0,0,0.82965016,0.2355178,-0.042884793 325 | 1989,4,1989q4,0.44414899,-0.29384151,0,0,0,1,0.49364179,-0.73559779,0.47670487,0.38582975,0.44736612,-0.33746278,0,0,0,0,0,0,0,0,0,0,0.88884759,0.059769668,-0.60277838 326 | 1990,1,1990q1,0.15917128,-0.2849777,0,0,0,1,0.087219283,-0.81334466,0.77737969,0.31430122,0.25116065,0.039221775,0,0,0,0,0,0,0,0,0,0,0.24542683,-0.0006434539,-1.0422492 327 | 1990,2,1990q2,0.22436137,0.065190092,0,0,0,1,0.0937846,-0.85184526,-0.056992106,0.27853695,0.87158895,-0.052438136,0,0,0,0,0,0,0,0,0,0,0.15803598,-0.062818065,-0.67970949 328 | 1990,3,1990q3,0.66868562,0.44432425,1,0,0,1,0.6949926,-0.52150786,-0.74593353,0.3500655,0.37639964,-1.0462575,0,0,0,0,0,0,0,0,0,0,1.2586403,0.53306913,0.020223474 329 | 1990,4,1990q4,0.09373042,-0.57495522,1,0,0,1,0.12316495,-1.0025197,-0.73642915,0.5288868,0.65746111,-0.60114199,0,0,0,0,0,0,0,0,0,0,-0.18735109,-0.48752591,-0.98666382 330 | 1991,1,1991q1,-0.21488485,-0.30861527,1,0,0,1,0.052574478,-1.8730115,-0.85928828,0.77923667,0.43111682,-0.55154514,0,0,0,0,0,0,0,0,0,0,-0.76569754,-0.72337788,-1.2163147 331 | 1991,2,1991q2,0.33748767,0.55237252,1,0,0,1,0.12482784,-0.42121279,-0.35857022,1.0653508,2.0224621,-0.44858474,0,0,0,0,0,0,0,0,0,0,0.30194545,0.15026496,-0.59735554 332 | 1991,3,1991q3,0.62565267,0.288165,1,0,0,1,0.48052478,-0.40501103,-0.56794071,1.2799364,0.97999901,-0.51324892,0,0,0,0,0,0,0,0,0,0,0.52333224,0.36921066,-0.69152075 333 | 1991,4,1991q4,0.4362728,-0.18937987,0,0,0,1,0.51749611,-0.8605901,-0.7131815,1.4229935,1.2885293,-0.24266383,0,0,0,0,0,0,0,0,0,0,0.50092477,0.2075054,-0.7701785 334 | 1992,1,1992q1,0.45713955,0.020866752,0,0,0,1,0.43136919,-0.52019101,-0.97967952,1.5660505,1.1196359,-0.32804567,0,0,0,0,0,0,0,0,0,0,0.20418343,0.12646204,-0.62231922 335 | 1992,2,1992q2,1.4534169,0.99627739,0,1,0,1,1.4614452,0.31578127,-1.2066433,1.6733433,1.6939486,-0.4209612,0,0,0,0,0,0,0,0,0,0,1.5249352,1.7832881,1.1732181 336 | 1992,3,1992q3,0.21209349,-1.2413235,0,0,0,1,0.16356988,-0.66481888,-0.72704351,1.7448719,1.1005238,0.066845305,0,0,0,0,0,0,0,0,0,0,-0.09001644,-0.011062073,-0.58651119 337 | 1992,4,1992q4,0.85222495,0.64013147,0,0,0,1,1.0751925,-0.63215458,-0.70861715,1.887929,1.5373799,0.10639718,0,0,0,0,0,0,0,0,0,0,1.3021051,0.56679732,-0.25329465 338 | 1993,1,1993q1,0.79796839,-0.054256558,0,0,0,1,0.65509301,-0.15937786,-0.58704704,1.9594575,1.3832295,0.12523142,0,0,0,0,0,0,0,0,0,0,1.0393345,0.58788472,-0.20246887 339 | 1993,2,1993q2,0.53810102,-0.25986737,0,0,0,1,0.49486911,-0.53310257,-0.56794071,1.9236932,1.965071,-0.019792141,0,0,0,0,0,0,0,0,0,0,0.43442994,0.40768424,-0.28134203 340 | 1993,3,1993q3,0.44331622,-0.094784796,0,0,0,1,0.34861875,-0.32895592,-1.2808605,1.887929,1.3683093,0.17043357,0,0,0,0,0,0,0,0,0,0,0.13384332,0.065885805,-0.24276879 341 | 1993,4,1993q4,0.90750301,0.46418679,0,0,0,1,1.1187116,-0.73322928,-0.64328575,1.8164004,1.3028038,0.10388596,0,0,0,0,0,0,0,0,0,0,0.80400234,0.85715395,-0.3116065 342 | 1994,1,1994q1,0.62727982,-0.28022319,0,0,0,1,0.78823572,-0.46014726,-2.5608497,1.7448719,1.1867325,0.44039083,0,0,0,0,0,0,0,0,0,0,0.42671803,0.28177929,-0.4394795 343 | 1994,2,1994q2,1.2382212,0.61094135,0,0,0,1,1.0600164,0.36956376,-0.9603436,1.6375791,1.5603853,0.39267746,0,0,0,0,0,0,0,0,0,0,1.2992114,0.8285594,0.92744285 344 | 1994,3,1994q3,1.5831751,0.34495389,0,0,0,1,1.3949572,0.54687738,-1.1554301,1.5302863,0.95718199,0.37070417,0,0,0,0,0,0,0,0,0,0,1.817398,1.3080204,0.80863285 345 | 1994,4,1994q4,0.87461764,-0.70855743,0,0,0,1,0.93869752,-0.58584666,-1.5472738,1.4229935,0.94787359,-0.019164333,0,0,0,0,0,0,0,0,0,0,1.0818641,0.90343153,0.20288144 346 | 1995,1,1995q1,1.2073368,0.33271915,0,0,0,1,1.0638742,0.20841907,-1.3524231,1.2441721,0.9664644,-0.17737184,0,0,0,0,0,0,0,0,0,0,1.1639259,0.82486165,0.039373461 347 | 1995,2,1995q2,0.79004568,-0.4172911,0,0,0,1,0.53282595,0.0044925548,-1.077147,1.1726437,1.1125796,-0.10831301,0,0,0,0,0,0,0,0,0,0,0.88109165,0.57849073,0.38536176 348 | 1995,3,1995q3,1.2632167,0.47317106,0,0,0,1,0.92294878,0.67731047,-1.3006206,1.1011151,0.63017452,0.32989669,0,0,0,0,0,0,0,0,0,0,0.62182832,1.0305173,0.3690815 349 | 1995,4,1995q4,1.1813362,-0.081880569,0,0,0,1,1.2469167,-0.19307101,-0.69508815,1.0653508,0.73768646,-0.053065944,0,0,0,0,0,0,0,0,0,0,0.9881233,0.77669132,0.26003036 350 | 1996,1,1996q1,0.58137125,-0.59996492,0,0,0,1,0.5394358,-0.6490373,-1.0697495,1.1011151,0.75851202,0.38577157,0,0,0,0,0,0,0,0,0,0,0.42066419,0.15055311,-0.38617653 351 | 1996,2,1996q2,0.7462393,0.16486806,0,0,0,1,0.56900585,-0.1819219,-0.36143056,1.0653508,0.90595156,-0.17737184,0,0,0,0,0,0,0,0,0,0,0.58345813,0.32242596,0.19592771 352 | 1996,3,1996q3,1.2806238,0.53438449,0,0,0,1,0.83823711,0.81890315,0.3902607,0.958058,0.4824954,-0.13405313,0,0,0,0,0,0,0,0,0,0,0.96662444,0.79569149,0.59705669 353 | 1996,4,1996q4,0.45535153,-0.82527226,0,0,0,1,0.47272611,-0.89623886,-0.47241148,0.77923667,0.49289545,-0.15163173,0,0,0,0,0,0,0,0,0,0,-0.54987448,-0.073226549,-0.63495821 354 | 1997,1,1997q1,0.47980681,0.024455279,0,0,0,1,0.36982849,-0.45490298,-1.3006206,0.49312255,0.036875296,-0.049926907,0,0,0,0,0,0,0,0,0,0,-0.25232142,-0.35427627,-0.44025695 355 | 1997,2,1997q2,2.5338299,2.054023,0,1,0,0,1.5037749,2.9377408,-1.107488,0.3500655,0.18275666,0.36442611,0,0,0,0,0,0,0,0,0,0,2.2211981,2.8306737,2.5767415 356 | 1997,3,1997q3,1.9995973,-0.53423262,0,0,0,0,1.3236871,1.8028862,-2.3721533,0.24277268,-0.078292467,0.098863497,0,0,0,0,0,0,0,0,0,0,1.1544979,2.1073966,1.8893653 357 | 1997,4,1997q4,1.7269574,-0.27263987,0,0,0,0,1.2098166,1.2764927,-1.6841677,0.1354799,-0.14119501,0.48810419,0,0,0,0,0,0,0,0,0,0,1.855882,1.750044,0.93926328 358 | 1998,1,1998q1,1.5120273,-0.21493018,0,0,0,0,0.84351313,1.5915627,-1.3741879,0.063951358,-0.42790654,0.04549985,0,0,0,0,0,0,0,0,0,0,1.1501441,1.4184589,0.99476582 359 | 1998,2,1998q2,1.1042211,-0.40780616,0,0,0,0,0.70283091,0.75136912,-1.1637267,0.028187091,-0.043479439,0.12083676,0,0,0,0,0,0,0,0,0,0,0.3416166,1.1336135,0.70977521 360 | 1998,3,1998q3,1.7405573,0.63633621,0,0,0,0,1.0086589,1.6250409,-1.172116,0.028187091,-0.42507547,0.097607881,0,0,0,0,0,0,0,0,0,0,1.2387445,1.5650015,1.0269902 361 | 1998,4,1998q4,1.6435117,-0.09704566,0,0,0,0,1.1806581,1.068011,-1.8517122,-0.0075771753,-0.24805236,0.28532234,0,0,0,0,0,0,0,0,0,0,1.0545391,1.6901804,0.55423492 362 | 1999,1,1999q1,1.2661012,-0.37741041,0,0,0,0,0.87615037,0.73344535,-1.3417727,-0.043341443,-0.47946846,-0.054949366,0,0,0,0,0,0,0,0,0,0,0.81491184,1.0938467,0.60625285 363 | 1999,2,1999q2,1.0864137,-0.1796875,0,0,0,0,0.74721086,0.5531379,-1.3524231,-0.079105705,-0.17741883,-0.20499538,0,0,0,0,0,0,0,0,0,0,0.20492703,0.76237243,-0.54998714 364 | 1999,3,1999q3,1.602473,0.51605928,0,0,0,0,1.317301,1.0676564,-1.7726036,-0.15063424,-0.6267547,0.66891277,0,0,0,0,0,0,0,0,0,0,0.78247148,1.5298617,1.1115267 365 | 1999,4,1999q4,1.6263397,0.023866653,0,0,0,0,1.2383178,0.93413603,-1.4557544,-0.18639851,-0.46995896,0.48622078,0,0,0,0,0,0,0,0,0,0,0.8009392,1.4059966,0.81539851 366 | 2000,1,2000q1,1.2093604,-0.41697931,0,0,0,0,0.94142932,0.76701611,-1.872864,-0.22216277,-0.7739315,0.30290097,0,0,0,0,0,0,0,0,0,0,0.33581054,1.0169969,0.32120708 367 | 2000,2,2000q2,1.3646277,0.15526736,0,0,0,0,0.99005121,0.89674383,-1.9401482,-0.29369131,-0.40608361,0.10011911,0,0,0,0,0,0,0,0,0,0,0.53641719,1.029088,0.11115617 368 | 2000,3,2000q3,1.3723102,0.0076824427,0,0,0,0,0.96632928,0.97600073,-0.41487238,-0.32945558,-0.76655924,-0.11773013,0,0,0,0,0,0,0,0,0,0,-0.022304541,1.1328971,0.50386035 369 | 2000,4,2000q4,1.445502,0.073191881,0,0,0,0,1.0013455,1.0337321,-0.61063546,-0.36521983,-0.54861188,-0.16921034,0,0,0,0,0,0,0,0,0,0,0.92500544,0.94751233,0.51668382 370 | 2001,1,2001q1,0.71813893,-0.72736311,0,0,0,0,0.41346067,0.27400157,-0.85374701,-0.43674839,-0.99155164,0.35626462,0,0,0,0,0,0,0,0,0,0,-0.065663442,-0.12934522,-0.55101115 371 | 2001,2,2001q2,1.2990234,0.58088446,0,1,0,0,1.0222366,0.95627743,-0.94149315,-0.43674839,-0.35557714,0.12711483,0,0,0,0,0,0,0,0,0,0,0.70018262,0.71787041,0.25331289 372 | 2001,3,2001q3,2.2437639,0.94474053,0,0,0,0,1.7631719,1.7840652,-1.3966027,-0.47251263,-0.84169787,-0.019164333,0,0,0,0,0,0,0,0,0,0,2.0958405,2.4397714,2.0762827 373 | 2001,4,2001q4,1.0996172,-1.1441467,0,0,0,0,0.56313878,0.73452479,-0.55299819,-0.47251263,-0.84170449,-0.15790981,0,0,0,0,0,0,0,0,0,0,0.67393363,0.4275488,0.32197538 374 | 2002,1,2002q1,1.809678,0.71006072,0,0,0,0,1.2829511,1.5489419,-0.57172328,-0.47251263,-0.88208914,-0.069388941,0,0,0,0,0,0,0,0,0,0,1.9418775,1.8122454,1.5043832 375 | 2002,2,2002q2,1.9999758,0.19029784,0,0,0,0,1.397366,1.6628766,-0.74593353,-0.50827688,-0.58327091,0.16227207,0,0,0,0,0,0,0,0,0,0,1.475134,1.8652925,1.3859346 376 | 2002,3,2002q3,1.3323777,-0.66759813,0,0,0,0,0.9856903,0.45451915,0.18713497,-0.50827688,-0.85990256,0.19742928,0,0,0,0,0,0,0,0,0,0,0.87276673,0.83334321,1.030436 377 | 2002,4,2002q4,1.1486602,-0.18371749,0,0,0,0,0.63252962,0.69150227,0.00001437,-0.50827688,-0.71389318,0.21877475,0,0,0,0,0,0,0,0,0,0,0.62535423,0.7963863,0.68319249 378 | 2003,1,2003q1,1.2433448,0.094684601,0,0,1,0,0.61207038,1.1071446,-0.95400739,-0.50827688,-0.88913578,0.25204855,0,0,0,0,0,1,0,0,0,0,0.95951438,0.79931706,0.69468141 379 | 2003,2,2003q2,1.5075419,0.26419711,0,0,1,0,1.1352775,0.97127181,-1.3632267,-0.50827688,-0.68577647,0.52828389,0,0,0,0,0,1,0,0,0,0,0.599042,1.3376424,1.3231245 380 | 2003,3,2003q3,1.3421476,-0.16539431,0,0,0,0,1.1649698,0.6634019,-1.2245213,-0.50827688,-0.84228665,0.43976301,0,0,0,0,0,0,0,0,0,0,0.91831386,1.3395797,1.0518419 381 | 2003,4,2003q4,1.6699209,0.32777333,0,0,0,0,1.2897273,1.1131599,-0.4625102,-0.54404116,-0.7344836,0.27339402,0,0,0,0,0,0,0,0,0,0,1.5215545,1.3446811,1.2482696 382 | 2004,1,2004q1,1.5349842,-0.13493669,0,0,0,0,1.0432535,1.1295347,-0.074386805,-0.57980543,-0.98315775,-0.15288734,0,0,0,0,0,0,0,0,0,0,1.1883613,1.5931541,1.6314709 383 | 2004,2,2004q2,1.4867733,-0.048210979,0,0,0,0,1.0035386,1.1846527,-0.3088643,-0.61556971,-0.74338996,-0.16481569,0,0,0,0,0,0,0,0,0,0,0.548917,1.092401,0.94303346 384 | 2004,3,2004q3,1.7372605,0.25048721,0,0,0,0,1.2432112,1.0713476,-0.96673375,-0.65133399,-1.0056322,-0.2727986,0,0,0,0,0,0,0,0,0,0,0.99576873,1.4943277,1.3366591 385 | 2004,4,2004q4,1.6690514,-0.068209052,0,0,0,0,1.2831481,1.1520268,-0.92310435,-0.65133399,-0.79221159,-0.099523708,0,0,0,0,0,0,0,0,0,0,1.1022975,1.5936149,0.87169659 386 | 2005,1,2005q1,1.4268585,-0.24219286,0,0,0,0,0.93555689,1.0970674,-2.6164925,-0.65133399,-0.96334666,0.089446381,0,0,0,0,0,0,0,0,0,0,0.64788634,1.2721851,1.0360862 387 | 2005,2,2005q2,2.3980191,0.97116053,0,1,0,0,1.8300183,2.1203647,-1.7183025,-0.61556971,-0.67364877,0.29097262,0,0,0,0,0,0,0,0,0,0,2.178932,2.7347019,2.4031987 388 | 2005,3,2005q3,1.4887248,-0.90929425,0,0,0,0,1.5722505,0.33858761,-1.4557544,-0.61556971,-0.8195802,0.29285604,0,0,0,0,0,0,0,0,0,0,0.72815645,1.4104377,1.4713542 389 | 2005,4,2005q4,1.2977811,-0.19094372,0,0,0,0,0.9848361,0.55788851,-1.7540635,-0.57980543,-0.80164832,0.49249884,0,0,0,0,0,0,0,0,0,0,0.2689929,1.0711026,1.2907946 390 | 2006,1,2006q1,2.0200937,0.72231257,0,0,1,0,1.6537228,1.3404453,0.22613081,-0.57980543,-0.95743805,0.012226047,0,0,0,0,0,0,1,0,0,0,1.5395142,1.8683717,1.7278693 391 | 2006,2,2006q2,2.0138986,-0.0061950684,0,0,1,0,1.5484488,1.600232,-1.1472241,-0.54404116,-0.47903791,-0.039254177,0,0,0,0,0,0,1,0,0,0,1.6708561,2.3852186,2.0452077 392 | 2006,3,2006q3,1.8113534,-0.20254517,0,0,1,0,1.1040461,1.5944095,-1.7183025,-0.54404116,-0.71703941,-0.24391945,0,0,0,0,0,0,1,0,0,0,0.34755662,1.622736,0.90527689 393 | 2006,4,2006q4,1.60105,-0.21030343,0,0,1,0,1.2361997,0.86060297,-1.6515174,-0.54404116,-0.67761374,0.11895334,0,0,0,0,0,0,1,0,0,0,0.73247391,1.1278585,0.86936492 394 | 2007,1,2007q1,2.174175,0.573125,0,0,1,0,1.7080452,1.5341512,-0.82133168,-0.57980543,-0.92499655,0.12523142,0,0,0,0,0,0,1,0,0,0,1.5142505,1.8766148,1.3437779 395 | 2007,2,2007q2,1.9780324,-0.19614267,0,0,1,0,1.6722312,1.2946229,-0.54930854,-0.61556971,-0.67528576,0.023526583,0,0,0,0,0,0,1,0,0,0,0.99621379,1.5767065,1.0315778 396 | 2007,3,2007q3,2.9021773,0.92414498,0,0,0,0,2.4502761,2.1133406,-0.30619702,-0.65133399,-1.0465031,0.17545602,0,0,0,0,0,0,0,0,0,0,2.3044741,2.901228,2.2063046 397 | 2007,4,2007q4,2.2555554,-0.64662194,0,0,0,0,1.8637995,1.4434521,-0.021542257,-0.68709826,-0.69510084,-0.059971828,0,0,0,0,0,0,0,0,0,0,2.0847857,2.3317053,1.6421627 398 | 2008,1,2008q1,2.291095,0.035539627,0,0,0,0,1.877279,1.7030667,-1.2245213,-0.72286248,-0.95334065,-0.15728201,0,0,0,0,0,0,0,0,0,0,1.6658257,2.1469188,1.6993695 399 | 2008,2,2008q2,1.8563952,-0.43469977,1,0,0,0,1.6340625,0.97951341,-0.75073123,-0.68709826,-0.39574194,-0.49880934,0,0,0,0,0,0,0,0,0,0,0.99707955,1.3715609,0.70271695 400 | 2008,3,2008q3,2.2322655,0.37587023,1,0,0,0,1.8885837,1.3799512,0.14014781,-0.57980543,-0.55530304,-1.3984576,0,0,0,0,0,0,0,0,0,0,1.2670194,2.1730199,1.4494987 401 | 2008,4,2008q4,2.3236177,0.091352224,1,0,0,0,1.8707303,1.679074,-2.8925941,-0.40098411,-0.7420274,-1.7531689,0,0,0,0,0,0,0,0,0,0,1.6222166,2.4622724,1.4130672 402 | 2009,1,2009q1,1.6341497,-0.68946803,1,0,0,0,1.3394305,0.86627996,-2.1255,-0.11486997,-0.6439898,-1.4775614,0,0,0,0,0,0,0,0,0,0,0.82633471,1.3250244,0.45836428 403 | 2009,2,2009q2,2.1090522,0.47490251,1,0,0,0,1.8348335,1.3700618,-1.5472738,0.099715628,0.012625713,-0.51764357,0,0,0,0,0,0,0,0,0,0,1.4773389,1.85511,1.2896076 404 | 2009,3,2009q3,1.5772594,-0.53179276,0,0,0,0,1.2219108,0.88693839,-0.86487073,0.1354799,-0.075514995,-0.22822426,0,0,0,0,0,0,0,0,0,0,-0.25563192,0.841088,0.63083947 405 | 2009,4,2009q4,2.1399024,0.56264293,0,0,0,0,1.7312551,1.5838183,-0.41795591,0.17124416,-0.040955082,-0.10956863,0,0,0,0,0,0,0,0,0,0,1.9098799,2.1353271,1.0825697 406 | 2010,1,2010q1,1.7257512,-0.41415119,0,0,0,0,1.3943932,1.058652,-0.30089071,0.1354799,-0.052024584,-0.034231715,0,0,0,0,0,0,0,0,0,0,1.131721,1.6188179,0.90900749 407 | 2010,2,2010q2,3.3992476,1.6734965,0,1,0,1,2.2367246,4.0216932,-2.221482,0.063951358,0.042694893,0.25958225,0,0,0,0,0,0,0,0,0,0,3.171381,4.5136886,2.8823278 408 | 2010,3,2010q3,2.4138584,-0.98538923,0,0,0,1,1.6327608,1.95309,-2.509047,-0.0075771753,-0.33405334,0.052405734,0,0,0,0,0,0,0,0,0,0,2.6729715,3.1174116,2.0385334 409 | 2010,4,2010q4,2.2753241,-0.13853431,0,0,0,1,1.5603627,1.579845,-1.7010411,-0.0075771753,-0.1155672,-0.3380906,0,0,0,0,0,0,0,0,0,0,2.1386266,2.8887119,2.0878577 410 | 2011,1,2011q1,2.5362637,0.2609396,0,0,0,1,1.7694594,1.8249753,-1.4681652,-0.0075771753,0.060830604,-0.01728091,0,0,0,0,0,0,0,0,0,0,2.1465786,2.9911122,1.9836904 411 | 2011,2,2011q2,2.1390169,-0.39724684,0,0,0,1,1.6771519,0.98007256,-0.26981714,0.063951358,0.14486873,-0.21064565,0,0,0,0,0,0,0,0,0,0,1.6813524,2.6439073,1.6305197 412 | 2011,3,2011q3,2.2998695,0.16085267,0,0,0,1,1.6592748,1.4242922,-2.4150698,0.1354799,-0.12491538,0.083168305,0,0,0,0,0,0,0,0,0,0,2.2593834,2.7879894,1.9290599 413 | 2011,4,2011q4,2.1536317,-0.14623785,0,0,0,1,1.7123544,0.68168306,0.65542835,0.17124416,0.034417804,-0.3632029,0,0,0,0,0,0,0,0,0,0,1.4264773,2.626523,1.0700088 414 | 2012,1,2012q1,2.1985033,0.044871569,0,0,0,1,1.6477541,0.99780774,-1.5334966,0.1354799,-0.19734719,-0.31109485,0,0,0,0,0,0,0,0,0,0,1.7395753,2.7051251,1.6071092 415 | 2012,2,2012q2,2.255573,0.057069778,0,0,0,1,1.9195768,0.94736397,-0.76531106,0.099715628,-0.058971923,-0.46553552,0,0,0,0,0,0,0,0,0,0,1.1728079,2.7951014,1.43857 416 | 2012,3,2012q3,2.450412,0.194839,0,0,0,1,1.9316281,1.1483122,-2.1255,0.099715628,-0.2425542,0.16917795,0,0,0,0,0,0,0,0,0,0,1.7605911,3.2122245,2.1763511 417 | 2012,4,2012q4,2.3618805,-0.088531494,0,0,0,1,1.7564219,1.4914253,-2.4605892,0.099715628,0.074721657,-0.56598473,0,0,0,0,0,0,0,0,0,0,1.5503719,2.7973626,1.4746572 418 | 2013,1,2013q1,1.9804429,-0.38143766,0,0,0,1,1.4821373,0.88653094,-1.123131,0.063951358,-0.23180717,0.020387545,0,0,0,0,0,0,0,0,0,0,1.3378586,2.6517322,0.99120575 419 | 2013,2,2013q2,2.2708473,0.29040444,0,0,0,1,1.6485268,1.7167462,-0.94149315,-0.0075771753,-0.18643819,0.049894501,0,0,0,0,0,0,0,0,0,0,1.34247,2.5876889,1.4381171 420 | 2013,3,2013q3,2.2398338,-0.031013489,0,0,0,1,1.7501242,1.4206462,-2.2564111,-0.079105705,-0.3970857,0.096352264,0,0,0,0,0,0,0,0,0,0,0.98519653,2.6641772,2.4419382 421 | 2013,4,2013q4,2.1094322,-0.13040161,0,0,0,1,1.786379,0.72110015,-0.53834718,-0.22216277,-0.42899454,-0.0988959,0,0,0,0,0,0,0,0,0,0,0.5106557,2.4097817,1.5985819 422 | -------------------------------------------------------------------------------- /emotion-main-models.R: -------------------------------------------------------------------------------- 1 | #!/usr/bin/Rscript 2 | 3 | #===========================================================================================# 4 | # 5 | # Description: 6 | # R script for empirical analysis --- Models and figures 7 | # 8 | # Usage: 9 | # For interactive computing in IDE. 10 | # 11 | # 1) Set working directory using setwd() below: 12 | setwd("/home/user") 13 | # 14 | # 2) Requires data files emotion-final-y.csv and emotion-final-q.csv in wd. 15 | # 16 | # 3) Output files will be saved in a subdirectory named of/ in the wd. 17 | # 18 | # Computed using R 3.2.4. 19 | # 20 | # Author: L. Rheault 21 | # 22 | #===========================================================================================# 23 | 24 | #-------------------------------------------------------------------------------------------# 25 | # Loading required packages 26 | #-------------------------------------------------------------------------------------------# 27 | # Note: Installation of those packages will be required if they are not already. 28 | 29 | library(zoo) # For basic time-series tools. 30 | library(urca) # For unit root testing. 31 | library(fUnitRoots) # For unit root testing. 32 | library(TSA) # For ARIMAX models. 33 | library(vars) # For vector autoregression models (VAR, VECM). 34 | library(nlWaldTest) # Non-linear Wald Tests. 35 | library(lmtest) # For tests of serial correlation. 36 | library(tseries) # For tests such as normality of residuals. 37 | library(quantmod) # For an alternative lag function. 38 | library(tsDyn) # For vector autoregression models (VAR, VECM). 39 | library(sandwich) # For HAC standard errors. 40 | library(aod) # For flexible Wald tests used for Granger causality analysis. 41 | library(forecast) # For automatic lag selection in ARIMA models and other tools. 42 | 43 | # For plotting: 44 | library(ggplot2) 45 | library(reshape2) 46 | library(colorRamps) 47 | library(gridExtra) 48 | #-------------------------------------------------------------------------------------------# 49 | # Note: Some figures below include transparency, incompatible with eps format. 50 | # Save figures as tiff or pdf to see intended result, or else reset alpha=1 to save as eps. 51 | #-------------------------------------------------------------------------------------------# 52 | 53 | #-------------------------------------------------------------------------------------------# 54 | # Loading data 55 | #-------------------------------------------------------------------------------------------# 56 | 57 | dy <- read.csv("emotion-final-y.csv") 58 | dq <- read.csv("emotion-final-q.csv") 59 | 60 | #-------------------------------------------------------------------------------------------# 61 | # Table 3. Mean difference tests (t-tests). 62 | #-------------------------------------------------------------------------------------------# 63 | 64 | # 1) The usual t-test, one-sided following theoretical expectation. 65 | # 2) Two-sided t-test. 66 | # 3) Welch t-test with unequal group variances. 67 | 68 | # Recession binary variable 69 | t.test(dpolar~recession, data = dy, var.equal=TRUE, alternative = "greater") 70 | t.test(dpolar~recession, data = dy, var.equal=TRUE, alternative = "two.sided") 71 | t.test(dpolar~recession, data = dy, var.equal=FALSE, alternative = "greater") 72 | 73 | # Election binary variable 74 | t.test(dpolar~election, data = dy, var.equal=TRUE, alternative = "less") 75 | t.test(dpolar~election, data = dy, var.equal=TRUE, alternative = "two.sided") 76 | t.test(dpolar~election, data = dy, var.equal=FALSE, alternative = "less") 77 | 78 | # Wars binary variable 79 | t.test(dpolar~wars, data = dy, var.equal=TRUE, alternative = "greater") 80 | t.test(dpolar~wars, data = dy, var.equal=TRUE, alternative = "two.sided") 81 | t.test(dpolar~wars, data = dy, var.equal=FALSE, alternative = "greater") 82 | 83 | #-------------------------------------------------------------------------------------------# 84 | # Quarterly Version 85 | 86 | # Recession binary variable 87 | t.test(dpolar~recession, data = dq, var.equal=TRUE, alternative = "greater") 88 | t.test(dpolar~recession, data = dq, var.equal=TRUE, alternative = "two.sided") 89 | t.test(dpolar~recession, data = dq, var.equal=FALSE, alternative = "greater") 90 | 91 | # Election binary variable 92 | t.test(dpolar~election, data = dq, var.equal=TRUE, alternative = "less") 93 | t.test(dpolar~election, data = dq, var.equal=TRUE, alternative = "two.sided") 94 | t.test(dpolar~election, data = dq, var.equal=FALSE, alternative = "less") 95 | 96 | # Wars binary variable 97 | t.test(dpolar~wars, data = dq, var.equal=TRUE, alternative = "greater") 98 | t.test(dpolar~wars, data = dq, var.equal=TRUE, alternative = "two.sided") 99 | t.test(dpolar~wars, data = dq, var.equal=FALSE, alternative = "greater") 100 | 101 | #-------------------------------------------------------------------------------------------# 102 | # Tables S5, S6, S12, S13. Unit root tests. 103 | #-------------------------------------------------------------------------------------------# 104 | # Will save six output files in /of directory. 105 | 106 | pol <- ts(dy$polar,start=c(1909),frequency=1, names="Polarity") 107 | disp <- ts(dy$ldisp,start=c(1909),frequency=1, names="Disputes") 108 | mis <- ts(dy$misery,start=c(1909),frequency=1, names="Misery") 109 | gdpg <- ts(dy$gdpg,start=c(1909),frequency=1, names="GDPg") 110 | unemp <- ts(dy$unemp,start=c(1909),frequency=1, names="Unemployment") 111 | polg <- ts(dy$polarg[38:nrow(dy)],start=c(1946),frequency=1, names="Polarity-Gov") 112 | polo <- ts(dy$polaro[38:nrow(dy)],start=c(1946),frequency=1, names="Polarity-Opp") 113 | series <- c("pol","polg","polo","disp","mis","gdpg","unemp") 114 | 115 | #Storing results in matrices. 116 | uradf <- matrix(nrow=14,ncol=7) 117 | colnames(uradf)<-c("Statistic","1pct","5pct","10pct","Lag","Model","Outcome") 118 | rownames(uradf) <- c(rep("Polarity",2), rep("Polarity-Gov",2), rep("Polarity-Opp",2), 119 | rep("Disputes",2), rep("Misery",2), rep("GDPg",2), rep("Unemployment",2)) 120 | urkpss <- matrix(nrow=14,ncol=8) 121 | colnames(urkpss)<-c("Statistic","10pct","5pct","2.5pct","1pct","Type","Lag","Outcome") 122 | rownames(urkpss) <- c(rep("Polarity",2), rep("Polarity-Gov",2), rep("Polarity-Opp",2), 123 | rep("Disputes",2), rep("Misery",2), rep("GDPg",2), rep("Unemployment",2)) 124 | 125 | for (l in 1:2){ 126 | j = 1 127 | for (i in 1:length(series)){ 128 | adf1 <- ur.df(get(series[i]) , type = "drift", lags=l) 129 | adf2 <- ur.df(get(series[i]) , type = "trend", lags=l) 130 | uradf[j,1:4] <- cbind(adf1@teststat[1], t(adf1@cval[1,1:3])); uradf[j,5]<-adf1@lags; uradf[j,6]<-adf1@model; 131 | uradf[j+1,1:4] <- cbind(adf2@teststat[1], t(adf2@cval[1,1:3])); uradf[j+1,5]<-adf2@lags; uradf[j+1,6]<-adf2@model; 132 | j = j+2 133 | } 134 | for (i in 1:nrow(uradf)){ 135 | if (as.numeric(uradf[i,1])<=as.numeric(uradf[i,3])){ 136 | uradf[i,7] <- "Stationary" 137 | } 138 | else { 139 | uradf[i,7] <- "Non-Stationary" 140 | } 141 | } 142 | write.table(uradf,paste("of/unitroot-y-adfs-",l,".csv",sep=""),sep=",",row.names=T,col.names=NA) 143 | } 144 | 145 | j = 1 146 | for (i in 1:length(series)){ 147 | kpss1 <- ur.kpss(get(series[i]), type = "mu", lags="short") 148 | kpss2 <- ur.kpss(get(series[i]), type = "tau", lags="short") 149 | urkpss[j,1:7] <- cbind(kpss1@teststat, kpss1@cval, kpss1@type, kpss1@lag) 150 | urkpss[j+1,1:7] <- cbind(kpss2@teststat, kpss2@cval, kpss2@type, kpss2@lag) 151 | j = j+2 152 | } 153 | rm(j) 154 | for (i in 1:nrow(urkpss)){ 155 | if (as.numeric(urkpss[i,1])>as.numeric(urkpss[i,3])){ 156 | urkpss[i,8] <- "Non-Stationary" 157 | } 158 | else { 159 | urkpss[i,8] <- "Stationary" 160 | } 161 | } 162 | write.table(urkpss,"of/unitroot-y-kpss.csv",sep=",",row.names=T,col.names=NA) 163 | 164 | #-------------------------------------------------------------------------------------------# 165 | # Quarterly Version 166 | 167 | pol <- ts(dq$polar,start=c(1909, 1),frequency=4, names="Polarity") 168 | disp <- ts(dq$ldisp[89:nrow(dq)],start=c(1931, 1),frequency=4, names="Disputes") 169 | mis <- ts(dq$misery[186:nrow(dq)],start=c(1955, 2),frequency=4, names="Misery") 170 | gdpg <- ts(dq$gdpg[186:nrow(dq)],start=c(1955, 2),frequency=4, names="GDPg") 171 | unemp <- ts(dq$unemp[185:nrow(dq)],start=c(1955, 1),frequency=4, names="Unemployment") 172 | polg <- ts(dq$polarg[147:nrow(dq)],start=c(1945, 3),frequency=4, names="Polarity-Gov") 173 | polo <- ts(dq$polaro[147:nrow(dq)],start=c(1945, 3),frequency=4, names="Polarity-Opp") 174 | series <- c("pol","polg","polo","disp","mis","gdpg","unemp") 175 | 176 | #Storing results in matrices. 177 | uradf <- matrix(nrow=14,ncol=7) 178 | colnames(uradf)<-c("Statistic","1pct","5pct","10pct","Lag","Model","Outcome") 179 | rownames(uradf) <- c(rep("Polarity",2), rep("Polarity-Gov",2), rep("Polarity-Opp",2), 180 | rep("Disputes",2), rep("Misery",2), rep("GDPg",2), rep("Unemployment",2)) 181 | urkpss <- matrix(nrow=14,ncol=8) 182 | colnames(urkpss)<-c("Statistic","10pct","5pct","2.5pct","1pct","Type","Lag","Outcome") 183 | rownames(urkpss) <- c(rep("Polarity",2), rep("Polarity-Gov",2), rep("Polarity-Opp",2), 184 | rep("Disputes",2), rep("Misery",2), rep("GDPg",2), rep("Unemployment",2)) 185 | 186 | for (l in 4:5){ 187 | j = 1 188 | for (i in 1:length(series)){ 189 | adf1 <- ur.df(get(series[i]) , type = "drift", lags=l) 190 | adf2 <- ur.df(get(series[i]) , type = "trend", lags=l) 191 | uradf[j,1:4] <- cbind(adf1@teststat[1], t(adf1@cval[1,1:3])); uradf[j,5]<-adf1@lags; uradf[j,6]<-adf1@model; 192 | uradf[j+1,1:4] <- cbind(adf2@teststat[1], t(adf2@cval[1,1:3])); uradf[j+1,5]<-adf2@lags; uradf[j+1,6]<-adf2@model; 193 | j = j+2 194 | } 195 | for (i in 1:nrow(uradf)){ 196 | if (as.numeric(uradf[i,1])<=as.numeric(uradf[i,3])){ 197 | uradf[i,7] <- "Stationary" 198 | } 199 | else { 200 | uradf[i,7] <- "Non-Stationary" 201 | } 202 | } 203 | write.table(uradf,paste("of/unitroot-q-adfs-",l,".csv",sep=""),sep=",",row.names=T,col.names=NA) 204 | } 205 | 206 | j = 1 207 | for (i in 1:length(series)){ 208 | kpss1 <- ur.kpss(get(series[i]), type = "mu", lags="short") 209 | kpss2 <- ur.kpss(get(series[i]), type = "tau", lags="short") 210 | urkpss[j,1:7] <- cbind(kpss1@teststat, kpss1@cval, kpss1@type, kpss1@lag) 211 | urkpss[j+1,1:7] <- cbind(kpss2@teststat, kpss2@cval, kpss2@type, kpss2@lag) 212 | j = j+2 213 | } 214 | rm(j) 215 | for (i in 1:nrow(urkpss)){ 216 | if (as.numeric(urkpss[i,1])>as.numeric(urkpss[i,3])){ 217 | urkpss[i,8] <- "Non-Stationary" 218 | } 219 | else { 220 | urkpss[i,8] <- "Stationary" 221 | } 222 | } 223 | write.table(urkpss,"of/unitroot-q-kpss.csv",sep=",",row.names=T,col.names=NA) 224 | 225 | #-------------------------------------------------------------------------------------------# 226 | # Tables 2, S10, S15. ARIMA models 227 | #-------------------------------------------------------------------------------------------# 228 | 229 | dpol <- ts(dy$dpolar[2:nrow(dy)],start=c(1910),frequency=1, names="Polarity") 230 | X <- ts(cbind(dy$recession[2:nrow(dy)],dy$election[2:nrow(dy)]),start=c(1910),frequency=1, 231 | names=c("Recession","Election")) 232 | 233 | a1 <- arimax(dpol, order = c(1, 0, 0), xreg= X) 234 | 235 | a2 <- arimax(dpol, order = c(2, 0, 0), xreg= X) 236 | cat(paste(replicate(72, "-"),collapse = "")) 237 | cat("ARMAX model 1: Arima(1,0,0)") 238 | a1 239 | AIC(a1) 240 | BIC(a1) 241 | a1$loglik 242 | # Stability test: Uncomment to perform. Requires function arroots from forecast package. 243 | # plot(arroots(a1)) 244 | cat("Normality test:") 245 | jarque.bera.test(resid(a1)) 246 | cat("ARMAX model 2: Arima(2,0,0)") 247 | a2 248 | AIC(a2) 249 | BIC(a2) 250 | a2$loglik 251 | # Stability test: Uncomment to perform. Requires function arroots from forecast package. 252 | # plot(arroots(a2)) 253 | cat("Normality test:") 254 | jarque.bera.test(resid(a2)) 255 | cat(paste(replicate(72, "-"),collapse = "")) 256 | 257 | 258 | X <- ts(cbind(dy$recession[2:nrow(dy)], dy$election[2:nrow(dy)], 259 | dy$wars[2:nrow(dy)]), 260 | start=c(1909),frequency=1, 261 | names=c("Recession","Election","Wars")) 262 | a3 <- arimax(dpol, order = c(1, 0, 0), xreg= X) 263 | a4 <- arimax(dpol, order = c(2, 0, 0), xreg= X) 264 | cat(paste(replicate(72, "-"),collapse = "")) 265 | cat("ARMAX model 3") 266 | a3 267 | AIC(a3) 268 | BIC(a3) 269 | a3$loglik 270 | # Stability test: Uncomment to perform. Requires function arroots from forecast package. 271 | # plot(arroots(a3)) 272 | cat("Normality test:") 273 | jarque.bera.test(resid(a3)) 274 | cat("ARMAX model 4") 275 | a4 276 | AIC(a4) 277 | BIC(a4) 278 | a4$loglik 279 | # Stability test: Uncomment to perform. Requires function arroots from forecast package. 280 | # plot(arroots(a4)) 281 | cat("Normality test:") 282 | jarque.bera.test(resid(a4)) 283 | cat(paste(replicate(72, "-"),collapse = "")) 284 | 285 | dpol <- ts(dy$dpolar[38:nrow(dy)],start=c(1946),frequency=1, names="Polarity") 286 | X <- ts(cbind(dy$recession[38:nrow(dy)], dy$election[38:nrow(dy)], 287 | dy$wars[38:nrow(dy)], dy$conservative[38:nrow(dy)]), 288 | start=c(1946),frequency=1, 289 | names=c("Recession","Election","Wars", "Party")) 290 | a5 <- arimax(dpol, order = c(1, 0, 0), xreg= X) 291 | a6 <- arimax(dpol, order = c(2, 0, 0), xreg= X) 292 | cat(paste(replicate(72, "-"),collapse = "")) 293 | cat("ARMAX model 5") 294 | a5 295 | AIC(a5) 296 | BIC(a5) 297 | a5$loglik 298 | # Stability test: Uncomment to perform. Requires function arroots from forecast package. 299 | # plot(arroots(a5)) 300 | cat("Normality test:") 301 | jarque.bera.test(resid(a5)) 302 | cat("ARMAX model 6") 303 | a6 304 | AIC(a6) 305 | BIC(a6) 306 | a6$loglik 307 | # Stability test: Uncomment to perform. Requires function arroots from forecast package. 308 | # plot(arroots(a6)) 309 | cat("Normality test:") 310 | jarque.bera.test(resid(a6)) 311 | cat(paste(replicate(72, "-"),collapse = "")) 312 | 313 | #-------------------------------------------------------------------------------------------# 314 | # Quarterly Version; In SI only. 315 | 316 | dpol <- ts(dq$dpolar[45:nrow(dq)],start=c(1920,2),frequency=4, names="Polarity") 317 | X <- ts(cbind(dq$recession[45:nrow(dq)],dq$election[45:nrow(dq)]),start=c(1920,2),frequency=4, 318 | names=c("Recession","Election")) 319 | 320 | # Automatic selection based on IC (package forecast required.) 321 | auto.arima(dpol,xreg=X) 322 | # Best model: (3,0,0)(2,0,0)[4] 323 | 324 | a1 <- arima( dpol, order = c(3, 0, 0), seasonal = list(order = c(2,0,0), period = 4), xreg=X) 325 | cat(paste(replicate(72, "-"),collapse = "")) 326 | cat("ARMAX model 1: Arima(3,0,0)(2,0,0)[4]") 327 | a1 328 | AIC(a1) 329 | BIC(a1) 330 | a1$loglik 331 | cat(paste(replicate(72, "-"),collapse = "")) 332 | 333 | X <- ts(cbind(dq$recession[45:nrow(dq)], dq$election[45:nrow(dq)], dq$wars[45:nrow(dq)]), 334 | start=c(1920,2),frequency=4, 335 | names=c("Recession","Election", "Wars")) 336 | a2 <- arima( dpol, order = c(3, 0, 0), seasonal = list(order = c(2,0,0), period = 4), xreg=X) 337 | cat(paste(replicate(72, "-"),collapse = "")) 338 | cat("ARMAX model 2") 339 | a2 340 | AIC(a2) 341 | BIC(a2) 342 | a2$loglik 343 | cat(paste(replicate(72, "-"),collapse = "")) 344 | 345 | dpol <- ts(dq$dpolar[186:nrow(dq)],start=c(1955,2),frequency=4, names="Polarity") 346 | X <- ts(cbind(dq$recession[186:nrow(dq)], dq$election[186:nrow(dq)], dq$wars[186:nrow(dq)], 347 | dq$conservatives[186:nrow(dq)]), 348 | start=c(1955,2),frequency=4, names=c("Recession","Election", "Wars", "Party")) 349 | a3 <- arima( dpol, order = c(3, 0, 0), seasonal = list(order = c(2,0,0), period = 4), xreg=X) 350 | cat(paste(replicate(72, "-"),collapse = "")) 351 | cat("ARMAX model 3") 352 | a3 353 | AIC(a3) 354 | BIC(a3) 355 | a3$loglik 356 | cat(paste(replicate(72, "-"),collapse = "")) 357 | 358 | #-------------------------------------------------------------------------------------------# 359 | # Table 4, Table S9, Table S14. Granger Causality/Cointegration 360 | #-------------------------------------------------------------------------------------------# 361 | # Will save 8 output files in the \of directory. 362 | 363 | # Declaring variables as multivariate time series. 364 | pol.disp <- ts(cbind(dy$polar, dy$ldisp), start=c(1909), frequency=1, names=c("Polarity", "Disputes")) 365 | pol.mis <- ts(cbind(dy$polar, dy$misery), start=c(1909), frequency=1, names=c("Polarity", "Misery")) 366 | pol.gdpg <- ts(cbind(dy$polar, dy$gdpg), start=c(1909), frequency=1, names=c("Polarity", "GDPg")) 367 | pol.unemp <- ts(cbind(dy$polar, dy$unemp), start=c(1909), frequency=1, names=c("Polarity", "Unemp")) 368 | # Government only. 369 | polg.disp <- ts(cbind(dy$polarg, dy$ldisp)[38:nrow(dy),], start=c(1946), frequency=1, names=c("Polarity-Gov", "Disputes")) 370 | # Opposition only. 371 | polo.disp <- ts(cbind(dy$polaro, dy$ldisp)[38:nrow(dy),], start=c(1946), frequency=1, names=c("Polarity-Opp", "Disputes")) 372 | 373 | series <- c("pol.disp","pol.mis","pol.gdpg","pol.unemp", 374 | "polg.disp","polo.disp") 375 | 376 | # Automatic lag length selection (Bayesian criterion) 377 | bc <- matrix(nrow=length(series),ncol=1) 378 | for (i in 1:nrow(bc)){ 379 | bc[i] <- VARselect(get(series[i]),lag=12,type="const")$selection[3] 380 | } 381 | 382 | # Automatic lag length selection (Hannan-Quinn criterion) 383 | hq <- matrix(nrow=length(series),ncol=1) 384 | for (i in 1:nrow(hq)){ 385 | hq[i] <- VARselect(get(series[i]),lag=12,type="const")$selection[2] 386 | } 387 | 388 | gc <- matrix(ncol=4,nrow=2*length(series)) 389 | colnames(gc) <- c("Chi-2","Lags","p-value","Outcome") 390 | rownames(gc) <- c("Disputes -> Polarity", "Polarity -> Disputes", 391 | "Misery -> Polarity", "Polarity -> Misery", 392 | "GDPg -> Polarity", "Polarity -> GDPg", 393 | "Unemployment -> Polarity", "Polarity -> Unemployment", 394 | "Disputes -> Polarity-Gov", "PolarityGov -> Disputes", 395 | "Disputes -> Polarity-Opp", "Polarity-Opp -> Disputes") 396 | 397 | j <- 1 398 | for (i in 1:length(series)){ 399 | tempvar <- VAR(get(series[i]),p=bc[i]+1,type="const") 400 | gc[j,1:3] <- wald.test(b=coef(tempvar$varresult[[1]]), 401 | Sigma=vcov(tempvar$varresult[[1]]), 402 | Terms=seq(from=2,to=bc[i]*2,by=2), df=bc[i])$result[[1]] 403 | gc[j+1,1:3] <- wald.test(b=coef(tempvar$varresult[[2]]), 404 | Sigma=vcov(tempvar$varresult[[2]]), 405 | Terms=seq(from=1,to=bc[i]*2-1,by=2), df=bc[i])$result[[1]] 406 | j = j + 2 407 | } 408 | rm(j) 409 | for (i in 1:nrow(gc)){ 410 | if (as.numeric(gc[i,3])<0.05){ 411 | gc[i,4] <- "*" 412 | } 413 | else { 414 | gc[i,4] <- "" 415 | } 416 | } 417 | write.table(gc,"of/y-granger-bic.csv",sep=",",row.names=T,col.names=NA) 418 | j <- 1 419 | for (i in 1:length(series)){ 420 | tempvar <- VAR(get(series[i]),p=hq[i]+1,type="const") 421 | gc[j,1:3] <- wald.test(b=coef(tempvar$varresult[[1]]), 422 | Sigma=vcov(tempvar$varresult[[1]]), 423 | Terms=seq(from=2,to=hq[i]*2,by=2), df=hq[i])$result[[1]] 424 | gc[j+1,1:3] <- wald.test(b=coef(tempvar$varresult[[2]]), 425 | Sigma=vcov(tempvar$varresult[[2]]), 426 | Terms=seq(from=1,to=hq[i]*2-1,by=2), df=hq[i])$result[[1]] 427 | j = j + 2 428 | } 429 | rm(j) 430 | for (i in 1:nrow(gc)){ 431 | if (as.numeric(gc[i,3])<0.05){ 432 | gc[i,4] <- "*" 433 | } 434 | else { 435 | gc[i,4] <- "" 436 | } 437 | } 438 | write.table(gc,"of/y-granger-hq.csv",sep=",",row.names=T,col.names=NA) 439 | 440 | # Cointegration tests (Johansen rank tests). 441 | # Restricting to pairs of series that are both I(1) according to the ADF tests: 442 | polg.mis <- ts(cbind(dy$polarg, dy$misery)[38:nrow(dy),], start=c(1946), frequency=1, names=c("Polarity-Gov", "Misery")) 443 | polg.unemp <- ts(cbind(dy$polarg, dy$unemp)[38:nrow(dy),], start=c(1946), frequency=1, names=c("Polarity-Gov", "Unemp")) 444 | polo.mis <- ts(cbind(dy$polaro, dy$misery)[38:nrow(dy),], start=c(1946), frequency=1, names=c("Polarity-Opp", "Misery")) 445 | polo.unemp <- ts(cbind(dy$polaro, dy$unemp)[38:nrow(dy),], start=c(1946), frequency=1, names=c("Polarity-Opp", "Unemp")) 446 | 447 | series <- c("pol.disp","pol.mis","pol.unemp", 448 | "polg.disp","polg.mis","polg.unemp", 449 | "polo.disp","polo.mis","polo.unemp") 450 | 451 | co <- matrix(nrow=18,ncol=6) 452 | colnames(co) <- c("trace","p-val","var1","var2","lags","outcome") 453 | rownames(co) <- c("r = 0","r = 1", "r = 0","r = 1", "r = 0","r = 1", 454 | "r = 0","r = 1", "r = 0","r = 1", "r = 0","r = 1", 455 | "r = 0","r = 1", "r = 0","r = 1", "r = 0","r = 1") 456 | 457 | j <- 1 458 | for (i in 1:length(series)){ 459 | temp.vecm <- suppressWarnings( VECM(get(series[i]), 0, LRinclude="const", estim="ML") ) 460 | jo <- rank.test(temp.vecm) 461 | co[j:(j+1),1] <- jo[["res_df"]]$trace; co[j:(j+1),2] <- jo[["res_df"]]$trace_pval 462 | co[j,3:4] <- attributes(get(series[i]))$dimnames[[2]]; co[j+1,3:4] <- attributes(get(series[i]))$dimnames[[2]] 463 | co[j,5] <- 1; co[(j+1),5] <- 1 464 | j = j + 2 465 | } 466 | rm(j) 467 | for (i in 1:nrow(co)){ 468 | if (as.numeric(co[i,2])<0.05){ 469 | co[i,6] <- "Cointegrated" 470 | } 471 | else { 472 | co[i,6] <- "Not Cointegrated" 473 | } 474 | } 475 | write.table(co,"of/y-coint-1lag.csv",sep=",",row.names=T,col.names=NA) 476 | 477 | j <- 1 478 | for (i in 1:length(series)){ 479 | temp.vecm <- suppressWarnings( VECM(get(series[i]), 1, LRinclude="const", estim="ML") ) 480 | jo <- rank.test(temp.vecm) 481 | co[j:(j+1),1] <- jo[["res_df"]]$trace; co[j:(j+1),2] <- jo[["res_df"]]$trace_pval 482 | co[j,3:4] <- attributes(get(series[i]))$dimnames[[2]]; co[j+1,3:4] <- attributes(get(series[i]))$dimnames[[2]] 483 | co[j,5] <- 2; co[(j+1),5] <- 2 484 | j = j + 2 485 | } 486 | rm(j) 487 | for (i in 1:nrow(co)){ 488 | if (as.numeric(co[i,2])<0.05){ 489 | co[i,6] <- "Cointegrated" 490 | } 491 | else { 492 | co[i,6] <- "Not Cointegrated" 493 | } 494 | } 495 | write.table(co,"of/y-coint-2lags.csv",sep=",",row.names=T,col.names=NA) 496 | 497 | 498 | #-------------------------------------------------------------------------------------------# 499 | # Quarterly Version 500 | 501 | # Declaring variables as multivariate time series. 502 | pol.disp <- ts(cbind(dq$polar,dq$ldisp)[89:nrow(dq),], start=c(1931, 1),frequency=4, names=c("Polarity", "Disputes")) 503 | pol.mis <- ts(cbind(dq$polar,dq$misery)[186:nrow(dq),], start=c(1955, 2),frequency=4, names=c("Polarity", "Misery")) 504 | pol.gdpg <- ts(cbind(dq$polar,dq$gdpg)[186:nrow(dq),], start=c(1955, 2),frequency=4, names=c("Polarity", "GDPg")) 505 | pol.unemp <- ts(cbind(dq$polar,dq$unemp)[185:nrow(dq),], start=c(1955, 1),frequency=4, names=c("Polarity", "Unemp")) 506 | # Government only. 507 | polg.disp <- ts(cbind(dq$polarg,dq$ldisp)[147:nrow(dq),], start=c(1945, 3),frequency=4, names=c("Polarity-gov", "Disputes")) 508 | # Opposition only. 509 | polo.disp <- ts(cbind(dq$polaro,dq$ldisp)[147:nrow(dq),], start=c(1945, 3),frequency=4, names=c("Polarity-opp", "Disputes")) 510 | 511 | series <- c("pol.disp","pol.mis","pol.gdpg","pol.unemp", 512 | "polg.disp","polo.disp") 513 | 514 | # Automatic lag length selection (Bayesian criterion) 515 | bc <- matrix(nrow=length(series),ncol=1) 516 | for (i in 1:nrow(bc)){ 517 | bc[i] <- VARselect(get(series[i]),lag=12,type="const")$selection[3] 518 | } 519 | 520 | # Automatic lag length selection (Hannan-Quinn criterion) 521 | hq <- matrix(nrow=length(series),ncol=1) 522 | for (i in 1:nrow(hq)){ 523 | hq[i] <- VARselect(get(series[i]),lag=12,type="const")$selection[2] 524 | } 525 | 526 | gc <- matrix(ncol=4,nrow=2*length(series)) 527 | colnames(gc) <- c("Chi-2","Lags","p-value","Outcome") 528 | rownames(gc) <- c("Disputes -> Polarity", "Polarity -> Disputes", 529 | "Misery -> Polarity", "Polarity -> Misery", 530 | "GDPg -> Polarity", "Polarity -> GDPg", 531 | "Unemployment -> Polarity", "Polarity -> Unemployment", 532 | "Disputes -> Polarity-Gov", "PolarityGov -> Disputes", 533 | "Disputes -> Polarity-Opp", "Polarity-Opp -> Disputes") 534 | 535 | j <- 1 536 | for (i in 1:length(series)){ 537 | tempvar <- VAR(get(series[i]),p=bc[i]+1,type="const") 538 | gc[j,1:3] <- wald.test(b=coef(tempvar$varresult[[1]]), 539 | Sigma=vcov(tempvar$varresult[[1]]), 540 | Terms=seq(from=2,to=bc[i]*2,by=2), df=bc[i])$result[[1]] 541 | gc[j+1,1:3] <- wald.test(b=coef(tempvar$varresult[[2]]), 542 | Sigma=vcov(tempvar$varresult[[2]]), 543 | Terms=seq(from=1,to=bc[i]*2-1,by=2), df=bc[i])$result[[1]] 544 | j = j + 2 545 | } 546 | rm(j) 547 | for (i in 1:nrow(gc)){ 548 | if (as.numeric(gc[i,3])<0.05){ 549 | gc[i,4] <- "*" 550 | } 551 | else { 552 | gc[i,4] <- "" 553 | } 554 | } 555 | write.table(gc,"of/q-granger-bic.csv",sep=",",row.names=T,col.names=NA) 556 | j <- 1 557 | for (i in 1:length(series)){ 558 | tempvar <- VAR(get(series[i]),p=hq[i]+1,type="const") 559 | gc[j,1:3] <- wald.test(b=coef(tempvar$varresult[[1]]), 560 | Sigma=vcov(tempvar$varresult[[1]]), 561 | Terms=seq(from=2,to=hq[i]*2,by=2), df=hq[i])$result[[1]] 562 | gc[j+1,1:3] <- wald.test(b=coef(tempvar$varresult[[2]]), 563 | Sigma=vcov(tempvar$varresult[[2]]), 564 | Terms=seq(from=1,to=hq[i]*2-1,by=2), df=hq[i])$result[[1]] 565 | j = j + 2 566 | } 567 | rm(j) 568 | for (i in 1:nrow(gc)){ 569 | if (as.numeric(gc[i,3])<0.05){ 570 | gc[i,4] <- "*" 571 | } 572 | else { 573 | gc[i,4] <- "" 574 | } 575 | } 576 | write.table(gc,"of/q-granger-hq.csv",sep=",",row.names=T,col.names=NA) 577 | 578 | # Cointegration tests (Johansen rank tests). 579 | # Restricting to pairs of series that are both I(1), at least according to the ADF tests: 580 | # Government only. 581 | polg.mis <- ts(cbind(dq$polarg,dq$misery)[186:nrow(dq),], start=c(1955, 2),frequency=4, names=c("Polarity-gov", "Misery")) 582 | polg.unemp <- ts(cbind(dq$polarg,dq$unemp)[185:nrow(dq),], start=c(1955, 1),frequency=4, names=c("Polarity-gov", "Unemp")) 583 | # Opposition only. 584 | polo.mis <- ts(cbind(dq$polaro,dq$misery)[186:nrow(dq),], start=c(1955, 2),frequency=4, names=c("Polarity-opp", "Misery")) 585 | polo.unemp <- ts(cbind(dq$polaro,dq$unemp)[185:nrow(dq),], start=c(1955, 1),frequency=4, names=c("Polarity-opp", "Unemp")) 586 | 587 | series <- c("pol.disp","pol.mis","pol.unemp", 588 | "polg.disp","polg.mis","polg.unemp", 589 | "polo.disp","polo.mis","polo.unemp") 590 | 591 | 592 | co <- matrix(nrow=18,ncol=6) 593 | colnames(co) <- c("trace","p-val","var1","var2","lags","outcome") 594 | rownames(co) <- c("r = 0","r = 1", "r = 0","r = 1", "r = 0","r = 1", 595 | "r = 0","r = 1", "r = 0","r = 1", "r = 0","r = 1", 596 | "r = 0","r = 1", "r = 0","r = 1", "r = 0","r = 1") 597 | 598 | j <- 1 599 | for (i in 1:length(series)){ 600 | temp.vecm <- suppressWarnings( VECM(get(series[i]), 4, include="const", estim="ML") ) 601 | jo <- rank.test(temp.vecm) 602 | co[j:(j+1),1] <- jo[["res_df"]]$trace; co[j:(j+1),2] <- jo[["res_df"]]$trace_pval 603 | co[j,3:4] <- attributes(get(series[i]))$dimnames[[2]]; co[j+1,3:4] <- attributes(get(series[i]))$dimnames[[2]] 604 | co[j,5] <- 5; co[(j+1),5] <- 5 605 | j = j + 2 606 | } 607 | rm(j) 608 | for (i in 1:nrow(co)){ 609 | if (as.numeric(co[i,2])<0.05){ 610 | co[i,6] <- "Cointegrated" 611 | } 612 | else { 613 | co[i,6] <- "" 614 | } 615 | } 616 | write.table(co,"of/q-coint-5lags.csv",sep=",",row.names=T,col.names=NA) 617 | 618 | j <- 1 619 | for (i in 1:length(series)){ 620 | temp.vecm <- suppressWarnings( VECM(get(series[i]), 5, include="const", estim="ML") ) 621 | jo <- rank.test(temp.vecm) 622 | co[j:(j+1),1] <- jo[["res_df"]]$trace; co[j:(j+1),2] <- jo[["res_df"]]$trace_pval 623 | co[j,3:4] <- attributes(get(series[i]))$dimnames[[2]]; co[j+1,3:4] <- attributes(get(series[i]))$dimnames[[2]] 624 | co[j,5] <- 6; co[(j+1),5] <- 6 625 | j = j + 2 626 | } 627 | rm(j) 628 | for (i in 1:nrow(co)){ 629 | if (as.numeric(co[i,2])<0.05){ 630 | co[i,6] <- "Cointegrated" 631 | } 632 | else { 633 | co[i,6] <- "" 634 | } 635 | } 636 | write.table(co,"of/q-coint-6lags.csv",sep=",",row.names=T,col.names=NA) 637 | 638 | #-------------------------------------------------------------------------------------------# 639 | # Tables S11, S16; Figures 3, S5, and main text: Vector Error Correction Models (VECMs) 640 | #-------------------------------------------------------------------------------------------# 641 | # VECM via tsDyn package 642 | 643 | #-------------------------------------------------------------------------------------------# 644 | # Yearly Version 645 | pol.disp <- ts(cbind(dy$polar,dy$ldisp), start=c(1909),frequency=1, names=c("Polarity","Disputes")) 646 | polg.disp <- ts(cbind(dy$polarg,dy$ldisp)[38:nrow(dy),], start=c(1946),frequency=1, names=c("PolarityGov","Disputes")) 647 | polo.disp <- ts(cbind(dy$polaro,dy$ldisp)[38:nrow(dy),], start=c(1946),frequency=1, names=c("PolarityOpp","Disputes")) 648 | 649 | pol.disp.exg <- ts(cbind(dy$polar,dy$ldisp)[38:nrow(dy),], start=c(1946),frequency=1, names=c("Polarity","Disputes")) 650 | exogvar <- ts(cbind(dy$conservative,dy$election)[38:nrow(dy),], start=c(1946),frequency=1, names=c("Party", "Elections")) 651 | 652 | ll0 <- VARselect(pol.disp,lag=12,type="const")$selection[3] 653 | ll1 <- VARselect(polg.disp,lag=12,type="const")$selection[3] 654 | ll2 <- VARselect(polo.disp,lag=12,type="const")$selection[3] 655 | 656 | # Note: VECM uses lags in difference as input, as opposed to lags in levels obtained from VARselect. 657 | # For the purposes of this study, we use 1 lag in first differences. 658 | vecm.00 <- VECM(pol.disp, ll0, include = "const", r=1, estim="ML") 659 | vecm.01 <- VECM(pol.disp.exg, ll0, include = "const", r=1, estim="ML", exogen=exogvar) 660 | vecm.1 <- VECM(polg.disp, ll1, include = "const", r=1, estim="ML") 661 | vecm.2 <- VECM(polo.disp, ll2, include = "const", r=1, estim="ML") 662 | summary(vecm.00) 663 | summary(vecm.01) 664 | summary(vecm.1) 665 | summary(vecm.2) 666 | 667 | ### Reorder variables from exogenous to endogenous in the Cholesky decomposition. 668 | pol.disp <- ts(cbind(dy$ldisp,dy$polar), start=c(1909),frequency=1, names=c("Disputes","Polarity")) 669 | pol.disp.exg <- ts(cbind(dy$ldisp,dy$polar)[38:nrow(dy),], start=c(1946),frequency=1, names=c("Disputes","Polarity")) 670 | polg.disp <- ts(cbind(dy$ldisp,dy$polarg)[38:nrow(dy),], start=c(1946),frequency=1, names=c("Disputes","PolarityGov")) 671 | polo.disp <- ts(cbind(dy$ldisp,dy$polaro)[38:nrow(dy),], start=c(1946),frequency=1, names=c("Disputes","PolarityOpp")) 672 | vecm.00 <- VECM(pol.disp, ll0, include = "const", r=1, estim="ML") 673 | vecm.01 <- VECM(pol.disp.exg, ll0, include = "const", r=1, estim="ML", exogen=exogvar) 674 | vecm.1 <- VECM(polg.disp, ll1, include = "const", r=1, estim="ML") 675 | vecm.2 <- VECM(polo.disp, ll2, include = "const", r=1, estim="ML") 676 | 677 | # Impulse response functions. 678 | ir <- irf(vecm.00, boot = T, impulse = "Disputes", response="Polarity", ortho=T, 679 | cumulative = FALSE, n.ahead = 40, seed=42) 680 | irdata <- data.frame(time=1:lengths(ir$irf)[[1]],Polarity=ir$irf[[1]], 681 | LB=as.vector(ir$Lower[[1]]),UB=as.vector(ir$Upper[[1]])) 682 | ggplot(irdata, aes(x = time)) + theme_bw() + 683 | geom_ribbon(aes(ymin = LB, ymax = UB), fill = "#BFC9D1") + 684 | geom_line(aes(y = Polarity), size = 2, colour="#002A48") + 685 | geom_line(aes(y = LB), linetype="dashed", size = 0.5, colour="#002A48") + 686 | geom_line(aes(y = UB), linetype="dashed", size = 0.5, colour="#002A48") + 687 | geom_hline(aes(yintercept=0), size = .5, colour="#3f3f3f") + 688 | labs(x="Time",y="Response") + 689 | theme(axis.text=element_text(size=10), 690 | axis.title=element_text(size=12), 691 | axis.title.y=element_text(margin=margin(0,10,0,0)), 692 | axis.title.x=element_text(margin=margin(10,0,0,0))) 693 | 694 | ir <- irf(vecm.01, boot = T, impulse = "Disputes", response="Polarity", ortho=T, 695 | cumulative = FALSE, n.ahead = 40, seed=42) 696 | irdata <- data.frame(time=1:lengths(ir$irf)[[1]],Polarity=ir$irf[[1]], 697 | LB=as.vector(ir$Lower[[1]]),UB=as.vector(ir$Upper[[1]])) 698 | ggplot(irdata, aes(x = time)) + theme_bw() + 699 | geom_ribbon(aes(ymin = LB, ymax = UB), fill = "#BFC9D1") + 700 | geom_line(aes(y = Polarity), size = 2, colour="#002A48") + 701 | geom_line(aes(y = LB), linetype="dashed", size = .5, colour="#002A48") + 702 | geom_line(aes(y = UB), linetype="dashed", size = .5, colour="#002A48") + 703 | geom_hline(aes(yintercept=0), size = .5, colour="#3f3f3f") + 704 | labs(x="Time",y="Response") + 705 | theme(axis.text=element_text(size=10), 706 | axis.title=element_text(size=12), 707 | axis.title.y=element_text(margin=margin(0,10,0,0)), 708 | axis.title.x=element_text(margin=margin(10,0,0,0))) 709 | 710 | # Impulse response functions (Government Only, S5 A) 711 | ir <- irf(vecm.1, boot = T, impulse = "Disputes", response="PolarityGov", 712 | ortho=T,cumulative = FALSE, n.ahead = 40, seed=42) 713 | irdata <- data.frame(time=1:lengths(ir$irf)[[1]],PolarityGov=ir$irf[[1]], 714 | LB=as.vector(ir$Lower[[1]]),UB=as.vector(ir$Upper[[1]])) 715 | irfp1 <- ggplot(irdata, aes(x = time)) + theme_bw() + 716 | geom_ribbon(aes(ymin = LB, ymax = UB), fill = "#BFC9D1") + 717 | geom_line(aes(y = PolarityGov), size = 2, colour="#002A48") + 718 | geom_line(aes(y = LB), linetype="dashed", size = .5, colour="#002A48") + 719 | geom_line(aes(y = UB), linetype="dashed", size = .5, colour="#002A48") + 720 | geom_hline(aes(yintercept=0), size = .5, colour="#3f3f3f") + 721 | labs(x=expression(atop("Time", paste("A. Polarity of Government (Yearly)"))),y="Response") + 722 | theme(axis.text=element_text(size=10), 723 | axis.title=element_text(size=10), 724 | axis.title.y=element_text(margin=margin(0,10,0,0)), 725 | axis.title.x=element_text(margin=margin(10,0,0,0))) + 726 | scale_y_continuous(lim=c(-0.7,0.1)) 727 | 728 | # Impulse response functions (Opposition Only, S5 B)) 729 | ir <- irf(vecm.2, boot = T, impulse = "Disputes", response="PolarityOpp", ortho=T, 730 | cumulative = FALSE, n.ahead = 40, seed=42) 731 | irdata <- data.frame(time=1:lengths(ir$irf)[[1]],PolarityOpp=ir$irf[[1]], 732 | LB=as.vector(ir$Lower[[1]]),UB=as.vector(ir$Upper[[1]])) 733 | irfp2 <- ggplot(irdata, aes(x = time)) + theme_bw() + 734 | geom_ribbon(aes(ymin = LB, ymax = UB), fill = "#BFC9D1") + 735 | geom_line(aes(y = PolarityOpp), size = 2, colour="#002A48") + 736 | geom_line(aes(y = LB), linetype="dashed", size = .5, colour="#002A48") + 737 | geom_line(aes(y = UB), linetype="dashed", size = .5, colour="#002A48") + 738 | geom_hline(aes(yintercept=0), size = .5, colour="#3f3f3f") + 739 | labs(x=expression(atop("Time", paste("B. Polarity of Opposition (Yearly)"))),y="Response") + 740 | theme(axis.text=element_text(size=10), 741 | axis.title=element_text(size=10), 742 | axis.title.y=element_text(margin=margin(0,10,0,0)), 743 | axis.title.x=element_text(margin=margin(10,0,0,0))) + 744 | scale_y_continuous(lim=c(-0.7,0.1)) 745 | 746 | #-------------------------------------------------------------------------------------------# 747 | # Quarterly Version 748 | 749 | pol.disp <- ts(cbind(dq$polar,dq$ldisp)[89:nrow(dq),], start=c(1931, 1),frequency=4, names=c("Polarity","Disputes")) 750 | polg.disp <- ts(cbind(dq$polarg,dq$ldisp)[147:nrow(dq),], start=c(1945, 3),frequency=4, names=c("PolarityGov","Disputes")) 751 | polo.disp <- ts(cbind(dq$polaro,dq$ldisp)[147:nrow(dq),], start=c(1945, 3),frequency=4, names=c("PolarityOpp","Disputes")) 752 | 753 | ll0 <- VARselect(pol.disp,lag=10,type="const")$selection[3] 754 | ll1 <- VARselect(polg.disp,lag=10,type="const")$selection[3] 755 | ll2 <- VARselect(polo.disp,lag=10,type="const")$selection[3] 756 | 757 | # We augment number of lags at 4 (5 in levels) to cover full cycle of periodicity. 758 | vecm.0 <- VECM(pol.disp, 4, include = "const", r=1, estim="ML") 759 | vecm.1 <- VECM(polg.disp, 4, include = "const", r=1, estim="ML") 760 | vecm.2 <- VECM(polo.disp, 4, include = "const", r=1, estim="ML") 761 | summary(vecm.0) 762 | summary(vecm.1) 763 | summary(vecm.2) 764 | 765 | ### Reorder variables from exogenous to endogenous in the Cholesky decomposition. 766 | pol.disp <- ts(cbind(dq$ldisp,dq$polar)[89:nrow(dq),], start=c(1931, 1),frequency=4, names=c("Disputes","Disputes")) 767 | polg.disp <- ts(cbind(dq$ldisp,dq$polarg)[147:nrow(dq),], start=c(1945, 3),frequency=4, names=c("Disputes","PolarityGov")) 768 | polo.disp <- ts(cbind(dq$ldisp,dq$polaro)[147:nrow(dq),], start=c(1945, 3),frequency=4, names=c("Disputes","PolarityOpp")) 769 | vecm.0 <- VECM(pol.disp, 4, include = "const", r=1, estim="ML") 770 | vecm.1 <- VECM(polg.disp, 4, include = "const", r=1, estim="ML") 771 | vecm.2 <- VECM(polo.disp, 4, include = "const", r=1, estim="ML") 772 | 773 | # Computing impulse responses. 774 | # Impulse response functions (Government Only, Fig S5 C and D) 775 | ir <- irf(vecm.1, boot = T, impulse = "Disputes", response="PolarityGov", 776 | ortho=T,cumulative = FALSE, n.ahead = 40, seed=42) 777 | irdata <- data.frame(time=1:lengths(ir$irf)[[1]],PolarityGov=ir$irf[[1]], 778 | LB=as.vector(ir$Lower[[1]]),UB=as.vector(ir$Upper[[1]])) 779 | irfp3 <- ggplot(irdata, aes(x = time)) + theme_bw() + 780 | geom_ribbon(aes(ymin = LB, ymax = UB), fill = "#BFC9D1") + 781 | geom_line(aes(y = PolarityGov), size = 2, colour="#002A48") + 782 | geom_line(aes(y = LB), linetype="dashed", size = .5, colour="#002A48") + 783 | geom_line(aes(y = UB), linetype="dashed", size = .5, colour="#002A48") + 784 | geom_hline(aes(yintercept=0), size = .5, colour="#3f3f3f") + 785 | labs(x=expression(atop("Time", paste("C. Polarity of Government (Quarterly)"))),y="Response") + 786 | theme(axis.text=element_text(size=10), 787 | axis.title=element_text(size=10), 788 | axis.title.y=element_text(margin=margin(0,10,0,0)), 789 | axis.title.x=element_text(margin=margin(10,0,0,0))) + 790 | scale_y_continuous(lim=c(-0.7,0.1)) 791 | 792 | # Impulse response functions (Opposition Only, SI6(d)) 793 | ir <- irf(vecm.2, boot = T, impulse = "Disputes", response="PolarityOpp", 794 | ortho=T,cumulative = FALSE, n.ahead = 40, seed=42) 795 | irdata <- data.frame(time=1:lengths(ir$irf)[[1]],PolarityOpp=ir$irf[[1]], 796 | LB=as.vector(ir$Lower[[1]]),UB=as.vector(ir$Upper[[1]])) 797 | irfp4 <- ggplot(irdata, aes(x = time)) + theme_bw() + 798 | geom_ribbon(aes(ymin = LB, ymax = UB), fill = "#BFC9D1") + 799 | geom_line(aes(y = LB), linetype="dashed", size = .5, colour="#002A48") + 800 | geom_line(aes(y = UB), linetype="dashed", size = .5, colour="#002A48") + 801 | geom_hline(aes(yintercept=0), size = .5, colour="#3f3f3f") + 802 | geom_line(aes(y = PolarityOpp), size = 2, colour="#002A48") + 803 | labs(x=expression(atop("Time", paste("D. Polarity of Opposition (Quarterly)"))),y="Response") + 804 | theme(axis.text=element_text(size=10), 805 | axis.title=element_text(size=10), 806 | axis.title.y=element_text(margin=margin(0,10,0,0)), 807 | axis.title.x=element_text(margin=margin(10,0,0,0))) + 808 | scale_y_continuous(lim=c(-0.7,0.1)) 809 | 810 | grid.arrange(irfp1, irfp2, irfp3, irfp4, ncol = 2) 811 | 812 | 813 | #-------------------------------------------------------------------------------------------# 814 | # Dynamic OLS (DOLS) Models - Not in Text 815 | #-------------------------------------------------------------------------------------------# 816 | 817 | #-------------------------------------------------------------------------------------------# 818 | # Custom lag function (this version based on StackOverflow snippet) 819 | 820 | l<-function(x,shift_by){ 821 | if (length(shift_by)>1) 822 | return(sapply(shift_by,l, x=x)) 823 | out<-NULL 824 | abs_shift_by=abs(shift_by) 825 | if (shift_by > 0 ) 826 | out<-c(tail(x,-abs_shift_by),rep(NA,abs_shift_by)) 827 | else if (shift_by < 0 ) 828 | out<-c(rep(NA,abs_shift_by), head(x,-abs_shift_by)) 829 | else 830 | out<-x 831 | out 832 | } 833 | 834 | # First-Differencing function 835 | d1 <- function(x) { 836 | x-l(x,-1) 837 | } 838 | #-------------------------------------------------------------------------------------------# 839 | 840 | 841 | #-------------------------------------------------------------------------------------------# 842 | # Yearly Version 843 | 844 | dols <- matrix(nrow=12,ncol=6) 845 | colnames(dols) <- c("Delta Leads/Lags = 1", "Delta Leads/Lags = 2", 846 | "Delta Leads/Lags = 1", "Delta Leads/Lags = 2", 847 | "Delta Leads/Lags = 1", "Delta Leads/Lags = 2") 848 | rownames(dols) <- c("Labor Disputes","","Misery Index","","Unemployment","","Intercept","", 849 | "","N","Adj-R2","sigma") 850 | 851 | # Main Polarity Measure and Labour Disputes 852 | 853 | dols.1 <- lm(polar ~ ldisp + l(d1(ldisp),-1:1), data=dy) 854 | dols.2 <- lm(polar ~ ldisp + l(d1(ldisp),-2:2), data=dy) 855 | coeftest(dols.1, vcov=vcovHAC) 856 | coeftest(dols.2, vcov=vcovHAC) 857 | dols[1:2,1] = t(coeftest(dols.1, vcov=vcovHAC)[2,1:2]) 858 | dols[7:8,1] = t(coeftest(dols.1, vcov=vcovHAC)[1,1:2]) 859 | dols[1:2,2] = t(coeftest(dols.2, vcov=vcovHAC)[2,1:2]) 860 | dols[7:8,2] = t(coeftest(dols.2, vcov=vcovHAC)[1,1:2]) 861 | dols[10,1] <- nobs(dols.1) 862 | dols[11,1] <- summary(dols.1)$adj.r.squared 863 | dols[12,1] <- summary(dols.1)$sigma 864 | dols[10,2] <- nobs(dols.2) 865 | dols[11,2] <- summary(dols.2)$adj.r.squared 866 | dols[12,2] <- summary(dols.2)$sigma 867 | 868 | # Main Polarity Measure and Misery Index 869 | 870 | dols.3 <- lm(polar ~ misery + l(d1(misery),-1:1), data=dy) 871 | dols.4 <- lm(polar ~ misery + l(d1(misery),-2:2), data=dy) 872 | coeftest(dols.3, vcov=vcovHAC) 873 | coeftest(dols.4, vcov=vcovHAC) 874 | dols[3:4,3] = t(coeftest(dols.3, vcov=vcovHAC)[2,1:2]) 875 | dols[7:8,3] = t(coeftest(dols.3, vcov=vcovHAC)[1,1:2]) 876 | dols[3:4,4] = t(coeftest(dols.4, vcov=vcovHAC)[2,1:2]) 877 | dols[7:8,4] = t(coeftest(dols.4, vcov=vcovHAC)[1,1:2]) 878 | dols[10,3] <- nobs(dols.3) 879 | dols[11,3] <- summary(dols.3)$adj.r.squared 880 | dols[12,3] <- summary(dols.3)$sigma 881 | dols[10,4] <- nobs(dols.4) 882 | dols[11,4] <- summary(dols.4)$adj.r.squared 883 | dols[12,4] <- summary(dols.4)$sigma 884 | 885 | # Main Polarity Measure and Unemployment 886 | 887 | dols.5 <- lm(polar ~ unemp + l(d1(unemp),-1:1), data=dy) 888 | dols.6 <- lm(polar ~ unemp + l(d1(unemp),-2:2), data=dy) 889 | summary(dols.5) 890 | summary(dols.6) 891 | dols[5:6,5] = t(coeftest(dols.5, vcov=vcovHAC)[2,1:2]) 892 | dols[7:8,5] = t(coeftest(dols.5, vcov=vcovHAC)[1,1:2]) 893 | dols[5:6,6] = t(coeftest(dols.6, vcov=vcovHAC)[2,1:2]) 894 | dols[7:8,6] = t(coeftest(dols.6, vcov=vcovHAC)[1,1:2]) 895 | dols[10,5] <- nobs(dols.5) 896 | dols[11,5] <- summary(dols.5)$adj.r.squared 897 | dols[12,5] <- summary(dols.5)$sigma 898 | dols[10,6] <- nobs(dols.6) 899 | dols[11,6] <- summary(dols.6)$adj.r.squared 900 | dols[12,6] <- summary(dols.6)$sigma 901 | 902 | write.table(dols,"of/y-dols.csv",sep=",",row.names=T,col.names=NA,na="") 903 | 904 | #-------------------------------------------------------------------------------------------# 905 | # Quarterly Version 906 | 907 | dols <- matrix(nrow=12,ncol=6) 908 | colnames(dols) <- c("Delta Leads/Lags = 1", "Delta Leads/Lags = 4", 909 | "Delta Leads/Lags = 1", "Delta Leads/Lags = 4", 910 | "Delta Leads/Lags = 1", "Delta Leads/Lags = 4") 911 | rownames(dols) <- c("Labor Disputes","","Misery Index","","Unemployment","","Intercept","", 912 | "","N","Adj-R2","sigma") 913 | 914 | # Main Polarity Measure and Labour Disputes 915 | 916 | dols.1 <- lm(polar ~ ldisp + l(d1(ldisp),-1:1), data=dq) 917 | dols.2 <- lm(polar ~ ldisp + l(d1(ldisp),-4:4), data=dq) 918 | coeftest(dols.1, vcov=vcovHAC) 919 | coeftest(dols.2, vcov=vcovHAC) 920 | dols[1:2,1] = t(coeftest(dols.1, vcov=vcovHAC)[2,1:2]) 921 | dols[7:8,1] = t(coeftest(dols.1, vcov=vcovHAC)[1,1:2]) 922 | dols[1:2,2] = t(coeftest(dols.2, vcov=vcovHAC)[2,1:2]) 923 | dols[7:8,2] = t(coeftest(dols.2, vcov=vcovHAC)[1,1:2]) 924 | dols[10,1] <- nobs(dols.1) 925 | dols[11,1] <- summary(dols.1)$adj.r.squared 926 | dols[12,1] <- summary(dols.1)$sigma 927 | dols[10,2] <- nobs(dols.2) 928 | dols[11,2] <- summary(dols.2)$adj.r.squared 929 | dols[12,2] <- summary(dols.2)$sigma 930 | 931 | # Main Polarity Measure and Misery Index 932 | 933 | dols.3 <- lm(polar ~ misery + l(d1(misery),-1:1), data=dq) 934 | dols.4 <- lm(polar ~ misery + l(d1(misery),-4:4), data=dq) 935 | coeftest(dols.3, vcov=vcovHAC) 936 | coeftest(dols.4, vcov=vcovHAC) 937 | dols[3:4,3] = t(coeftest(dols.3, vcov=vcovHAC)[2,1:2]) 938 | dols[7:8,3] = t(coeftest(dols.3, vcov=vcovHAC)[1,1:2]) 939 | dols[3:4,4] = t(coeftest(dols.4, vcov=vcovHAC)[2,1:2]) 940 | dols[7:8,4] = t(coeftest(dols.4, vcov=vcovHAC)[1,1:2]) 941 | dols[10,3] <- nobs(dols.3) 942 | dols[11,3] <- summary(dols.3)$adj.r.squared 943 | dols[12,3] <- summary(dols.3)$sigma 944 | dols[10,4] <- nobs(dols.4) 945 | dols[11,4] <- summary(dols.4)$adj.r.squared 946 | dols[12,4] <- summary(dols.4)$sigma 947 | 948 | # Main Polarity Measure and Unemployment 949 | 950 | dols.5 <- lm(polar ~ unemp + l(d1(unemp),-1:1), data=dq) 951 | dols.6 <- lm(polar ~ unemp + l(d1(unemp),-4:4), data=dq) 952 | summary(dols.5) 953 | summary(dols.6) 954 | dols[5:6,5] = t(coeftest(dols.5, vcov=vcovHAC)[2,1:2]) 955 | dols[7:8,5] = t(coeftest(dols.5, vcov=vcovHAC)[1,1:2]) 956 | dols[5:6,6] = t(coeftest(dols.6, vcov=vcovHAC)[2,1:2]) 957 | dols[7:8,6] = t(coeftest(dols.6, vcov=vcovHAC)[1,1:2]) 958 | dols[10,5] <- nobs(dols.5) 959 | dols[11,5] <- summary(dols.5)$adj.r.squared 960 | dols[12,5] <- summary(dols.5)$sigma 961 | dols[10,6] <- nobs(dols.6) 962 | dols[11,6] <- summary(dols.6)$adj.r.squared 963 | dols[12,6] <- summary(dols.6)$sigma 964 | 965 | write.table(dols,"of/q-dols.csv",sep=",",row.names=T,col.names=NA,na="") 966 | 967 | #-------------------------------------------------------------------------------------------# 968 | # Single Equation Error Correction Models (ECM) - Not in Text 969 | #-------------------------------------------------------------------------------------------# 970 | 971 | #-------------------------------------------------------------------------------------------# 972 | # Yearly Version 973 | 974 | # Main Polarity Measure and Labour Disputes 975 | 976 | ecms <- matrix(nrow=22,ncol=3) 977 | colnames(ecms) <- c("Labor Disputes Model", "Misery Index Model", 978 | "Unemployment Model") 979 | rownames(ecms) <- c("Lag-Polarity","", 980 | "Diff-Labor Disputes","","Lag-Labor Disputes","", 981 | "Diff-Misery Index","","Lag-Misery Index","", 982 | "Diff-Unemployment","","Lag-Unemployment","", 983 | "Intercept","","","Long-Run Effect","", 984 | "N","Adj-R2","sigma") 985 | 986 | 987 | ecm.1 <- lm(d1(polar) ~ d1(ldisp) + l(polar,-1) + l(ldisp,-1), data=dy) 988 | coeftest(ecm.1, vcov=vcovHAC) 989 | # Long-run effect: 990 | lr <- -coef(ecm.1)[[4]]/coef(ecm.1)[[3]] 991 | ecms[1:2,1] <- t(coeftest(ecm.1, vcov=vcovHAC)[3,1:2]) 992 | ecms[3:4,1] <- t(coeftest(ecm.1, vcov=vcovHAC)[2,1:2]) 993 | ecms[5:6,1] <- t(coeftest(ecm.1, vcov=vcovHAC)[4,1:2]) 994 | ecms[15:16,1] <- t(coeftest(ecm.1, vcov=vcovHAC)[1,1:2]) 995 | ecms[18,1] <- lr 996 | ecms[20,1] <- nobs(ecm.1) 997 | ecms[21,1] <- summary(ecm.1)$adj.r.squared 998 | ecms[22,1] <- summary(ecm.1)$sigma 999 | 1000 | # Main Polarity Measure and Misery Index 1001 | 1002 | ecm.2 <- lm(d1(polar) ~ d1(misery) + l(polar,-1) + l(misery,-1), data=dy) 1003 | coeftest(ecm.2, vcov=vcovHAC) 1004 | # Long-run effect: 1005 | lr <- -coef(ecm.2)[[4]]/coef(ecm.2)[[3]] 1006 | ecms[1:2,2] = t(coeftest(ecm.2, vcov=vcovHAC)[3,1:2]) 1007 | ecms[7:8,2] = t(coeftest(ecm.2, vcov=vcovHAC)[2,1:2]) 1008 | ecms[9:10,2] = t(coeftest(ecm.2, vcov=vcovHAC)[4,1:2]) 1009 | ecms[15:16,2] = t(coeftest(ecm.2, vcov=vcovHAC)[1,1:2]) 1010 | ecms[18,2] = lr 1011 | ecms[20,2] <- nobs(ecm.2) 1012 | ecms[21,2] <- summary(ecm.2)$adj.r.squared 1013 | ecms[22,2] <- summary(ecm.2)$sigma 1014 | 1015 | # Main Polarity Measure and Unemployment 1016 | 1017 | ecm.3 <- lm(d1(polar) ~ d1(unemp) + l(polar,-1) + l(unemp,-1), data=dy) 1018 | coeftest(ecm.3, vcov=vcovHAC) 1019 | # Long-run effect: 1020 | lr <- -coef(ecm.3)[[4]]/coef(ecm.3)[[3]] 1021 | ecms[1:2,3] = t(coeftest(ecm.3, vcov=vcovHAC)[3,1:2]) 1022 | ecms[11:12,3] = t(coeftest(ecm.3, vcov=vcovHAC)[2,1:2]) 1023 | ecms[13:14,3] = t(coeftest(ecm.3, vcov=vcovHAC)[4,1:2]) 1024 | ecms[15:16,3] = t(coeftest(ecm.3, vcov=vcovHAC)[1,1:2]) 1025 | ecms[18,3] = lr 1026 | ecms[20,3] <- nobs(ecm.3) 1027 | ecms[21,3] <- summary(ecm.3)$adj.r.squared 1028 | ecms[22,3] <- summary(ecm.3)$sigma 1029 | 1030 | write.table(ecms,"of/y-ecms.csv",sep=",",row.names=T,col.names=NA,na="") 1031 | 1032 | #-------------------------------------------------------------------------------------------# 1033 | # Quarterly Version 1034 | 1035 | # Main Polarity Measure and Labour Disputes 1036 | 1037 | ecms <- matrix(nrow=22,ncol=3) 1038 | colnames(ecms) <- c("Labor Disputes Model", "Misery Index Model", 1039 | "Unemployment Model") 1040 | rownames(ecms) <- c("Lag-Polarity","", 1041 | "Diff-Labor Disputes","","Lag-Labor Disputes","", 1042 | "Diff-Misery Index","","Lag-Misery Index","", 1043 | "Diff-Unemployment","","Lag-Unemployment","", 1044 | "Intercept","","","Long-Run Effect","", 1045 | "N","Adj-R2","sigma") 1046 | 1047 | 1048 | ecm.1 <- lm(d1(polar) ~ d1(ldisp) + l(polar,-1) + l(ldisp,-1), data=dq) 1049 | coeftest(ecm.1, vcov=vcovHAC) 1050 | # Long-run effect: 1051 | lr <- -coef(ecm.1)[[4]]/coef(ecm.1)[[3]] 1052 | ecms[1:2,1] = t(coeftest(ecm.1, vcov=vcovHAC)[3,1:2]) 1053 | ecms[3:4,1] = t(coeftest(ecm.1, vcov=vcovHAC)[2,1:2]) 1054 | ecms[5:6,1] = t(coeftest(ecm.1, vcov=vcovHAC)[4,1:2]) 1055 | ecms[15:16,1] = t(coeftest(ecm.1, vcov=vcovHAC)[1,1:2]) 1056 | ecms[18,1] = lr 1057 | ecms[20,1] <- nobs(ecm.1) 1058 | ecms[21,1] <- summary(ecm.1)$adj.r.squared 1059 | ecms[22,1] <- summary(ecm.1)$sigma 1060 | 1061 | # Main Polarity Measure and Misery Index 1062 | 1063 | ecm.2 <- lm(d1(polar) ~ d1(misery) + l(polar,-1) + l(misery,-1), data=dq) 1064 | coeftest(ecm.2, vcov=vcovHAC) 1065 | # Long-run effect: 1066 | lr <- -coef(ecm.2)[[4]]/coef(ecm.2)[[3]] 1067 | ecms[1:2,2] = t(coeftest(ecm.2, vcov=vcovHAC)[3,1:2]) 1068 | ecms[7:8,2] = t(coeftest(ecm.2, vcov=vcovHAC)[2,1:2]) 1069 | ecms[9:10,2] = t(coeftest(ecm.2, vcov=vcovHAC)[4,1:2]) 1070 | ecms[15:16,2] = t(coeftest(ecm.2, vcov=vcovHAC)[1,1:2]) 1071 | ecms[18,2] = lr 1072 | ecms[20,2] <- nobs(ecm.2) 1073 | ecms[21,2] <- summary(ecm.2)$adj.r.squared 1074 | ecms[22,2] <- summary(ecm.2)$sigma 1075 | 1076 | # Main Polarity Measure and Unemployment 1077 | 1078 | ecm.3 <- lm(d1(polar) ~ d1(unemp) + l(polar,-1) + l(unemp,-1), data=dq) 1079 | coeftest(ecm.3, vcov=vcovHAC) 1080 | # Long-run effect: 1081 | lr <- -coef(ecm.3)[[4]]/coef(ecm.3)[[3]] 1082 | ecms[1:2,3] = t(coeftest(ecm.3, vcov=vcovHAC)[3,1:2]) 1083 | ecms[11:12,3] = t(coeftest(ecm.3, vcov=vcovHAC)[2,1:2]) 1084 | ecms[13:14,3] = t(coeftest(ecm.3, vcov=vcovHAC)[4,1:2]) 1085 | ecms[15:16,3] = t(coeftest(ecm.3, vcov=vcovHAC)[1,1:2]) 1086 | ecms[18,3] = lr 1087 | ecms[20,3] <- nobs(ecm.3) 1088 | ecms[21,3] <- summary(ecm.3)$adj.r.squared 1089 | ecms[22,3] <- summary(ecm.3)$sigma 1090 | 1091 | write.table(ecms,"of/q-ecms.csv",sep=",",row.names=T,col.names=NA,na="") 1092 | 1093 | 1094 | # Testing a model with longer lag length (not reported in text) 1095 | 1096 | ecm.1 <- lm(d1(polar) ~ d1(ldisp) + l(polar,-4:-1) + l(ldisp,-4:-1), data=dq) 1097 | coeftest(ecm.1, vcov=vcovHAC) 1098 | # Long-run effect: 1099 | - (coef(ecm.1)[[7]] + coef(ecm.1)[[7]] + coef(ecm.1)[[7]] + coef(ecm.1)[[7]]) / 1100 | (coef(ecm.1)[[3]] + coef(ecm.1)[[4]] + coef(ecm.1)[[5]] + coef(ecm.1)[[6]]) 1101 | 1102 | #-------------------------------------------------------------------------------------------# 1103 | # Figure 1. Emotional Polarity 1104 | #-------------------------------------------------------------------------------------------# 1105 | 1106 | mblue <- "#002A48" 1107 | pblue <- "#3F5F75" 1108 | 1109 | #-------------------------------------------------------------------------------------------# 1110 | # Yearly 1111 | 1112 | dy$polsm <- smooth.spline(dy$polar,spar=.5)$y 1113 | p1a <- ggplot(dy, aes(x = year)) + theme_bw() + 1114 | geom_line(aes(y = polar), size = .1, colour=pblue) + 1115 | geom_point(aes(y = polar), size = 6, colour=pblue, shape=1, stroke=.25) + 1116 | geom_line(aes(y = polsm), size = 2, colour=mblue) + 1117 | theme(axis.text=element_text(size=10), 1118 | axis.title=element_text(size=12), 1119 | axis.title.y=element_text(margin=margin(0,10,0,0)), 1120 | axis.title.x=element_text(margin=margin(10,0,0,0)), 1121 | plot.margin = unit(c(.3,.3,.3,.3), "cm")) + 1122 | labs(x="A. Year", y="Emotional Polarity") + 1123 | scale_x_continuous(breaks=c(1910,1920,1930,1940,1950,1960,1970,1980,1990,2000,2010)) + 1124 | scale_y_continuous(breaks=c(-2,-1,0,1,2,3),limits = c(-2.5,3.5)) + 1125 | geom_hline(aes(yintercept=0), linetype="dashed", size = .25, colour="#3f3f3f") 1126 | 1127 | #-------------------------------------------------------------------------------------------# 1128 | # Quarterly 1129 | 1130 | dq$polsm <- smooth.spline(dq$polar)$y 1131 | dq$time <- 1:420 1132 | p1b <- ggplot(dq, aes(x = time)) + theme_bw() + 1133 | geom_line(aes(y = polar), size = .1, colour=pblue) + 1134 | geom_point(aes(y = polar), size = 6, colour=pblue, shape=1, stroke=.25) + 1135 | geom_line(aes(y = polsm), size = 2, colour=mblue) + 1136 | theme(axis.text=element_text(size=10), 1137 | axis.title=element_text(size=12), 1138 | axis.title.y=element_text(margin=margin(0,10,0,0)), 1139 | axis.title.x=element_text(margin=margin(10,0,0,0)), 1140 | plot.margin = unit(c(.3,.3,.3,.3), "cm")) + 1141 | labs(x="B. Quarter",y="Emotional Polarity") + 1142 | scale_x_continuous(breaks=c(5,85,165,245,325,405), 1143 | labels=c("1910-Q1","1930-Q1","1950-Q1","1970-Q1","1990-Q1","2010-Q1")) + 1144 | scale_y_continuous(breaks=c(-2,-1,0,1,2,3),limits = c(-2.5,3.5)) + 1145 | geom_hline(aes(yintercept=0), linetype="dashed", size = .25, colour="#3f3f3f") 1146 | 1147 | grid.arrange(p1a, p1b, ncol = 1) 1148 | 1149 | #-------------------------------------------------------------------------------------------# 1150 | # Figure 2. Emotional Polarity by Government/Opposition 1151 | #-------------------------------------------------------------------------------------------# 1152 | 1153 | dys <- dy[38:nrow(dy),] 1154 | dys$polgsm <- smooth.spline(dys$polarg_graph,spar=.5)$y 1155 | dys$polosm <- smooth.spline(dys$polaro_graph,spar=.5)$y 1156 | 1157 | ggplot(dys, aes(x = year)) + theme_bw() + 1158 | geom_line(aes(y = polarg_graph), size = .1, colour="#3F5F75") + 1159 | geom_point(aes(y = polarg_graph), size = 6, colour="#3F5F75", shape=1, stroke=.25) + 1160 | geom_line(aes(y = polaro_graph), size = .1, colour="#3FA8C3") + 1161 | geom_point(aes(y = polaro_graph), size = 6, colour="#3FA8C3", shape=1, stroke=.25) + 1162 | geom_line(aes(y = polgsm, colour="Government"), size = 2) + 1163 | geom_line(aes(y = polosm, colour="Opposition"), size = 2) + 1164 | scale_colour_manual(name="", values=c(Government="#002A48",Opposition="#008BB0")) + 1165 | theme(legend.position=c(0.25,0.8), 1166 | legend.text = element_text(size = 10), 1167 | legend.key.height=unit(1.5,"line"), 1168 | legend.key.size=unit(2.5,"line"), 1169 | legend.key = element_blank(), 1170 | axis.text=element_text(size=10), 1171 | axis.title=element_text(size=12), 1172 | axis.title.y=element_text(margin=margin(0,10,0,0)), 1173 | axis.title.x=element_text(margin=margin(10,0,0,0)), 1174 | plot.margin = unit(c(.1,.1,.1,.1), "cm")) + 1175 | labs(x="Year",y="Emotional Polarity") + 1176 | scale_x_continuous(breaks=c(1950,1960,1970,1980,1990,2000,2010)) 1177 | 1178 | 1179 | #-------------------------------------------------------------------------------------------# 1180 | # Figure S1: Autocorrelation Functions 1181 | #-------------------------------------------------------------------------------------------# 1182 | 1183 | #-------------------------------------------------------------------------------------------# 1184 | # Yearly 1185 | 1186 | poly <- ts(dy$polar,start=c(1909),freq=1) 1187 | 1188 | conf.level <- 0.95 1189 | ciline <- qnorm((1 - conf.level)/2)/sqrt(length(poly)) 1190 | acfy <- acf(poly, plot = TRUE, lag.max=20) 1191 | dacfy <- data.frame(acf=acfy[[1]][2:21],lag=1:20) 1192 | 1193 | pacf1 <- ggplot(data=dacfy, mapping=aes(x=lag, y=acf)) + theme_bw() + 1194 | geom_bar(stat = "identity", position = "identity", colour="#002A48", fill="#002A48", width=.7, size=.1) + 1195 | labs(x=expression(atop("Lags", paste("A. Yearly"))),y="Autocorrelation Function") + 1196 | theme(axis.text=element_text(size=10), 1197 | axis.title=element_text(size=12), 1198 | axis.title.y=element_text(margin=margin(0,10,0,0)), 1199 | axis.title.x=element_text(margin=margin(10,0,0,0)), 1200 | plot.margin = unit(c(.3,.3,.3,.3), "cm")) + 1201 | scale_y_continuous(breaks=c(0.00,0.25,0.50,0.75,1.00), lim=c(0,1)) + 1202 | geom_hline(aes(yintercept=(-ciline)), linetype="dashed", size = .5, colour="#000000") 1203 | 1204 | #-------------------------------------------------------------------------------------------# 1205 | # Quarterly 1206 | 1207 | polq <- ts(dq$polar,start=c(1909,1),freq=4) 1208 | 1209 | conf.level <- 0.95 1210 | ciline <- qnorm((1 - conf.level)/2)/sqrt(length(polq)) 1211 | acfq <- acf(polq, plot = TRUE, lag.max=40) 1212 | dacfq <- data.frame(acf=acfq[[1]][2:41],lag=1:40) 1213 | 1214 | pacf2 <- ggplot(data=dacfq, mapping=aes(x=lag, y=acf)) + theme_bw() + 1215 | geom_bar(stat = "identity", position = "identity", colour="#002A48", fill="#002A48", width=.7, size=.1) + 1216 | labs(x=expression(atop("Lags", paste("B. Quarterly"))) ,y="Autocorrelation Function") + 1217 | theme(axis.text=element_text(size=10), 1218 | axis.title=element_text(size=12), 1219 | axis.title.y=element_text(margin=margin(0,10,0,0)), 1220 | axis.title.x=element_text(margin=margin(10,0,0,0)), 1221 | plot.margin = unit(c(.3,.3,.3,.3), "cm")) + 1222 | scale_y_continuous(breaks=c(0.00,0.25,0.50,0.75,1.00), lim=c(0,1)) + 1223 | geom_hline(aes(yintercept=(-ciline)), linetype="dashed", size = .5, colour="#000000") 1224 | 1225 | grid.arrange(pacf1, pacf2, ncol = 1) 1226 | 1227 | #-------------------------------------------------------------------------------------------# 1228 | # Figure S2: Power Spectral Densities 1229 | #-------------------------------------------------------------------------------------------# 1230 | 1231 | #-------------------------------------------------------------------------------------------# 1232 | # Yearly 1233 | 1234 | x <- dy$polar 1235 | N <- nrow(dy) 1236 | yspec <- (abs(fft(x))^2/(2*pi*N))[2:(((N+1)/2)+1)] 1237 | yfreq = seq(1,(N+1)/2)*(2*pi/N) 1238 | plot(log(yspec)~log(yfreq),type="l") 1239 | regalpha <- lm(log(yspec)~log(yfreq)) 1240 | # Coefficients mentioned in the text (alpha, CI lower bound, CI upper bound): 1241 | -coef(regalpha)[[2]]; confint(regalpha)[2,2]*-1; confint(regalpha)[2,1]*-1 1242 | powsy = data.frame(frequency=log(yfreq),spectrum=log(yspec)) 1243 | 1244 | psp1 <- ggplot(powsy, aes(x = frequency)) + theme_bw() + 1245 | geom_line(aes(y = spectrum, colour="#b20000", size = 2),colour="#002A48", size = 1.5) + 1246 | labs(x=expression(atop("Log Frequency", paste("A. Yearly"))),y="Log Power Spectrum") + 1247 | theme(axis.text=element_text(size=10), 1248 | axis.title=element_text(size=12), 1249 | axis.title.y=element_text(margin=margin(0,10,0,0)), 1250 | axis.title.x=element_text(margin=margin(10,0,0,0)), 1251 | plot.margin = unit(c(.3,.3,.3,.3), "cm")) + 1252 | scale_y_continuous(lim=c(-10,3)) 1253 | 1254 | #-------------------------------------------------------------------------------------------# 1255 | # Quarterly 1256 | 1257 | x <- dq$polar 1258 | N <- nrow(dq) 1259 | qspec <- (abs(fft(x))^2/(2*pi*N))[2:((N/2)+1)] 1260 | qfreq = seq(1,N/2)*(2*pi/N) 1261 | plot(log(qspec)~log(qfreq),type="l") 1262 | regalpha <- lm(log(qspec)~log(qfreq)) 1263 | # Coefficients mentioned in the text (alpha, CI lower bound, CI upper bound): 1264 | -coef(regalpha)[[2]]; confint(regalpha)[2,2]*-1; confint(regalpha)[2,1]*-1 1265 | powsq = data.frame(frequency=log(qfreq),spectrum=log(qspec)) 1266 | 1267 | psp2 <- ggplot(powsq, aes(x = frequency)) + theme_bw() + 1268 | geom_line(aes(y = spectrum, colour="#002A48", size = 2),colour="#002A48", size = 1.5) + 1269 | labs(x=expression(atop("Log Frequency", paste("B. Quarterly"))),y="Log Power Spectrum") + 1270 | theme(axis.text=element_text(size=10), 1271 | axis.title=element_text(size=12), 1272 | axis.title.y=element_text(margin=margin(0,10,0,0)), 1273 | axis.title.x=element_text(margin=margin(10,0,0,0)), 1274 | plot.margin = unit(c(.3,.3,.3,.3), "cm")) + 1275 | scale_y_continuous(lim=c(-10,3)) 1276 | 1277 | grid.arrange(psp1, psp2, ncol = 1) 1278 | 1279 | #-------------------------------------------------------------------------------------------# 1280 | # Figure 3: Superimposed Time Series - Mood and the National Economy 1281 | #-------------------------------------------------------------------------------------------# 1282 | 1283 | sp1 <- ggplot(dy, aes(x = year)) + theme_bw() + 1284 | geom_line(aes(y = polar, colour="Polarity", size = 2), size = 2) + 1285 | geom_line(aes(y = ldisp, colour="Labor Disputes", size = 2), size = 2) + 1286 | theme(legend.position=c(0.4,0.85), 1287 | legend.text = element_text(size = 12), 1288 | legend.key.height=unit(1.3,"line"), 1289 | legend.key.size=unit(2,"line"), 1290 | legend.key = element_blank(), 1291 | axis.text=element_text(size=10), 1292 | axis.title=element_text(size=12), 1293 | axis.title.y=element_text(margin=margin(0,10,0,0)), 1294 | axis.title.x=element_text(margin=margin(10,0,0,0)), 1295 | plot.margin = unit(c(.3,.3,.3,.3), "cm")) + 1296 | scale_colour_manual(name="", values=c('Labor Disputes'="#b20000", 'Polarity'="#002A48")) + 1297 | labs(x="Year",y="Standard Scores") + 1298 | scale_x_continuous(breaks=c(1910,1920,1930,1940,1950,1960,1970,1980,1990,2000,2010)) + 1299 | scale_y_continuous(breaks=c(-2,-1,0,1,2,3), lim=c(-2.5,3.5)) 1300 | 1301 | sp2 <- ggplot(dy, aes(x = year)) + theme_bw() + 1302 | geom_line(aes(y = polar, colour="Polarity", size = 2), size = 2) + 1303 | geom_line(aes(y = misery, colour="Misery Index", size = 2), size = 2) + 1304 | theme(legend.position=c(0.4,0.85), 1305 | legend.text = element_text(size = 12), 1306 | legend.key.height=unit(1.3,"line"), 1307 | legend.key.size=unit(2,"line"), 1308 | legend.key = element_blank(), 1309 | axis.text=element_text(size=10), 1310 | axis.title=element_text(size=12), 1311 | axis.title.y=element_text(margin=margin(0,10,0,0)), 1312 | axis.title.x=element_text(margin=margin(10,0,0,0)), 1313 | plot.margin = unit(c(.3,.3,.3,.3), "cm")) + 1314 | scale_colour_manual(name="", values=c('Misery Index'="#FF8C00", 'Polarity'="#002A48")) + 1315 | labs(x="Year",y="Standard Scores") + 1316 | scale_x_continuous(breaks=c(1910,1920,1930,1940,1950,1960,1970,1980,1990,2000,2010)) + 1317 | scale_y_continuous(breaks=c(-2,-1,0,1,2,3), lim=c(-2.5,3.5)) 1318 | 1319 | grid.arrange(sp1, sp2, ncol = 1) 1320 | 1321 | #-------------------------------------------------------------------------------------------# 1322 | # Figure S3. Heat Map 1323 | #-------------------------------------------------------------------------------------------# 1324 | 1325 | range01 <- function(x){(x-min(x))/(max(x)-min(x))} 1326 | 1327 | d2 <- data.frame(year=dy$year,polarity=range01(dy$polar), 1328 | disputes=range01(dy$ldisp), 1329 | unemployment=range01(dy$unemp), 1330 | misery=range01(dy$misery), 1331 | gdpg=range01(dy$gdpg)) 1332 | d2 <- melt(d2,id.vars="year") 1333 | colf <- colorRampPalette(c("#00007F","#7F0000")) 1334 | jetc <- colorRampPalette(c("#00007F", "blue", "#007FFF", "cyan", 1335 | "#7FFF7F", "yellow", "#FF7F00", "red", "#7F0000")) 1336 | 1337 | ggplot(d2, aes(x = year, y = variable, fill = value)) + theme_bw() + 1338 | geom_tile() + 1339 | scale_fill_gradientn(colours = jetc(100),name = "", 1340 | breaks=c(0,1), 1341 | labels=c("Low","High")) + 1342 | scale_y_discrete(limits=c("gdpg","misery","unemployment","disputes","polarity"), 1343 | labels=c("GDP Growth","Misery Index","Unemployment","Labor Disputes","Polarity"), 1344 | name="") + 1345 | labs(x="Year") + 1346 | theme(axis.text=element_text(size=10), 1347 | axis.title=element_text(size=12), 1348 | axis.title.x=element_text(margin=margin(10,0,0,0)), 1349 | legend.text=element_text(size=10), 1350 | legend.title=element_text(size=12), 1351 | legend.key.size = unit(1.5, "lines"), 1352 | plot.margin = unit(c(.3,.3,.3,.3), "cm")) 1353 | 1354 | #-------------------------------------------------------------------------------------------# 1355 | # Figure S4. Alternative, General-Purpose Polarity Lexicons 1356 | #-------------------------------------------------------------------------------------------# 1357 | 1358 | dq$time <- 1:nrow(dq) 1359 | dq$polsn <- smooth.spline(dq$nrcpol)$y 1360 | dq$polss <- smooth.spline(dq$swnpol)$y 1361 | dq$polso <- smooth.spline(dq$ofpol)$y 1362 | 1363 | ggplot(dq, aes(x = time)) + theme_bw() + 1364 | geom_point(aes(y = nrcpol), size = 6, colour="#C53F3F", shape=1, stroke=.1) + 1365 | geom_point(aes(y = ofpol), size = 6, colour="#3F8C3F", shape=1, stroke=.1) + 1366 | geom_point(aes(y = swnpol), size = 6, colour="#033FC5", shape=1, stroke=.1) + 1367 | geom_line(aes(y = polsn, colour="NRC", size = 2), size = 2) + 1368 | geom_line(aes(y = polso, colour="OpinionFinder"), size = 2) + 1369 | geom_line(aes(y = polss, colour="SentiWordNet"), size = 2) + 1370 | 1371 | theme(legend.position=c(0.2,0.82), 1372 | legend.text = element_text(size = 12), 1373 | legend.key.height=unit(1.1,"line"), 1374 | legend.key.size=unit(2,"line"), 1375 | legend.key = element_blank(), 1376 | axis.text=element_text(size=10), 1377 | axis.title=element_text(size=12), 1378 | axis.title.y=element_text(margin=margin(0,10,0,0)), 1379 | axis.title.x=element_text(margin=margin(10,0,0,0)), 1380 | plot.margin = unit(c(.3,.3,.3,.3), "cm")) + 1381 | scale_colour_manual(name="", values=c(NRC="#b20000",OpinionFinder="#006600", 1382 | SentiWordNet="#0000b2")) + 1383 | labs(x="Quarter",y="Emotional Polarity") + 1384 | scale_x_continuous(breaks=c(5,85,165,245,325,405), 1385 | labels=c("1910-Q1","1930-Q1","1950-Q1","1970-Q1","1990-Q1","2010-Q1")) + 1386 | scale_y_continuous(breaks=c(-3,-2,-1,0,1,2,3,4)) + 1387 | geom_hline(aes(yintercept=0), linetype="dashed", size = .5, colour="#3f3f3f") 1388 | 1389 | --------------------------------------------------------------------------------