├── .gitignore ├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── persister ├── assembly.xml ├── pom.xml └── src │ └── main │ └── resources │ └── persister_conf.xml ├── speech01 ├── README.md ├── assembly.xml ├── pom.xml └── src │ └── main │ ├── java │ └── io │ │ └── s4 │ │ └── example │ │ └── speech01 │ │ ├── Highlight.java │ │ ├── Sentence.java │ │ ├── SentenceReceiverPE.java │ │ └── Speech.java │ └── resources │ └── speech01_conf.xml ├── speech01_scala ├── README.md ├── assembly.xml ├── pom.xml └── src │ └── main │ ├── resources │ └── speech01_scala_conf.xml │ └── scala │ ├── Events.scala │ └── SentenceReceiverPE.scala ├── speech02 ├── README.md ├── assembly.xml ├── pom.xml └── src │ └── main │ └── resources │ └── speech02_conf.xml ├── testinput ├── pe-query ├── proto-query ├── sentence.in ├── speech.in └── speeches.txt ├── twittertopiccount ├── NOTICE.txt ├── README.md ├── assembly.xml ├── pom.xml └── src │ └── main │ ├── java │ └── io │ │ └── s4 │ │ └── example │ │ └── twittertopiccount │ │ ├── DirectToFilePersister.java │ │ ├── Status.java │ │ ├── TopNTopicPE.java │ │ ├── TopicCountAndReportPE.java │ │ ├── TopicExtractorPE.java │ │ ├── TopicSeen.java │ │ ├── TwitterFeedListener.java │ │ └── User.java │ └── resources │ ├── adapter_conf.xml │ └── twittertopiccount_conf.xml └── twittertopiccount_scala ├── README.md ├── assembly.xml ├── pom.xml └── src └── main ├── java └── io │ └── s4 │ └── example │ └── twittertopiccount │ └── DirectToFilePersister.java ├── resources ├── adapter_conf.xml └── twittertopiccount_scala_conf.xml └── scala ├── events └── Events.scala ├── listener └── TwitterStreamListener.scala ├── processor ├── TopNTopicPE.scala ├── TopicCountAndReportPE.scala └── TopicExtractorPE.scala └── util └── TwitterStreamClient.scala /.gitignore: -------------------------------------------------------------------------------- 1 | # Eclipse meta-information 2 | .project 3 | .classpath 4 | .settings 5 | 6 | # Build directory 7 | */target/* 8 | 9 | # SVN 10 | .svn 11 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, and 10 | distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by the 13 | copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all other 16 | entities that control, are controlled by, or are under common control with 17 | that entity. For the purposes of this definition, "control" means (i) the 18 | power, direct or indirect, to cause the direction or management of such 19 | entity, whether by contract or otherwise, or (ii) ownership of fifty 20 | percent (50%) or more of the outstanding shares, or (iii) beneficial 21 | ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity exercising 24 | permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation source, 28 | and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical transformation 31 | or translation of a Source form, including but not limited to compiled 32 | object code, generated documentation, and conversions to other media types. 33 | 34 | "Work" shall mean the work of authorship, whether in Source or Object 35 | form, made available under the License, as indicated by a copyright notice 36 | that is included in or attached to the work (an example is provided in the 37 | Appendix below). 38 | 39 | "Derivative Works" shall mean any work, whether in Source or Object form, 40 | that is based on (or derived from) the Work and for which the editorial 41 | revisions, annotations, elaborations, or other modifications represent, as 42 | a whole, an original work of authorship. For the purposes of this License, 43 | Derivative Works shall not include works that remain separable from, or 44 | merely link (or bind by name) to the interfaces of, the Work and Derivative 45 | Works thereof. 46 | 47 | "Contribution" shall mean any work of authorship, including the original 48 | version of the Work and any modifications or additions to that Work or 49 | Derivative Works thereof, that is intentionally submitted to Licensor for 50 | inclusion in the Work by the copyright owner or by an individual or Legal 51 | Entity authorized to submit on behalf of the copyright owner. For the 52 | purposes of this definition, "submitted" means any form of electronic, 53 | verbal, or written communication sent to the Licensor or its 54 | representatives, including but not limited to communication on electronic 55 | mailing lists, source code control systems, and issue tracking systems that 56 | are managed by, or on behalf of, the Licensor for the purpose of discussing 57 | and improving the Work, but excluding communication that is conspicuously 58 | marked or otherwise designated in writing by the copyright owner as "Not a 59 | Contribution." 60 | 61 | "Contributor" shall mean Licensor and any individual or Legal Entity on 62 | behalf of whom a Contribution has been received by Licensor and 63 | subsequently incorporated within the Work. 64 | 65 | 2. Grant of Copyright License. 66 | 67 | Subject to the terms and conditions of this License, each Contributor 68 | hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, 69 | royalty-free, irrevocable copyright license to reproduce, prepare 70 | Derivative Works of, publicly display, publicly perform, sublicense, and 71 | distribute the Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. 74 | 75 | Subject to the terms and conditions of this License, each Contributor 76 | hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, 77 | royalty-free, irrevocable (except as stated in this section) patent license 78 | to make, have made, use, offer to sell, sell, import, and otherwise 79 | transfer the Work, where such license applies only to those patent claims 80 | licensable by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) with the 82 | Work to which such Contribution(s) was submitted. If You institute patent 83 | litigation against any entity (including a cross-claim or counterclaim in a 84 | lawsuit) alleging that the Work or a Contribution incorporated within the 85 | Work constitutes direct or contributory patent infringement, then any 86 | patent licenses granted to You under this License for that Work shall 87 | terminate as of the date such litigation is filed. 88 | 89 | 4. Redistribution. 90 | 91 | You may reproduce and distribute copies of the Work or Derivative Works 92 | thereof in any medium, with or without modifications, and in Source or 93 | Object form, provided that You meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or Derivative Works a 96 | copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices stating 99 | that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works that You 102 | distribute, all copyright, patent, trademark, and attribution notices 103 | from the Source form of the Work, excluding those notices that do not 104 | pertain to any part of the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its distribution, 107 | then any Derivative Works that You distribute must include a readable 108 | copy of the attribution notices contained within such NOTICE file, 109 | excluding those notices that do not pertain to any part of the 110 | Derivative Works, in at least one of the following places: within a 111 | NOTICE text file distributed as part of the Derivative Works; within 112 | the Source form or documentation, if provided along with the 113 | Derivative Works; or, within a display generated by the Derivative 114 | Works, if and wherever such third-party notices normally appear. The 115 | contents of the NOTICE file are for informational purposes only and 116 | do not modify the License. You may add Your own attribution notices 117 | within Derivative Works that You distribute, alongside or as an 118 | addendum to the NOTICE text from the Work, provided that such 119 | additional attribution notices cannot be construed as modifying 120 | the License. 121 | 122 | You may add Your own copyright statement to Your modifications and may 123 | provide additional or different license terms and conditions for use, 124 | reproduction, or distribution of Your modifications, or for any such 125 | Derivative Works as a whole, provided Your use, reproduction, and 126 | distribution of the Work otherwise complies with the conditions stated in 127 | this License. 128 | 129 | 5. Submission of Contributions. 130 | 131 | Unless You explicitly state otherwise, any Contribution intentionally 132 | submitted for inclusion in the Work by You to the Licensor shall be under 133 | the terms and conditions of this License, without any additional terms or 134 | conditions. Notwithstanding the above, nothing herein shall supersede or 135 | modify the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. 139 | 140 | This License does not grant permission to use the trade names, trademarks, 141 | service marks, or product names of the Licensor, except as required for 142 | reasonable and customary use in describing the origin of the Work and 143 | reproducing the content of the NOTICE file. 144 | 145 | 7. Disclaimer of Warranty. 146 | 147 | Unless required by applicable law or agreed to in writing, Licensor 148 | provides the Work (and each Contributor provides its Contributions) on an 149 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 150 | or implied, including, without limitation, any warranties or conditions of 151 | TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR 152 | PURPOSE. You are solely responsible for determining the appropriateness of 153 | using or redistributing the Work and assume any risks associated with Your 154 | exercise of permissions under this License. 155 | 156 | 8. Limitation of Liability. 157 | 158 | In no event and under no legal theory, whether in tort (including 159 | negligence), contract, or otherwise, unless required by applicable law 160 | (such as deliberate and grossly negligent acts) or agreed to in writing, 161 | shall any Contributor be liable to You for damages, including any direct, 162 | indirect, special, incidental, or consequential damages of any character 163 | arising as a result of this License or out of the use or inability to use 164 | the Work (including but not limited to damages for loss of goodwill, work 165 | stoppage, computer failure or malfunction, or any and all other commercial 166 | damages or losses), even if such Contributor has been advised of the 167 | possibility of such damages. 168 | 169 | 9. Accepting Warranty or Additional Liability. 170 | 171 | While redistributing the Work or Derivative Works thereof, You may choose 172 | to offer, and charge a fee for, acceptance of support, warranty, indemnity, 173 | or other liability obligations and/or rights consistent with this License. 174 | However, in accepting such obligations, You may act only on Your own behalf 175 | and on Your sole responsibility, not on behalf of any other Contributor, 176 | and only if You agree to indemnify, defend, and hold each Contributor 177 | harmless for any liability incurred by, or claims asserted against, such 178 | Contributor by reason of your accepting any such warranty or additional 179 | liability. 180 | 181 | END OF TERMS AND CONDITIONS 182 | 183 | APPENDIX: How to apply the Apache License to your work. 184 | 185 | To apply the Apache License to your work, attach the following boilerplate 186 | notice, with the fields enclosed by brackets "[]" replaced with your own 187 | identifying information. (Don't include the brackets!) The text should be 188 | enclosed in the appropriate comment syntax for the file format. We also 189 | recommend that a file or class name and description of purpose be included on 190 | the same "printed page" as the copyright notice for easier identification 191 | within third-party archives. 192 | 193 | Copyright [yyyy] [name of copyright owner] 194 | 195 | Licensed under the Apache License, Version 2.0 (the "License"); 196 | you may not use this file except in compliance with the License. 197 | You may obtain a copy of the License at 198 | 199 | http://www.apache.org/licenses/LICENSE-2.0 200 | 201 | Unless required by applicable law or agreed to in writing, software 202 | distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 203 | WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 204 | 205 | See the License for the specific language governing permissions and 206 | limitations under the License. 207 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | ======================================================================= 2 | NOTICE file for use with, and corresponding to Section 4 of, 3 | the Apache License, Version 2.0, in this case for the S4 project. 4 | ========================================================================= 5 | 6 | This product includes software developed by 7 | Yahoo! Inc. (www.yahoo.com) 8 | Copyright (c) 2010 Yahoo! Inc. All rights reserved. 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | S4 Example Applications 2 | ======================= 3 | 4 | This is a set of example applications using S4. To build and deploy an example 5 | 6 | 1. Read the example's README file for build instructions 7 | 2. Refer to the [Getting Started](http://wiki.s4.io/Tutorials/GettingStarted) wiki for instructions on how to deploy and run the application 8 | 9 | Realtime Twitter Topic Count 10 | ---------------------------- 11 | 12 | This application detects popular hashtags on Twitter by listening to the 13 | Twitter gardenhose. 14 | 15 | Build instructions: twittertopiccount/README.md 16 | -------------------------------------------------------------------------------- /persister/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | dir 6 | 7 | false 8 | 9 | 10 | false 11 | lib 12 | runtime 13 | false 14 | 15 | 16 | 17 | 18 | src/main/resources 19 | 20 | 21 | persister_conf.xml 22 | 23 | 24 | 25 | target 26 | 27 | 28 | *.jar 29 | 30 | 31 | 32 | 33 | 34 | 35 | pom.xml 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /persister/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | io.s4.examples.persister 5 | persister 6 | jar 7 | 0.0.0.1 8 | persister 9 | http://maven.apache.org 10 | 11 | 12 | log4j 13 | log4j 14 | 1.2.15 15 | 16 | 17 | javax.mail 18 | mail 19 | 20 | 21 | javax.jms 22 | jms 23 | 24 | 25 | com.sun.jdmk 26 | jmxtools 27 | 28 | 29 | com.sun.jmx 30 | jmxri 31 | 32 | 33 | 34 | 35 | io.s4 36 | s4_core 37 | 2.0.0.0 38 | provided 39 | 40 | 41 | 42 | ${artifactId}-${version} 43 | 44 | 45 | maven-compiler-plugin 46 | 2.0.1 47 | 48 | 1.6 49 | 1.6 50 | 51 | 52 | 53 | maven-assembly-plugin 54 | 55 | false 56 | 57 | assembly.xml 58 | 59 | 60 | 61 | 62 | org.codehaus.mojo 63 | findbugs-maven-plugin 64 | 2.0 65 | 66 | true 67 | true 68 | High 69 | 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /persister/src/main/resources/persister_conf.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | myapp:(viewCount|clickCount|viewQueryLinkCount|top10Topics):.* 13 | 14 | 15 | 16 | myapp:top10Topics 17 | 18 | 19 | 20 | myapp:clickViewCount:.* 21 | 22 | 23 | 24 | myapp:clickViewCountWindow:.* 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /speech01/README.md: -------------------------------------------------------------------------------- 1 | Hello World Example 2 | =============== 3 | 4 | Introduction 5 | ------------ 6 | This sample application demonstrates getting an event into an s4 node cluster 7 | 8 | Requirements 9 | ------------ 10 | 11 | * Linux 12 | * Java 1.6 13 | * Maven 14 | * S4 Communication Layer 15 | * S4 Core 16 | 17 | Build Instructions 18 | ------------------ 19 | 20 | 1. First build and install the comm and core packages in your Maven repository. 21 | 22 | 2. Build and install using Maven 23 | 24 | mvn install assembly:assembly 25 | -------------------------------------------------------------------------------- /speech01/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | tar.gz 6 | 7 | true 8 | ${artifactId} 9 | 10 | 11 | false 12 | lib 13 | runtime 14 | false 15 | 16 | log4j:log4j 17 | 18 | 19 | 20 | 21 | 22 | ${project.basedir}/target 23 | lib 24 | 25 | *.jar 26 | 27 | 28 | 29 | ${project.basedir}/src/main/resources 30 | 31 | 32 | speech01_conf.xml 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /speech01/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | io.s4.examples 5 | speech01 6 | jar 7 | 0.0.0.1 8 | speech01 9 | http://maven.apache.org 10 | 11 | 12 | log4j 13 | log4j 14 | 1.2.15 15 | 16 | 17 | javax.mail 18 | mail 19 | 20 | 21 | javax.jms 22 | jms 23 | 24 | 25 | com.sun.jdmk 26 | jmxtools 27 | 28 | 29 | com.sun.jmx 30 | jmxri 31 | 32 | 33 | 34 | 35 | io.s4 36 | s4_core 37 | 0.3.0.0 38 | provided 39 | 40 | 41 | 42 | ${artifactId}-${version} 43 | 44 | 45 | src/main/resources 46 | 47 | * 48 | 49 | 50 | 51 | 52 | 53 | maven-compiler-plugin 54 | 2.0.1 55 | 56 | 1.6 57 | 1.6 58 | 59 | 60 | 61 | maven-assembly-plugin 62 | 63 | 64 | assembly.xml 65 | 66 | 67 | 68 | 69 | org.codehaus.mojo 70 | findbugs-maven-plugin 71 | 2.0 72 | 73 | true 74 | true 75 | High 76 | 77 | 78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /speech01/src/main/java/io/s4/example/speech01/Highlight.java: -------------------------------------------------------------------------------- 1 | package io.s4.example.speech01; 2 | 3 | public class Highlight { 4 | private long sentenceId; 5 | private long time; 6 | 7 | public long getSentenceId() { 8 | return sentenceId; 9 | } 10 | 11 | public void setSentenceId(long sentenceId) { 12 | this.sentenceId = sentenceId; 13 | } 14 | 15 | public long getTime() { 16 | return time; 17 | } 18 | 19 | public void setTime(long time) { 20 | this.time = time; 21 | } 22 | 23 | public String toString() { 24 | StringBuilder sb = new StringBuilder(); 25 | sb.append("{sentenceId:") 26 | .append(sentenceId) 27 | .append(",time:") 28 | .append(time) 29 | .append("}"); 30 | return sb.toString(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /speech01/src/main/java/io/s4/example/speech01/Sentence.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.example.speech01; 17 | 18 | public class Sentence { 19 | private long id; 20 | private long speechId; 21 | private String text; 22 | private long time; 23 | private String location; 24 | 25 | public long getId() { 26 | return id; 27 | } 28 | 29 | public void setId(long id) { 30 | this.id = id; 31 | } 32 | 33 | public long getSpeechId() { 34 | return speechId; 35 | } 36 | 37 | public void setSpeechId(long speechId) { 38 | this.speechId = speechId; 39 | } 40 | 41 | public String getText() { 42 | return text; 43 | } 44 | 45 | public void setText(String text) { 46 | this.text = text; 47 | } 48 | 49 | public long getTime() { 50 | return time; 51 | } 52 | 53 | public void setTime(long time) { 54 | this.time = time; 55 | } 56 | 57 | public String getLocation() { 58 | return location; 59 | } 60 | 61 | public void setLocation(String location) { 62 | this.location = location; 63 | } 64 | 65 | public String toString() { 66 | StringBuffer sb = new StringBuffer(); 67 | sb.append("{id:") 68 | .append(id) 69 | .append(",speechId:") 70 | .append(speechId) 71 | .append(",text:") 72 | .append(text) 73 | .append(",time:") 74 | .append(time) 75 | .append(",location:") 76 | .append(location) 77 | .append("}"); 78 | 79 | return sb.toString(); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /speech01/src/main/java/io/s4/example/speech01/SentenceReceiverPE.java: -------------------------------------------------------------------------------- 1 | package io.s4.example.speech01; 2 | 3 | import io.s4.processor.AbstractPE; 4 | 5 | public class SentenceReceiverPE extends AbstractPE { 6 | 7 | public void processEvent(Sentence sentence) { 8 | System.out.printf("Sentence is '%s', location %s\n", sentence.getText(), sentence.getLocation()); 9 | } 10 | 11 | @Override 12 | public void output() { 13 | // not called in this example 14 | } 15 | 16 | @Override 17 | public String getId() { 18 | return this.getClass().getName(); 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /speech01/src/main/java/io/s4/example/speech01/Speech.java: -------------------------------------------------------------------------------- 1 | package io.s4.example.speech01; 2 | 3 | public class Speech { 4 | private long id; 5 | private String location; 6 | private String speaker; 7 | private long time; 8 | 9 | public long getId() { 10 | return id; 11 | } 12 | 13 | public void setId(long id) { 14 | this.id = id; 15 | } 16 | 17 | public String getLocation() { 18 | return location; 19 | } 20 | 21 | public void setLocation(String location) { 22 | this.location = location; 23 | } 24 | 25 | public String getSpeaker() { 26 | return speaker; 27 | } 28 | 29 | public void setSpeaker(String speaker) { 30 | this.speaker = speaker; 31 | } 32 | 33 | public long getTime() { 34 | return time; 35 | } 36 | 37 | public void setTime(long time) { 38 | this.time = time; 39 | } 40 | 41 | public String toString() { 42 | StringBuffer sb = new StringBuffer(); 43 | sb.append("{id:") 44 | .append(id) 45 | .append(",location:") 46 | .append(location) 47 | .append(",speaker") 48 | .append(speaker) 49 | .append(",time") 50 | .append(time) 51 | .append("}"); 52 | 53 | return sb.toString(); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /speech01/src/main/resources/speech01_conf.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | RawSentence * 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /speech01_scala/README.md: -------------------------------------------------------------------------------- 1 | Speech01 in Scala 2 | ================= 3 | 4 | Introduction 5 | ------------ 6 | This is a rewrite of the (hello world-ish) [speech01](http://docs.s4.io/manual/getting_events_into_s4.html) application in the [Scala](http://www.scala-lang.org/) programming language. 7 | 8 | Requirements 9 | ------------ 10 | 11 | * Linux 12 | * Java 1.6 13 | * Maven 14 | * S4 Communication Layer 15 | * S4 Core 16 | 17 | Build Instructions 18 | ------------------ 19 | 20 | 1. First build and install the comm and core packages in your Maven repository. 21 | 22 | 2. Build and install using Maven 23 | 24 | mvn install assembly:assembly 25 | 26 | Run Instructions 27 | ------------------ 28 | 29 | Follow instructions for the [speech01](http://docs.s4.io/manual/getting_events_into_s4.html#building-and-running-the-speech01-example) example in Java. The only change is in the command for piping events into the application. Do the following: 30 | 31 | head -10 ${SOURCE_BASE}/examples/testinput/speeches.txt | \ 32 | ./generate_load.sh -x -r 2 -u ../s4_apps/speech01_scala/lib/\* - 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /speech01_scala/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | tar.gz 6 | 7 | true 8 | ${artifactId} 9 | 10 | 11 | false 12 | lib 13 | runtime 14 | false 15 | 16 | log4j:log4j 17 | 18 | 19 | 20 | 21 | 22 | ${project.basedir}/target 23 | lib 24 | 25 | *.jar 26 | 27 | 28 | 29 | ${project.basedir}/src/main/resources 30 | 31 | 32 | speech01_scala_conf.xml 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /speech01_scala/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | io.s4.examples 5 | speech01_scala 6 | jar 7 | 0.0.0.1 8 | speech01_scala 9 | http://maven.apache.org 10 | 11 | 12 | scala-tools.org 13 | Scala-tools Maven2 Repository 14 | http://scala-tools.org/repo-releases 15 | 16 | 17 | 18 | 19 | scala-tools.org 20 | Scala-tools Maven2 Repository 21 | http://scala-tools.org/repo-releases 22 | 23 | 24 | 25 | 26 | org.scala-lang 27 | scala-library 28 | 2.8.0 29 | 30 | 31 | log4j 32 | log4j 33 | 1.2.15 34 | 35 | 36 | javax.mail 37 | mail 38 | 39 | 40 | javax.jms 41 | jms 42 | 43 | 44 | com.sun.jdmk 45 | jmxtools 46 | 47 | 48 | com.sun.jmx 49 | jmxri 50 | 51 | 52 | 53 | 54 | io.s4 55 | s4_core 56 | 0.2.1.0 57 | provided 58 | 59 | 60 | commons-codec 61 | commons-codec 62 | 1.4 63 | 64 | 65 | commons-httpclient 66 | commons-httpclient 67 | 3.1 68 | 69 | 70 | 71 | ${artifactId}-${version} 72 | 73 | 74 | src/main/resources 75 | 76 | * 77 | 78 | 79 | 80 | 81 | 82 | maven-assembly-plugin 83 | 84 | 85 | assembly.xml 86 | 87 | 88 | 89 | 90 | org.scala-tools 91 | maven-scala-plugin 92 | 93 | 94 | scala-compile-first 95 | process-resources 96 | 97 | add-source 98 | compile 99 | 100 | 101 | 102 | scala-test-compile 103 | process-test-resources 104 | 105 | testCompile 106 | 107 | 108 | 109 | 110 | 111 | org.apache.maven.plugins 112 | maven-compiler-plugin 113 | 2.0.1 114 | 115 | 1.6 116 | 1.6 117 | 118 | 119 | 120 | compile 121 | 122 | compile 123 | 124 | 125 | 126 | 127 | 128 | 129 | maven-eclipse-plugin 130 | 131 | 132 | **/*.scala 133 | 134 | 135 | ch.epfl.lamp.sdt.core.scalabuilder 136 | 137 | 138 | 139 | ch.epfl.lamp.sdt.core.scalanature 140 | 141 | 142 | org.eclipse.jdt.launching.JRE_CONTAINER 143 | ch.epfl.lamp.sdt.launching.SCALA_CONTAINER 144 | 145 | 146 | 147 | 148 | 149 | org.codehaus.mojo 150 | build-helper-maven-plugin 151 | 152 | 153 | add-source 154 | generate-sources 155 | 156 | add-source 157 | 158 | 159 | 160 | src/main/scala 161 | 162 | 163 | 164 | 165 | add-test-source 166 | generate-sources 167 | 168 | add-test-source 169 | 170 | 171 | 172 | src/test/scala 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | -------------------------------------------------------------------------------- /speech01_scala/src/main/resources/speech01_scala_conf.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | RawSentence * 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /speech01_scala/src/main/scala/Events.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | 17 | package io.s4.example.speech01 18 | 19 | import scala.reflect.BeanProperty 20 | 21 | class Highlight { 22 | @BeanProperty var sentenceId: Long = _ 23 | @BeanProperty var time: Long = _ 24 | 25 | override def toString()= "{sentenceId:" + sentenceId + ",time" + time + "}" 26 | } 27 | 28 | class Sentence { 29 | @BeanProperty var id: Long = _ 30 | @BeanProperty var speechId: Long = _ 31 | @BeanProperty var location: String = _ 32 | @BeanProperty var text: String = _ 33 | @BeanProperty var time: Long = _ 34 | 35 | override def toString()= "{id:" + id + ",speechId:" + speechId + ",location:" + location + ",text:" + text + ",time:" + time + "}" 36 | } 37 | 38 | class Speech { 39 | @BeanProperty var id: Long = _ 40 | @BeanProperty var location: String = _ 41 | @BeanProperty var speaker: String = _ 42 | @BeanProperty var time: Long = _ 43 | 44 | override def toString()= "{id:" + id + ",location:" + location + ",speaker:" + speaker + ",time:" + time + "}" 45 | } 46 | -------------------------------------------------------------------------------- /speech01_scala/src/main/scala/SentenceReceiverPE.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | 17 | package io.s4.example.speech01 18 | 19 | import io.s4.processor.AbstractPE 20 | 21 | class SentenceReceiverPE extends AbstractPE { 22 | 23 | def processEvent(sentence: Sentence): Unit= println("Sentence is '" + sentence.text + "', location '" + sentence.location + "'") 24 | 25 | def output(): Unit= {} 26 | 27 | def getId(): String= return this.getClass().getName() 28 | } 29 | -------------------------------------------------------------------------------- /speech02/README.md: -------------------------------------------------------------------------------- 1 | Hello World Example 2 | =============== 3 | 4 | Introduction 5 | ------------ 6 | This sample application demonstrates joining and rerouting streams 7 | 8 | Requirements 9 | ------------ 10 | 11 | * Linux 12 | * Java 1.6 13 | * Maven 14 | * S4 Communication Layer 15 | * S4 Core 16 | 17 | Build Instructions 18 | ------------------ 19 | 20 | 1. First build and install the comm and core packages in your Maven repository. 21 | 22 | 2. Build the speech01 example application 23 | 24 | 3. Build and install using Maven 25 | 26 | mvn assembly:assembly 27 | 28 | -------------------------------------------------------------------------------- /speech02/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | tar.gz 6 | 7 | true 8 | ${artifactId} 9 | 10 | 11 | false 12 | lib 13 | runtime 14 | false 15 | 16 | log4j:log4j 17 | 18 | 19 | 20 | 21 | 22 | ${project.basedir}/target 23 | lib 24 | 25 | *.jar 26 | 27 | 28 | 29 | ${project.basedir}/src/main/resources 30 | 31 | 32 | speech02_conf.xml 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /speech02/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | io.s4.examples 5 | speech02 6 | jar 7 | 0.0.0.2 8 | speech02 9 | http://maven.apache.org 10 | 11 | 12 | log4j 13 | log4j 14 | 1.2.15 15 | 16 | 17 | javax.mail 18 | mail 19 | 20 | 21 | javax.jms 22 | jms 23 | 24 | 25 | com.sun.jdmk 26 | jmxtools 27 | 28 | 29 | com.sun.jmx 30 | jmxri 31 | 32 | 33 | 34 | 35 | io.s4 36 | s4_core 37 | 0.3.0.0 38 | provided 39 | 40 | 41 | io.s4.examples 42 | speech01 43 | 0.0.0.1 44 | 45 | 46 | 47 | ${artifactId}-${version} 48 | 49 | 50 | src/main/resources 51 | 52 | * 53 | 54 | 55 | 56 | 57 | 58 | maven-compiler-plugin 59 | 2.0.1 60 | 61 | 1.6 62 | 1.6 63 | 64 | 65 | 66 | maven-assembly-plugin 67 | 68 | 69 | assembly.xml 70 | 71 | 72 | 73 | 74 | org.codehaus.mojo 75 | findbugs-maven-plugin 76 | 2.0 77 | 78 | true 79 | true 80 | High 81 | 82 | 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /speech02/src/main/resources/speech02_conf.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | SentenceJoined * 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | RawSentence * 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | RawSpeech * 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | Sentence speechId 39 | Speech id 40 | 41 | 42 | 43 | 44 | Sentence * 45 | Speech location 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | Sentence 58 | SentenceJoined 59 | 60 | 61 | 62 | 63 | speechId 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | Speech 74 | 75 | 76 | 77 | 78 | id 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | SentenceJoined 111 | Sentence 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | -------------------------------------------------------------------------------- /testinput/pe-query: -------------------------------------------------------------------------------- 1 | {"target":["16000000"],"query":["$outputClassName"],"rinfo":{"id":0,"stream":"@client"}} 2 | {"target":["*"],"query":["$outputClassName"],"rinfo":{"id":1,"stream":"@client"}} 3 | -------------------------------------------------------------------------------- /testinput/proto-query: -------------------------------------------------------------------------------- 1 | {"query":["count"],"rinfo":{"id":123}} 2 | -------------------------------------------------------------------------------- /testinput/speech.in: -------------------------------------------------------------------------------- 1 | {"_index":0,"id":12000000,"location":"gettysburg, pa, us","speaker":"abraham lincoln","time":1242799200000} 2 | {"_index":0,"id":14000000,"location":"washington, dc, us","speaker":"franklin delano roosevelt","time":1242799500000} 3 | {"_index":0,"id":16000000,"location":"washington, dc, us","speaker":"franklin delano roosevelt","time":1242799800000} 4 | {"_index":0,"id":18000000,"location":"houston, tx","speaker":"john f kennedy","time":1242800100000} 5 | {"_index":0,"id":20000000,"location":"berlin, de","speaker":"john f kennedy","time":1242800400000} 6 | {"_index":0,"id":22000000,"location":"washington, dc, us","speaker":"john f kennedy","time":1242800700000} 7 | {"_index":0,"id":24000000,"location":"cleveland, oh, us","speaker":"malcolm x","time":1242801000000} 8 | {"_index":0,"id":26000000,"location":"washington, dc, us","speaker":"martin luther king","time":1242801300000} 9 | {"_index":0,"id":28000000,"location":"los angeles, ca, us","speaker":"richard m nixon","time":1242801600000} 10 | {"_index":0,"id":30000000,"location":"washington, dc, us","speaker":"richard m nixon","time":1242801900000} 11 | {"_index":0,"id":32000000,"location":"washington, dc, us","speaker":"ronald reagan","time":1242802200000} 12 | -------------------------------------------------------------------------------- /twittertopiccount/NOTICE.txt: -------------------------------------------------------------------------------- 1 | ======================================================================= 2 | NOTICE file for use with, and corresponding to Section 4 of, 3 | the Apache License, Version 2.0, in this case for the S4 project. 4 | ========================================================================= 5 | 6 | This product includes software developed by 7 | Yahoo! Inc. (www.yahoo.com) 8 | Copyright (c) 2010 Yahoo! Inc. All rights reserved. 9 | 10 | This product includes software developed by 11 | The Apache Software Foundation (http://www.apache.org/). 12 | -------------------------------------------------------------------------------- /twittertopiccount/README.md: -------------------------------------------------------------------------------- 1 | Twitter Hash Tag Count Example 2 | =============== 3 | 4 | Introduction 5 | ------------ 6 | This sample application listens to the twitter garden hose and keeps track of the top 10 hash tags. 7 | 8 | Requirements 9 | ------------ 10 | 11 | * Linux 12 | * Java 1.6 13 | * Maven 14 | * S4 Communication Layer 15 | * S4 Core 16 | 17 | Build Instructions 18 | ------------------ 19 | 20 | 1. First build and install the comm and core packages in your Maven repository. 21 | 22 | 2. Build and install using Maven 23 | 24 | mvn assembly:assembly install 25 | 26 | -------------------------------------------------------------------------------- /twittertopiccount/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | tar.gz 6 | 7 | true 8 | ${artifactId} 9 | 10 | 11 | false 12 | lib 13 | runtime 14 | false 15 | 16 | log4j:log4j 17 | 18 | 19 | 20 | 21 | 22 | ${project.basedir}/NOTICE.txt 23 | 24 | NOTICE.txt 25 | 26 | 27 | 28 | 29 | ${project.basedir}/target 30 | lib 31 | 32 | *.jar 33 | 34 | 35 | 36 | ${project.basedir}/src/main/resources 37 | 38 | 39 | twittertopiccount_conf.xml 40 | adapter_conf.xml 41 | 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /twittertopiccount/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | io.s4.examples.twittertopiccount 5 | twittertopiccount 6 | jar 7 | 0.0.0.3 8 | twittertopiccount 9 | http://maven.apache.org 10 | 11 | 12 | log4j 13 | log4j 14 | 1.2.15 15 | 16 | 17 | javax.mail 18 | mail 19 | 20 | 21 | javax.jms 22 | jms 23 | 24 | 25 | com.sun.jdmk 26 | jmxtools 27 | 28 | 29 | com.sun.jmx 30 | jmxri 31 | 32 | 33 | 34 | 35 | io.s4 36 | s4_core 37 | 0.3.0.0 38 | provided 39 | 40 | 41 | commons-codec 42 | commons-codec 43 | 1.4 44 | 45 | 46 | commons-httpclient 47 | commons-httpclient 48 | 3.1 49 | 50 | 51 | 52 | ${artifactId}-${version} 53 | 54 | 55 | src/main/resources 56 | 57 | * 58 | 59 | 60 | 61 | 62 | 63 | maven-compiler-plugin 64 | 2.0.1 65 | 66 | 1.6 67 | 1.6 68 | 69 | 70 | 71 | maven-assembly-plugin 72 | 73 | 74 | assembly.xml 75 | 76 | 77 | 78 | 79 | org.codehaus.mojo 80 | findbugs-maven-plugin 81 | 2.0 82 | 83 | true 84 | true 85 | High 86 | 87 | 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /twittertopiccount/src/main/java/io/s4/example/twittertopiccount/DirectToFilePersister.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.example.twittertopiccount; 17 | 18 | import java.io.FileWriter; 19 | import java.io.IOException; 20 | import java.util.HashMap; 21 | import java.util.HashSet; 22 | import java.util.Map; 23 | import java.util.Set; 24 | 25 | import org.apache.log4j.Logger; 26 | 27 | import io.s4.persist.Persister; 28 | 29 | public class DirectToFilePersister implements Persister { 30 | private String outputFilename; 31 | private int persistCount; 32 | 33 | public void setOutputFilename(String outputFilename) { 34 | this.outputFilename = outputFilename; 35 | } 36 | 37 | @Override 38 | public int cleanOutGarbage() throws InterruptedException { 39 | return 0; 40 | } 41 | 42 | @Override 43 | public Object get(String arg0) throws InterruptedException { 44 | return null; 45 | } 46 | 47 | @Override 48 | public Map getBulk(String[] arg0) 49 | throws InterruptedException { 50 | return new HashMap(); 51 | } 52 | 53 | @Override 54 | public Map getBulkObjects(String[] arg0) 55 | throws InterruptedException { 56 | return new HashMap(); 57 | } 58 | 59 | @Override 60 | public int getCacheEntryCount() { 61 | return 1; 62 | } 63 | 64 | @Override 65 | public Object getObject(String arg0) throws InterruptedException { 66 | return null; 67 | } 68 | 69 | @Override 70 | public int getPersistCount() { 71 | return persistCount; 72 | } 73 | 74 | @Override 75 | public int getQueueSize() { 76 | return 0; 77 | } 78 | 79 | @Override 80 | public Set keySet() { 81 | return new HashSet(); 82 | } 83 | 84 | @Override 85 | public void remove(String arg0) throws InterruptedException { 86 | 87 | } 88 | 89 | @Override 90 | public void set(String key, Object value, int persistTime) 91 | throws InterruptedException { 92 | 93 | FileWriter fw = null; 94 | try { 95 | fw = new FileWriter(outputFilename); 96 | fw.write(String.valueOf(value)); 97 | } catch (IOException e) { 98 | // TODO Auto-generated catch block 99 | Logger.getLogger("s4").error(e); 100 | } finally { 101 | if (fw != null) { 102 | try { 103 | fw.close(); 104 | } catch (Exception e) { 105 | } 106 | } 107 | } 108 | } 109 | 110 | @Override 111 | public void setAsynch(String key, Object value, int persistTime) { 112 | try { 113 | set(key, value, persistTime); 114 | } catch (InterruptedException ie) { 115 | } 116 | } 117 | 118 | } 119 | -------------------------------------------------------------------------------- /twittertopiccount/src/main/java/io/s4/example/twittertopiccount/Status.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.example.twittertopiccount; 17 | 18 | public class Status { 19 | private long id; 20 | private long inReplyToStatusId; 21 | private String text; 22 | private boolean truncated; 23 | private String source; 24 | private String inReplyToScreenName; 25 | private boolean favorited; 26 | private User user; 27 | private long inReplyToUserId; 28 | private String createdAt; 29 | 30 | public long getId() { 31 | return id; 32 | } 33 | 34 | public void setId(long id) { 35 | this.id = id; 36 | } 37 | 38 | public long getInReplyToStatusId() { 39 | return inReplyToStatusId; 40 | } 41 | 42 | public void setInReplyToStatusId(long inReplyToStatusId) { 43 | this.inReplyToStatusId = inReplyToStatusId; 44 | } 45 | 46 | public String getText() { 47 | return text; 48 | } 49 | 50 | public void setText(String text) { 51 | this.text = text; 52 | } 53 | 54 | public boolean isTruncated() { 55 | return truncated; 56 | } 57 | 58 | public void setTruncated(boolean truncated) { 59 | this.truncated = truncated; 60 | } 61 | 62 | public String getSource() { 63 | return source; 64 | } 65 | 66 | public void setSource(String source) { 67 | this.source = source; 68 | } 69 | 70 | public String getInReplyToScreenName() { 71 | return inReplyToScreenName; 72 | } 73 | 74 | public void setInReplyToScreenName(String inReplyToScreenName) { 75 | this.inReplyToScreenName = inReplyToScreenName; 76 | } 77 | 78 | public boolean isFavorited() { 79 | return favorited; 80 | } 81 | 82 | public void setFavorited(boolean favorited) { 83 | this.favorited = favorited; 84 | } 85 | 86 | public User getUser() { 87 | return user; 88 | } 89 | 90 | public void setUser(User user) { 91 | this.user = user; 92 | } 93 | 94 | public long getInReplyToUserId() { 95 | return inReplyToUserId; 96 | } 97 | 98 | public void setInReplyToUserId(long inReplyToUserId) { 99 | this.inReplyToUserId = inReplyToUserId; 100 | } 101 | 102 | public String getCreatedAt() { 103 | return createdAt; 104 | } 105 | 106 | public void setCreatedAt(String createdAt) { 107 | this.createdAt = createdAt; 108 | } 109 | 110 | public String toString() { 111 | StringBuffer sb = new StringBuffer(); 112 | sb.append("{") 113 | .append("id:") 114 | .append(id) 115 | .append(",") 116 | .append("inReplyToStatusId:") 117 | .append(inReplyToStatusId) 118 | .append(",") 119 | .append("text:") 120 | .append(text) 121 | .append(",") 122 | .append("truncated:") 123 | .append(truncated) 124 | .append(",") 125 | .append("source:") 126 | .append(source) 127 | .append(",") 128 | .append("inReplyToScreenName:") 129 | .append(inReplyToScreenName) 130 | .append(",") 131 | .append("favorited:") 132 | .append(favorited) 133 | .append(",") 134 | .append("user:") 135 | .append(user) 136 | .append(",") 137 | .append("inReplyToUserId:") 138 | .append(inReplyToUserId) 139 | .append(",") 140 | .append("createdAt:") 141 | .append(createdAt) 142 | .append("}"); 143 | 144 | return sb.toString(); 145 | } 146 | 147 | public Object clone() { 148 | try { 149 | return super.clone(); 150 | } catch (CloneNotSupportedException cnse) { 151 | throw new RuntimeException(cnse); 152 | } 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /twittertopiccount/src/main/java/io/s4/example/twittertopiccount/TopNTopicPE.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.example.twittertopiccount; 17 | 18 | import io.s4.persist.Persister; 19 | import io.s4.processor.AbstractPE; 20 | 21 | import java.util.ArrayList; 22 | import java.util.Collections; 23 | import java.util.List; 24 | import java.util.Map; 25 | import java.util.concurrent.ConcurrentHashMap; 26 | 27 | import org.apache.log4j.Logger; 28 | import org.json.JSONArray; 29 | import org.json.JSONObject; 30 | 31 | import com.google.gson.Gson; 32 | 33 | public class TopNTopicPE extends AbstractPE { 34 | private String id; 35 | private Persister persister; 36 | private int entryCount = 10; 37 | private Map topicMap = new ConcurrentHashMap(); 38 | private int persistTime; 39 | private String persistKey = "myapp:topNTopics"; 40 | 41 | public void setId(String id) { 42 | this.id = id; 43 | } 44 | 45 | public Persister getPersister() { 46 | return persister; 47 | } 48 | 49 | public void setPersister(Persister persister) { 50 | this.persister = persister; 51 | } 52 | 53 | public int getEntryCount() { 54 | return entryCount; 55 | } 56 | 57 | public void setEntryCount(int entryCount) { 58 | this.entryCount = entryCount; 59 | } 60 | 61 | public int getPersistTime() { 62 | return persistTime; 63 | } 64 | 65 | public void setPersistTime(int persistTime) { 66 | this.persistTime = persistTime; 67 | } 68 | 69 | public String getPersistKey() { 70 | return persistKey; 71 | } 72 | 73 | public void setPersistKey(String persistKey) { 74 | this.persistKey = persistKey; 75 | } 76 | 77 | public void processEvent(TopicSeen topicSeen) { 78 | topicMap.put(topicSeen.getTopic(), topicSeen.getCount()); 79 | } 80 | 81 | public ArrayList getTopTopics() { 82 | if (entryCount < 1) 83 | return null; 84 | 85 | ArrayList sortedList = new ArrayList(); 86 | 87 | for (String key : topicMap.keySet()) { 88 | sortedList.add(new TopNEntry(key, topicMap.get(key))); 89 | } 90 | 91 | Collections.sort(sortedList); 92 | 93 | // truncate: Yuck!! 94 | // unfortunately, Kryo cannot deserialize RandomAccessSubList 95 | // if we use ArrayList.subList(...) 96 | while (sortedList.size() > entryCount) 97 | sortedList.remove(sortedList.size() - 1); 98 | 99 | return sortedList; 100 | } 101 | 102 | @Override 103 | public void output() { 104 | List sortedList = new ArrayList(); 105 | 106 | for (String key : topicMap.keySet()) { 107 | sortedList.add(new TopNEntry(key, topicMap.get(key))); 108 | } 109 | 110 | Collections.sort(sortedList); 111 | 112 | try { 113 | JSONObject message = new JSONObject(); 114 | JSONArray jsonTopN = new JSONArray(); 115 | 116 | for (int i = 0; i < entryCount; i++) { 117 | if (i == sortedList.size()) { 118 | break; 119 | } 120 | TopNEntry tne = sortedList.get(i); 121 | JSONObject jsonEntry = new JSONObject(); 122 | jsonEntry.put("topic", tne.getTopic()); 123 | jsonEntry.put("count", tne.getCount()); 124 | jsonTopN.put(jsonEntry); 125 | } 126 | message.put("topN", jsonTopN); 127 | persister.set(persistKey, message.toString()+"\n", persistTime); 128 | } catch (Exception e) { 129 | Logger.getLogger("s4").error(e); 130 | } 131 | } 132 | 133 | @Override 134 | public String getId() { 135 | return this.id; 136 | } 137 | 138 | public static class TopNEntry implements Comparable { 139 | public TopNEntry(String topic, int count) { 140 | this.topic = topic; 141 | this.count = count; 142 | } 143 | 144 | public TopNEntry() {} 145 | 146 | String topic = null; 147 | int count = 0; 148 | 149 | public String getTopic() { 150 | return topic; 151 | } 152 | 153 | public void setTopic(String topic) { 154 | this.topic = topic; 155 | } 156 | 157 | public int getCount() { 158 | return count; 159 | } 160 | 161 | public void setCount(int count) { 162 | this.count = count; 163 | } 164 | 165 | public int compareTo(TopNEntry topNEntry) { 166 | if (topNEntry.getCount() < this.count) { 167 | return -1; 168 | } else if (topNEntry.getCount() > this.count) { 169 | return 1; 170 | } 171 | return 0; 172 | } 173 | 174 | public String toString() { 175 | return "topic:" + topic + " count:" + count; 176 | } 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /twittertopiccount/src/main/java/io/s4/example/twittertopiccount/TopicCountAndReportPE.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.example.twittertopiccount; 17 | 18 | import io.s4.dispatcher.EventDispatcher; 19 | import io.s4.processor.AbstractPE; 20 | 21 | public class TopicCountAndReportPE extends AbstractPE { 22 | private String id; 23 | private EventDispatcher dispatcher; 24 | private String outputStreamName; 25 | private int threshold; 26 | private int count; 27 | 28 | public void setId(String id) { 29 | this.id = id; 30 | } 31 | 32 | public EventDispatcher getDispatcher() { 33 | return dispatcher; 34 | } 35 | 36 | public void setDispatcher(EventDispatcher dispatcher) { 37 | this.dispatcher = dispatcher; 38 | } 39 | 40 | public String getOutputStreamName() { 41 | return outputStreamName; 42 | } 43 | 44 | public void setOutputStreamName(String outputStreamName) { 45 | this.outputStreamName = outputStreamName; 46 | } 47 | 48 | public int getThreshold() { 49 | return threshold; 50 | } 51 | 52 | public void setThreshold(int threshold) { 53 | this.threshold = threshold; 54 | } 55 | 56 | public void processEvent(TopicSeen topicSeen) { 57 | count += topicSeen.getCount(); 58 | } 59 | 60 | @Override 61 | public void output() { 62 | if (count < threshold) { 63 | return; 64 | } 65 | TopicSeen topicSeen = new TopicSeen((String) this.getKeyValue().get(0), 66 | count); 67 | topicSeen.setReportKey("1"); 68 | dispatcher.dispatchEvent(outputStreamName, topicSeen); 69 | } 70 | 71 | @Override 72 | public String getId() { 73 | return this.id; 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /twittertopiccount/src/main/java/io/s4/example/twittertopiccount/TopicExtractorPE.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.example.twittertopiccount; 17 | 18 | import io.s4.dispatcher.Dispatcher; 19 | import io.s4.dispatcher.EventDispatcher; 20 | import io.s4.processor.AbstractPE; 21 | 22 | public class TopicExtractorPE extends AbstractPE { 23 | private String id; 24 | private EventDispatcher dispatcher; 25 | private String outputStreamName; 26 | 27 | public EventDispatcher getDispatcher() { 28 | return dispatcher; 29 | } 30 | 31 | public void setDispatcher(EventDispatcher dispatcher) { 32 | this.dispatcher = dispatcher; 33 | } 34 | 35 | public String getOutputStreamName() { 36 | return outputStreamName; 37 | } 38 | 39 | public void setOutputStreamName(String outputStreamName) { 40 | this.outputStreamName = outputStreamName; 41 | } 42 | 43 | public void setId(String id) { 44 | this.id = id; 45 | } 46 | 47 | public void processEvent(Status status) { 48 | String text = status.getText(); 49 | if (text == null) { 50 | return; 51 | } 52 | 53 | int textLength = text.length(); 54 | int index = 0; 55 | int prevIndex = 0; 56 | while ((index = text.indexOf('#', prevIndex)) != -1) { 57 | prevIndex = index + 1; 58 | if (prevIndex == textLength) { // if hash is the last character 59 | break; // get out 60 | } 61 | StringBuffer sb = new StringBuffer(); 62 | for (int i = index + 1; i < textLength; i++) { 63 | char ch = text.charAt(i); 64 | if (!Character.isLetterOrDigit(ch)) { 65 | break; 66 | } 67 | sb.append(ch); 68 | } 69 | 70 | if (sb.length() == 0) { 71 | continue; 72 | } 73 | 74 | TopicSeen topicSeen = new TopicSeen(sb.toString().toLowerCase(), 1); 75 | dispatcher.dispatchEvent(outputStreamName, topicSeen); 76 | } 77 | } 78 | 79 | @Override 80 | public void output() { 81 | // TODO Auto-generated method stub 82 | 83 | } 84 | 85 | @Override 86 | public String getId() { 87 | return this.id; 88 | } 89 | 90 | static class DummyDispatcher extends Dispatcher { 91 | public void dispatchEvent(String streamName, Object event) { 92 | System.out.println(event); 93 | } 94 | } 95 | 96 | public static void main(String args[]) { 97 | TopicExtractorPE te = new TopicExtractorPE(); 98 | te.setDispatcher(new DummyDispatcher()); 99 | te.setOutputStreamName("test"); 100 | 101 | Status status = new Status(); 102 | status.setText("Hey this is a test"); 103 | te.processEvent(status); 104 | 105 | status.setText("This is an edge test #"); 106 | te.processEvent(status); 107 | 108 | status.setText("#GLOB this is a test"); 109 | te.processEvent(status); 110 | 111 | status.setText("Hey there #FLOB, this is a test #GLOB"); 112 | te.processEvent(status); 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /twittertopiccount/src/main/java/io/s4/example/twittertopiccount/TopicSeen.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.example.twittertopiccount; 17 | 18 | public class TopicSeen { 19 | private String topic; 20 | private int count; 21 | private String reportKey; 22 | 23 | public TopicSeen() { 24 | 25 | } 26 | 27 | public TopicSeen(String topic, int count) { 28 | this.topic = topic; 29 | this.count = count; 30 | } 31 | 32 | public String getTopic() { 33 | return topic; 34 | } 35 | 36 | public void setTopic(String topic) { 37 | this.topic = topic; 38 | } 39 | 40 | public int getCount() { 41 | return count; 42 | } 43 | 44 | public void setCount(int count) { 45 | this.count = count; 46 | } 47 | 48 | public String getReportKey() { 49 | return reportKey; 50 | } 51 | 52 | public void setReportKey(String reportKey) { 53 | this.reportKey = reportKey; 54 | } 55 | 56 | public String toString() { 57 | return "{topic:" + topic + "}"; 58 | } 59 | 60 | public Object clone() { 61 | try { 62 | return super.clone(); 63 | } catch (CloneNotSupportedException cnse) { 64 | throw new RuntimeException(cnse); 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /twittertopiccount/src/main/java/io/s4/example/twittertopiccount/TwitterFeedListener.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.example.twittertopiccount; 17 | 18 | import java.io.BufferedReader; 19 | import java.io.InputStream; 20 | import java.io.InputStreamReader; 21 | import java.net.URL; 22 | import java.net.URLConnection; 23 | import java.util.HashSet; 24 | import java.util.Set; 25 | import java.util.concurrent.LinkedBlockingQueue; 26 | 27 | import org.apache.log4j.Logger; 28 | import org.json.JSONObject; 29 | import org.apache.commons.codec.binary.Base64; 30 | import org.apache.commons.httpclient.util.EncodingUtil; 31 | 32 | import io.s4.collector.EventWrapper; 33 | import io.s4.listener.EventHandler; 34 | import io.s4.listener.EventProducer; 35 | 36 | public class TwitterFeedListener implements EventProducer, Runnable { 37 | private String userid; 38 | private String password; 39 | private String urlString; 40 | private long maxBackoffTime = 30 * 1000; // 5 seconds 41 | private long messageCount = 0; 42 | private long blankCount = 0; 43 | private String streamName; 44 | 45 | private LinkedBlockingQueue messageQueue = new LinkedBlockingQueue(); 46 | private Set handlers = new HashSet(); 47 | 48 | public void setUserid(String userid) { 49 | this.userid = userid; 50 | } 51 | 52 | public void setPassword(String password) { 53 | this.password = password; 54 | } 55 | 56 | public void setUrlString(String urlString) { 57 | this.urlString = urlString; 58 | } 59 | 60 | public void setMaxBackoffTime(long maxBackoffTime) { 61 | this.maxBackoffTime = maxBackoffTime; 62 | } 63 | 64 | public void setStreamName(String streamName) { 65 | this.streamName = streamName; 66 | } 67 | 68 | public void init() { 69 | for (int i = 0; i < 12; i++) { 70 | Dequeuer dequeuer = new Dequeuer(i); 71 | Thread t = new Thread(dequeuer); 72 | t.start(); 73 | } 74 | (new Thread(this)).start(); 75 | } 76 | 77 | public void run() { 78 | long backoffTime = 1000; 79 | while (!Thread.interrupted()) { 80 | try { 81 | connectAndRead(); 82 | } catch (Exception e) { 83 | Logger.getLogger("s4").error("Exception reading feed", e); 84 | try { 85 | Thread.sleep(backoffTime); 86 | } catch (InterruptedException ie) { 87 | Thread.currentThread().interrupt(); 88 | } 89 | backoffTime = backoffTime * 2; 90 | if (backoffTime > maxBackoffTime) { 91 | backoffTime = maxBackoffTime; 92 | } 93 | } 94 | } 95 | } 96 | 97 | public void connectAndRead() throws Exception { 98 | URL url = new URL(urlString); 99 | 100 | URLConnection connection = url.openConnection(); 101 | String userPassword = userid + ":" + password; 102 | String encoded = EncodingUtil.getAsciiString(Base64.encodeBase64(EncodingUtil.getAsciiBytes(userPassword))); 103 | connection.setRequestProperty("Authorization", "Basic " + encoded); 104 | connection.connect(); 105 | 106 | InputStream is = connection.getInputStream(); 107 | InputStreamReader isr = new InputStreamReader(is); 108 | BufferedReader br = new BufferedReader(isr); 109 | 110 | String inputLine = null; 111 | while ((inputLine = br.readLine()) != null) { 112 | if (inputLine.trim().length() == 0) { 113 | blankCount++; 114 | continue; 115 | } 116 | messageCount++; 117 | messageQueue.add(inputLine); 118 | } 119 | } 120 | 121 | class Dequeuer implements Runnable { 122 | private int id; 123 | 124 | public Dequeuer(int id) { 125 | this.id = id; 126 | } 127 | 128 | public void run() { 129 | while (!Thread.interrupted()) { 130 | try { 131 | String message = messageQueue.take(); 132 | JSONObject jsonObject = new JSONObject(message); 133 | 134 | // ignore delete records for now 135 | if (jsonObject.has("delete")) { 136 | continue; 137 | } 138 | 139 | Status status = getStatus(jsonObject); 140 | 141 | EventWrapper ew = new EventWrapper(streamName, status, null); 142 | for (io.s4.listener.EventHandler handler : handlers) { 143 | try { 144 | handler.processEvent(ew); 145 | } catch (Exception e) { 146 | Logger.getLogger("s4") 147 | .error("Exception in raw event handler", e); 148 | } 149 | } 150 | } catch (InterruptedException ie) { 151 | Thread.currentThread().interrupt(); 152 | } catch (Exception e) { 153 | Logger.getLogger("s4") 154 | .error("Exception processing message", e); 155 | } 156 | } 157 | } 158 | 159 | public Status getStatus(JSONObject jsonObject) { 160 | try { 161 | if (jsonObject == null || jsonObject.equals(JSONObject.NULL)) { 162 | return null; 163 | } 164 | 165 | Status status = new Status(); 166 | 167 | status.setUser(getUser((JSONObject) jsonObject.opt("user"))); 168 | 169 | Object value = jsonObject.opt("id"); 170 | if (value != null && !value.equals(JSONObject.NULL)) { 171 | status.setId(((Number) value).longValue()); 172 | } 173 | 174 | value = jsonObject.opt("in_reply_to_status_id"); 175 | if (value != null && !value.equals(JSONObject.NULL)) { 176 | status.setInReplyToStatusId(((Number) value).longValue()); 177 | } 178 | 179 | value = jsonObject.opt("text"); 180 | if (value != null && !value.equals(JSONObject.NULL)) { 181 | status.setText((String) value); 182 | } 183 | 184 | value = jsonObject.opt("truncated"); 185 | if (value != null && !value.equals(JSONObject.NULL)) { 186 | status.setTruncated((Boolean) value); 187 | } 188 | 189 | value = jsonObject.opt("source"); 190 | if (value != null && !value.equals(JSONObject.NULL)) { 191 | status.setSource((String) value); 192 | } 193 | 194 | value = jsonObject.opt("in_reply_to_screen_name"); 195 | if (value != null && !value.equals(JSONObject.NULL)) { 196 | status.setInReplyToScreenName((String) value); 197 | } 198 | 199 | value = jsonObject.opt("favorited"); 200 | if (value != null && !value.equals(JSONObject.NULL)) { 201 | status.setFavorited((Boolean) value); 202 | } 203 | 204 | value = jsonObject.opt("in_reply_to_user_id"); 205 | if (value != null && !value.equals(JSONObject.NULL)) { 206 | status.setInReplyToUserId(((Number) value).longValue()); 207 | } 208 | 209 | value = jsonObject.opt("created_at"); 210 | if (value != null && !value.equals(JSONObject.NULL)) { 211 | status.setCreatedAt((String) value); 212 | } 213 | 214 | return status; 215 | } catch (Exception e) { 216 | Logger.getLogger("s4").error(e); 217 | } 218 | 219 | return null; 220 | } 221 | 222 | public User getUser(JSONObject jsonObject) { 223 | try { 224 | if (jsonObject == null || jsonObject.equals(JSONObject.NULL)) { 225 | return null; 226 | } 227 | 228 | User user = new User(); 229 | 230 | Object value = jsonObject.opt("id"); 231 | if (value != null && !value.equals(JSONObject.NULL)) { 232 | user.setId(((Number) value).longValue()); 233 | } 234 | 235 | value = jsonObject.opt("screen_name"); 236 | if (value != null && !value.equals(JSONObject.NULL)) { 237 | user.setScreenName((String) value); 238 | } 239 | 240 | value = jsonObject.opt("name"); 241 | if (value != null && !value.equals(JSONObject.NULL)) { 242 | user.setName((String) value); 243 | } 244 | 245 | value = jsonObject.opt("url"); 246 | if (value != null && !value.equals(JSONObject.NULL)) { 247 | user.setUrl((String) value); 248 | } 249 | 250 | value = jsonObject.opt("followers_count"); 251 | if (value != null && !value.equals(JSONObject.NULL)) { 252 | user.setFollowersCount(((Number) value).intValue()); 253 | } 254 | 255 | value = jsonObject.opt("lang"); 256 | if (value != null && !value.equals(JSONObject.NULL)) { 257 | user.setLang((String) value); 258 | } 259 | 260 | value = jsonObject.opt("verified"); 261 | if (value != null && !value.equals(JSONObject.NULL)) { 262 | user.setVerified((Boolean) value); 263 | } 264 | 265 | value = jsonObject.opt("profile_image_url"); 266 | if (value != null && !value.equals(JSONObject.NULL)) { 267 | user.setProfileImageUrl((String) value); 268 | } 269 | 270 | value = jsonObject.opt("friends_count"); 271 | if (value != null && !value.equals(JSONObject.NULL)) { 272 | user.setFriendsCount(((Number) value).intValue()); 273 | } 274 | 275 | value = jsonObject.opt("description"); 276 | if (value != null && !value.equals(JSONObject.NULL)) { 277 | user.setDescription((String) value); 278 | } 279 | 280 | value = jsonObject.opt("favourites_Count"); 281 | if (value != null && !value.equals(JSONObject.NULL)) { 282 | user.setFavouritesCount(((Number) value).intValue()); 283 | } 284 | 285 | value = jsonObject.opt("geo_enabled"); 286 | if (value != null && !value.equals(JSONObject.NULL)) { 287 | user.setGeoEnabled((Boolean) value); 288 | } 289 | 290 | value = jsonObject.opt("listed_count"); 291 | if (value != null && !value.equals(JSONObject.NULL)) { 292 | user.setListedCount(((Number) value).intValue()); 293 | } 294 | 295 | value = jsonObject.opt("profile_background_image_url"); 296 | if (value != null && !value.equals(JSONObject.NULL)) { 297 | user.setProfileBackgroundImageUrl((String) value); 298 | } 299 | 300 | value = jsonObject.opt("protected_user"); 301 | if (value != null && !value.equals(JSONObject.NULL)) { 302 | user.setProtectedUser((Boolean) value); 303 | } 304 | 305 | value = jsonObject.opt("location"); 306 | if (value != null && !value.equals(JSONObject.NULL)) { 307 | user.setLocation((String) value); 308 | } 309 | 310 | value = jsonObject.opt("statuses_count"); 311 | if (value != null && !value.equals(JSONObject.NULL)) { 312 | user.setStatusesCount(((Number) value).longValue()); 313 | } 314 | 315 | value = jsonObject.opt("time_zone"); 316 | if (value != null && !value.equals(JSONObject.NULL)) { 317 | user.setTimeZone((String) value); 318 | } 319 | 320 | value = jsonObject.opt("contributors_enabled"); 321 | if (value != null && !value.equals(JSONObject.NULL)) { 322 | user.setContributorsEnabled((Boolean) value); 323 | } 324 | 325 | value = jsonObject.opt("utc_offset"); 326 | if (value != null && !value.equals(JSONObject.NULL)) { 327 | user.setUtcOffset(((Number) value).intValue()); 328 | } 329 | 330 | value = jsonObject.opt("created_at"); 331 | if (value != null && !value.equals(JSONObject.NULL)) { 332 | user.setCreatedAt((String) value); 333 | } 334 | 335 | return user; 336 | } catch (Exception e) { 337 | Logger.getLogger("s4").error(e); 338 | } 339 | 340 | return null; 341 | } 342 | } 343 | 344 | @Override 345 | public void addHandler(EventHandler handler) { 346 | handlers.add(handler); 347 | 348 | } 349 | 350 | @Override 351 | public boolean removeHandler(EventHandler handler) { 352 | return handlers.remove(handler); 353 | } 354 | 355 | } 356 | -------------------------------------------------------------------------------- /twittertopiccount/src/main/java/io/s4/example/twittertopiccount/User.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.example.twittertopiccount; 17 | 18 | public class User { 19 | private long id; 20 | private String screenName; 21 | private String name; 22 | private String url; 23 | private int followersCount; 24 | private String lang; 25 | private boolean verified; 26 | private String profileImageUrl; 27 | private int friendsCount; 28 | private String description; 29 | private int favouritesCount; 30 | private boolean geoEnabled; 31 | private int listedCount; 32 | private String profileBackgroundImageUrl; 33 | private boolean protectedUser; 34 | private String location; 35 | private long statusesCount; 36 | private String timeZone; 37 | private boolean contributorsEnabled; 38 | private int utcOffset; 39 | private String createdAt; 40 | 41 | public long getId() { 42 | return id; 43 | } 44 | 45 | public void setId(long id) { 46 | this.id = id; 47 | } 48 | 49 | public String getScreenName() { 50 | return screenName; 51 | } 52 | 53 | public void setScreenName(String screenName) { 54 | this.screenName = screenName; 55 | } 56 | 57 | public String getName() { 58 | return name; 59 | } 60 | 61 | public void setName(String name) { 62 | this.name = name; 63 | } 64 | 65 | public String getUrl() { 66 | return url; 67 | } 68 | 69 | public void setUrl(String url) { 70 | this.url = url; 71 | } 72 | 73 | public int getFollowersCount() { 74 | return followersCount; 75 | } 76 | 77 | public void setFollowersCount(int followersCount) { 78 | this.followersCount = followersCount; 79 | } 80 | 81 | public String getLang() { 82 | return lang; 83 | } 84 | 85 | public void setLang(String lang) { 86 | this.lang = lang; 87 | } 88 | 89 | public boolean isVerified() { 90 | return verified; 91 | } 92 | 93 | public void setVerified(boolean verified) { 94 | this.verified = verified; 95 | } 96 | 97 | public String getProfileImageUrl() { 98 | return profileImageUrl; 99 | } 100 | 101 | public void setProfileImageUrl(String profileImageUrl) { 102 | this.profileImageUrl = profileImageUrl; 103 | } 104 | 105 | public int getFriendsCount() { 106 | return friendsCount; 107 | } 108 | 109 | public void setFriendsCount(int friendsCount) { 110 | this.friendsCount = friendsCount; 111 | } 112 | 113 | public String getDescription() { 114 | return description; 115 | } 116 | 117 | public void setDescription(String description) { 118 | this.description = description; 119 | } 120 | 121 | public int getFavouritesCount() { 122 | return favouritesCount; 123 | } 124 | 125 | public void setFavouritesCount(int favouritesCount) { 126 | this.favouritesCount = favouritesCount; 127 | } 128 | 129 | public boolean isGeoEnabled() { 130 | return geoEnabled; 131 | } 132 | 133 | public void setGeoEnabled(boolean geoEnabled) { 134 | this.geoEnabled = geoEnabled; 135 | } 136 | 137 | public int getListedCount() { 138 | return listedCount; 139 | } 140 | 141 | public void setListedCount(int listedCount) { 142 | this.listedCount = listedCount; 143 | } 144 | 145 | public String getProfileBackgroundImageUrl() { 146 | return profileBackgroundImageUrl; 147 | } 148 | 149 | public void setProfileBackgroundImageUrl(String profileBackgroundImageUrl) { 150 | this.profileBackgroundImageUrl = profileBackgroundImageUrl; 151 | } 152 | 153 | public boolean isProtectedUser() { 154 | return protectedUser; 155 | } 156 | 157 | public void setProtectedUser(boolean protectedUser) { 158 | this.protectedUser = protectedUser; 159 | } 160 | 161 | public String getLocation() { 162 | return location; 163 | } 164 | 165 | public void setLocation(String location) { 166 | this.location = location; 167 | } 168 | 169 | public long getStatusesCount() { 170 | return statusesCount; 171 | } 172 | 173 | public void setStatusesCount(long statusesCount) { 174 | this.statusesCount = statusesCount; 175 | } 176 | 177 | public String getTimeZone() { 178 | return timeZone; 179 | } 180 | 181 | public void setTimeZone(String timeZone) { 182 | this.timeZone = timeZone; 183 | } 184 | 185 | public boolean isContributorsEnabled() { 186 | return contributorsEnabled; 187 | } 188 | 189 | public void setContributorsEnabled(boolean contributorsEnabled) { 190 | this.contributorsEnabled = contributorsEnabled; 191 | } 192 | 193 | public int getUtcOffset() { 194 | return utcOffset; 195 | } 196 | 197 | public void setUtcOffset(int utcOffset) { 198 | this.utcOffset = utcOffset; 199 | } 200 | 201 | public String getCreatedAt() { 202 | return createdAt; 203 | } 204 | 205 | public void setCreatedAt(String createdAt) { 206 | this.createdAt = createdAt; 207 | } 208 | 209 | public String toString() { 210 | StringBuffer sb = new StringBuffer(); 211 | sb.append("{") 212 | .append("id:") 213 | .append(id) 214 | .append(",") 215 | .append("screenName:") 216 | .append(screenName) 217 | .append(",") 218 | .append("name:") 219 | .append(name) 220 | .append(",") 221 | .append("url:") 222 | .append(url) 223 | .append(",") 224 | .append("followersCount:") 225 | .append(followersCount) 226 | .append(",") 227 | .append("lang:") 228 | .append(lang) 229 | .append(",") 230 | .append("verified:") 231 | .append(verified) 232 | .append(",") 233 | .append("profileImageUrl:") 234 | .append(profileImageUrl) 235 | .append(",") 236 | .append("friendsCount:") 237 | .append(friendsCount) 238 | .append(",") 239 | .append("description:") 240 | .append(description) 241 | .append(",") 242 | .append("favouritesCount:") 243 | .append(favouritesCount) 244 | .append(",") 245 | .append("geoEnabled:") 246 | .append(geoEnabled) 247 | .append(",") 248 | .append("listedCount:") 249 | .append(listedCount) 250 | .append(",") 251 | .append("profileBackgroundImageUrl:") 252 | .append(profileBackgroundImageUrl) 253 | .append(",") 254 | .append("protectedUser:") 255 | .append(protectedUser) 256 | .append(",") 257 | .append("location:") 258 | .append(location) 259 | .append(",") 260 | .append("statusesCount:") 261 | .append(statusesCount) 262 | .append(",") 263 | .append("timeZone:") 264 | .append(timeZone) 265 | .append(",") 266 | .append("contributorsEnabled:") 267 | .append(contributorsEnabled) 268 | .append(",") 269 | .append("utcOffset:") 270 | .append(utcOffset) 271 | .append(",") 272 | .append("createdAt:") 273 | .append(createdAt) 274 | .append("}"); 275 | 276 | return sb.toString(); 277 | 278 | } 279 | 280 | } 281 | -------------------------------------------------------------------------------- /twittertopiccount/src/main/resources/adapter_conf.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 8 | 9 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /twittertopiccount/src/main/resources/twittertopiccount_conf.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 12 | 13 | 14 | 15 | 16 | 17 | RawStatus * 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | TopicSeen topic 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | AggregatedTopicSeen reportKey 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | TopicSeen 61 | 62 | 63 | 64 | 65 | topic 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | AggregatedTopicSeen 76 | 77 | 78 | 79 | 80 | reportKey 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | AggregatedTopicSeen 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | -------------------------------------------------------------------------------- /twittertopiccount_scala/README.md: -------------------------------------------------------------------------------- 1 | Twitter Hash Tag Count Example (Scala) 2 | =============== 3 | 4 | Introduction 5 | ------------ 6 | This sample application listens to the twitter garden hose and keeps track of the top 10 hash tags. 7 | 8 | Requirements 9 | ------------ 10 | 11 | * Linux 12 | * Java 1.6 13 | * Maven 14 | * S4 Communication Layer 15 | * S4 Core 16 | 17 | Build Instructions 18 | ------------------ 19 | 20 | 1. First build and install the comm and core packages in your Maven repository. 21 | 22 | 2. Build and install using Maven 23 | 24 | mvn assembly:assembly install 25 | 26 | -------------------------------------------------------------------------------- /twittertopiccount_scala/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | tar.gz 6 | 7 | true 8 | ${artifactId} 9 | 10 | 11 | false 12 | lib 13 | runtime 14 | false 15 | 16 | log4j:log4j 17 | 18 | 19 | 20 | 21 | 22 | ${project.basedir}/target 23 | lib 24 | 25 | *.jar 26 | 27 | 28 | 29 | ${project.basedir}/src/main/resources 30 | 31 | 32 | twittertopiccount_scala_conf.xml 33 | adapter_conf.xml 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /twittertopiccount_scala/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | io.s4.examples 5 | twittertopiccount_scala 6 | jar 7 | 0.0.0.2 8 | twittertopiccount_scala 9 | http://maven.apache.org 10 | 11 | 12 | scala-tools.org 13 | Scala-tools Maven2 Repository 14 | http://scala-tools.org/repo-releases 15 | 16 | 17 | 18 | 19 | scala-tools.org 20 | Scala-tools Maven2 Repository 21 | http://scala-tools.org/repo-releases 22 | 23 | 24 | 25 | 26 | org.scala-lang 27 | scala-library 28 | 2.8.1 29 | 30 | 31 | net.liftweb 32 | lift-json_2.8.1 33 | 2.2 34 | 35 | 36 | log4j 37 | log4j 38 | 1.2.15 39 | 40 | 41 | javax.mail 42 | mail 43 | 44 | 45 | javax.jms 46 | jms 47 | 48 | 49 | com.sun.jdmk 50 | jmxtools 51 | 52 | 53 | com.sun.jmx 54 | jmxri 55 | 56 | 57 | 58 | 59 | io.s4 60 | s4_core 61 | 0.3.0.0 62 | 63 | 64 | commons-codec 65 | commons-codec 66 | 1.4 67 | 68 | 69 | commons-httpclient 70 | commons-httpclient 71 | 3.1 72 | 73 | 74 | 75 | ${artifactId}-${version} 76 | 77 | 78 | src/main/resources 79 | 80 | * 81 | 82 | 83 | 84 | 85 | 86 | maven-assembly-plugin 87 | 88 | 89 | assembly.xml 90 | 91 | 92 | 93 | 94 | org.scala-tools 95 | maven-scala-plugin 96 | 97 | 98 | scala-compile-first 99 | process-resources 100 | 101 | add-source 102 | compile 103 | 104 | 105 | 106 | scala-test-compile 107 | process-test-resources 108 | 109 | testCompile 110 | 111 | 112 | 113 | 114 | 115 | org.apache.maven.plugins 116 | maven-compiler-plugin 117 | 2.0.1 118 | 119 | 1.6 120 | 1.6 121 | 122 | 123 | 124 | compile 125 | 126 | compile 127 | 128 | 129 | 130 | 131 | 132 | 133 | maven-eclipse-plugin 134 | 135 | 136 | **/*.scala 137 | 138 | 139 | ch.epfl.lamp.sdt.core.scalabuilder 140 | 141 | 142 | 143 | ch.epfl.lamp.sdt.core.scalanature 144 | 145 | 146 | org.eclipse.jdt.launching.JRE_CONTAINER 147 | ch.epfl.lamp.sdt.launching.SCALA_CONTAINER 148 | 149 | 150 | 151 | 152 | 153 | org.codehaus.mojo 154 | build-helper-maven-plugin 155 | 156 | 157 | add-source 158 | generate-sources 159 | 160 | add-source 161 | 162 | 163 | 164 | src/main/scala 165 | 166 | 167 | 168 | 169 | add-test-source 170 | generate-sources 171 | 172 | add-test-source 173 | 174 | 175 | 176 | src/test/scala 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | -------------------------------------------------------------------------------- /twittertopiccount_scala/src/main/java/io/s4/example/twittertopiccount/DirectToFilePersister.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.example.twittertopiccount; 17 | 18 | import java.io.FileWriter; 19 | import java.io.IOException; 20 | import java.util.HashMap; 21 | import java.util.HashSet; 22 | import java.util.Map; 23 | import java.util.Set; 24 | 25 | import org.apache.log4j.Logger; 26 | 27 | import io.s4.persist.Persister; 28 | 29 | public class DirectToFilePersister implements Persister { 30 | private String outputFilename; 31 | private int persistCount; 32 | 33 | public void setOutputFilename(String outputFilename) { 34 | this.outputFilename = outputFilename; 35 | } 36 | 37 | @Override 38 | public int cleanOutGarbage() throws InterruptedException { 39 | return 0; 40 | } 41 | 42 | @Override 43 | public Object get(String arg0) throws InterruptedException { 44 | return null; 45 | } 46 | 47 | @Override 48 | public Map getBulk(String[] arg0) 49 | throws InterruptedException { 50 | return new HashMap(); 51 | } 52 | 53 | @Override 54 | public Map getBulkObjects(String[] arg0) 55 | throws InterruptedException { 56 | return new HashMap(); 57 | } 58 | 59 | @Override 60 | public int getCacheEntryCount() { 61 | return 1; 62 | } 63 | 64 | @Override 65 | public Object getObject(String arg0) throws InterruptedException { 66 | return null; 67 | } 68 | 69 | @Override 70 | public int getPersistCount() { 71 | return persistCount; 72 | } 73 | 74 | @Override 75 | public int getQueueSize() { 76 | return 0; 77 | } 78 | 79 | @Override 80 | public Set keySet() { 81 | return new HashSet(); 82 | } 83 | 84 | @Override 85 | public void remove(String arg0) throws InterruptedException { 86 | 87 | } 88 | 89 | @Override 90 | public void set(String key, Object value, int persistTime) 91 | throws InterruptedException { 92 | 93 | FileWriter fw = null; 94 | try { 95 | fw = new FileWriter(outputFilename); 96 | fw.write(String.valueOf(value)); 97 | } catch (IOException e) { 98 | // TODO Auto-generated catch block 99 | Logger.getLogger("s4").error(e); 100 | } finally { 101 | if (fw != null) { 102 | try { 103 | fw.close(); 104 | } catch (Exception e) { 105 | } 106 | } 107 | } 108 | } 109 | 110 | @Override 111 | public void setAsynch(String key, Object value, int persistTime) { 112 | try { 113 | set(key, value, persistTime); 114 | } catch (InterruptedException ie) { 115 | } 116 | } 117 | 118 | } 119 | -------------------------------------------------------------------------------- /twittertopiccount_scala/src/main/resources/adapter_conf.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 7 | 8 | 9 | 10 | 12 | 13 | 14 | 15 | 16 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /twittertopiccount_scala/src/main/resources/twittertopiccount_scala_conf.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | 10 | 11 | TopicSeen topic 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | RawStatus * 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | TopicSeen topic 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | AggregatedTopicSeen reportKey 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | TopicSeen 64 | 65 | 66 | 67 | 68 | topic 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | AggregatedTopicSeen 79 | 80 | 81 | 82 | 83 | reportKey 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | AggregatedTopicSeen 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | -------------------------------------------------------------------------------- /twittertopiccount_scala/src/main/scala/events/Events.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | 17 | package io.s4.example.twittertopiccount.event 18 | 19 | import scala.reflect.BeanProperty 20 | 21 | // Event: Status 22 | case class Status( 23 | @BeanProperty var id: Long, 24 | @BeanProperty var text: String, 25 | @BeanProperty var created_at: String, 26 | @BeanProperty var user: User 27 | ) { 28 | def this()= this(0L, null, null, null) 29 | } 30 | 31 | // Event: Topic 32 | case class Topic ( 33 | @BeanProperty var topic: String, 34 | @BeanProperty var count: Int 35 | ) { 36 | @BeanProperty var reportKey: String = _ 37 | def this()= this(null, 0) 38 | } 39 | 40 | // Event: User 41 | case class User ( 42 | @BeanProperty var id: Long, 43 | @BeanProperty var name: String, 44 | @BeanProperty var screen_name: String 45 | ) { 46 | def this()= this(0L, null, null) 47 | } 48 | 49 | -------------------------------------------------------------------------------- /twittertopiccount_scala/src/main/scala/listener/TwitterStreamListener.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | 17 | package io.s4.example.twittertopiccount.listener 18 | 19 | import scala.reflect.BeanProperty 20 | import scala.concurrent.ops._ 21 | import scala.collection.JavaConversions._ 22 | import org.apache.log4j.Logger 23 | 24 | import java.util.concurrent._ 25 | import java.util.HashSet 26 | import java.util.concurrent.LinkedBlockingQueue 27 | 28 | import io.s4.collector.EventWrapper 29 | import io.s4.listener.EventHandler 30 | import io.s4.listener.EventProducer 31 | 32 | import net.liftweb.json._ 33 | import net.liftweb.json.Extraction._ 34 | 35 | import io.s4.example.twittertopiccount.event._ 36 | 37 | class TwitterStreamListener extends EventProducer { 38 | 39 | @BeanProperty var queue: LinkedBlockingQueue[String] = _ 40 | @BeanProperty var threads = 10 41 | @BeanProperty var streamName: String = _ 42 | 43 | var handlers = new HashSet[EventHandler] 44 | override def addHandler(h: EventHandler)= handlers.add(h) 45 | override def removeHandler(h: EventHandler)= handlers.remove(h) 46 | 47 | def init()= { 48 | spawn { 49 | var pool = Executors.newFixedThreadPool(10) 50 | for (i <- 1 to threads) { 51 | pool.execute(new Dequeuer) 52 | } 53 | } 54 | } 55 | 56 | class Dequeuer extends Runnable { 57 | 58 | implicit val formats = DefaultFormats 59 | 60 | def run()= { 61 | while(!Thread.interrupted()) { 62 | try { 63 | var message = queue.take() 64 | var json = JsonParser.parse(message) 65 | //println(Printer.pretty(JsonAST.render(json))) 66 | var status = json.extract[Status] 67 | // println(status) 68 | var ew = new EventWrapper(streamName, status, null) 69 | try { 70 | handlers.foreach(_.processEvent(ew)) 71 | } catch { 72 | case e: Exception => Logger.getLogger("s4").error("Exception in raw event handler", e) 73 | } 74 | } catch { 75 | case e: InterruptedException => Thread.currentThread().interrupt() 76 | case _ => 77 | } 78 | } 79 | } 80 | 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /twittertopiccount_scala/src/main/scala/processor/TopNTopicPE.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.example.twittertopiccount.processor 17 | 18 | import scala.math._ 19 | import scala.reflect.BeanProperty 20 | import scala.collection.JavaConversions._ 21 | 22 | import java.util.concurrent.ConcurrentHashMap 23 | import org.apache.log4j.Logger 24 | 25 | import net.liftweb.json._ 26 | import net.liftweb.json.JsonDSL._ 27 | import net.liftweb.json.JsonAST._ 28 | 29 | import io.s4.persist.Persister 30 | import io.s4.processor.AbstractPE 31 | 32 | import io.s4.example.twittertopiccount.event._ 33 | 34 | class TopNTopicPE extends AbstractPE { 35 | @BeanProperty var id: String = _ 36 | @BeanProperty var persistKey = "myapp:topNTopics" 37 | @BeanProperty var persister: Persister = _ 38 | @BeanProperty var entryCount = 10 39 | @BeanProperty var persistTime = 0 40 | var topicMap = new ConcurrentHashMap[String, Int] 41 | 42 | def processEvent(topic: Topic): Unit= topicMap.put(topic.topic, topic.count) 43 | 44 | def output(): Unit= { 45 | 46 | // sort list of tuples by second value 47 | var sorted = topicMap.toList.sortBy(-_._2) 48 | // limit to entryCount 49 | var tops = sorted.slice(0, min(entryCount, sorted.length)) 50 | 51 | // use lift-json dsl to generate json 52 | val json = 53 | ("topN" -> 54 | tops.map { (x: (String, Int)) => 55 | (("topic" -> x._1) ~ 56 | ("count" -> x._2)) 57 | } 58 | ) 59 | 60 | try { 61 | persister.set(persistKey, pretty(render(json)), persistTime) 62 | } catch { 63 | case e: Exception => Logger.getLogger("s4").error(e) 64 | } 65 | } 66 | 67 | } 68 | 69 | 70 | 71 | /* 72 | try { 73 | var jsonMessage = new JSONObject 74 | var jsonTopN = new JSONArray 75 | for (i <- 0 until(scala.math.min(entryCount, topN.length))) { 76 | var tne: TopNEntry = topN(i) 77 | var jsonEntry = new JSONObject() 78 | jsonEntry.put("topic", tne.topic) 79 | jsonEntry.put("count", tne.count) 80 | jsonTopN.put(jsonEntry) 81 | } 82 | jsonMessage.put("topN", jsonTopN) 83 | 84 | class TopNEntry(@BeanProperty var topic: String, @BeanProperty var count: Int) extends Ordered[TopNEntry] { 85 | def compare(that: TopNEntry)= this.count-that.count 86 | override def toString= "{topic:" + topic + ", count: " + count + "}" 87 | } 88 | */ 89 | -------------------------------------------------------------------------------- /twittertopiccount_scala/src/main/scala/processor/TopicCountAndReportPE.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.example.twittertopiccount.processor 17 | 18 | import org.apache.log4j.Logger 19 | import scala.reflect.BeanProperty 20 | 21 | import io.s4.dispatcher.EventDispatcher 22 | import io.s4.processor.AbstractPE 23 | 24 | import io.s4.example.twittertopiccount.event._ 25 | 26 | class TopicCountAndReportPE extends AbstractPE { 27 | @BeanProperty var id: String = _ 28 | @BeanProperty var dispatcher: EventDispatcher = _ 29 | @BeanProperty var outputStreamName: String = _ 30 | @BeanProperty var threshold: Int = _ 31 | @BeanProperty var count: Int = _ 32 | 33 | def processEvent(topic: Topic): Unit= { 34 | count += topic.count; 35 | } 36 | 37 | def output(): Unit= { 38 | if (count < threshold) return 39 | var topic: Topic = new Topic(this.getKeyValue.get(0).toString, count) 40 | topic.reportKey = "1" 41 | dispatcher.dispatchEvent(outputStreamName, topic) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /twittertopiccount_scala/src/main/scala/processor/TopicExtractorPE.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | package io.s4.example.twittertopiccount.processor 17 | 18 | import org.apache.log4j.Logger 19 | import scala.reflect.BeanProperty 20 | 21 | import io.s4.dispatcher.EventDispatcher 22 | import io.s4.processor.AbstractPE 23 | 24 | import io.s4.example.twittertopiccount.event._ 25 | 26 | class TopicExtractorPE extends AbstractPE { 27 | @BeanProperty var id: String = _ 28 | @BeanProperty var dispatcher: EventDispatcher = _ 29 | @BeanProperty var outputStreamName: String = _ 30 | 31 | def processEvent(status: Status): Unit= { 32 | var text = status.text 33 | if (text == null) return 34 | hashtags(text).foreach { x => 35 | dispatcher.dispatchEvent(outputStreamName, new Topic(x, 1)) 36 | } 37 | } 38 | 39 | def hashtags(in: String): Array[String]= in.split(" ").filter(_.startsWith("#")).map(_.replaceAll("[^a-zA-Z0-9]", "")) 40 | 41 | def output(): Unit= { 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /twittertopiccount_scala/src/main/scala/util/TwitterStreamClient.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, 10 | * software distributed under the License is distributed on an 11 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific 13 | * language governing permissions and limitations under the 14 | * License. See accompanying LICENSE file. 15 | */ 16 | 17 | package io.s4.example.twittertopiccount.util 18 | 19 | import java.io.InputStream 20 | import java.io.InputStreamReader 21 | import java.io.BufferedReader 22 | import java.io.IOException 23 | import org.apache.commons.httpclient.HttpClient 24 | import org.apache.commons.httpclient.HttpException 25 | import org.apache.commons.httpclient.HttpStatus 26 | import org.apache.commons.httpclient.HttpURL 27 | import org.apache.commons.httpclient.UsernamePasswordCredentials 28 | import org.apache.commons.httpclient.auth.AuthScope 29 | import org.apache.commons.httpclient.HttpMethod 30 | import org.apache.commons.httpclient.methods.GetMethod 31 | 32 | import java.util.concurrent.LinkedBlockingQueue 33 | 34 | import scala.reflect.BeanProperty 35 | import scala.concurrent.ops._ 36 | 37 | class TwitterStreamClient { 38 | 39 | @BeanProperty var user: String = _ 40 | @BeanProperty var pass: String = _ 41 | @BeanProperty var url: String = _ 42 | @BeanProperty var backoffTime = 1000 43 | @BeanProperty var maxBackoffTime = 30 * 1000 44 | @BeanProperty var queue: LinkedBlockingQueue[String] = _ 45 | 46 | def init() = { 47 | spawn { 48 | val http = new HttpClient 49 | val getter = new GetMethod(url) 50 | http.getState.setCredentials(this.getAuthScope(getter), new UsernamePasswordCredentials(user + ":" + pass)) 51 | while(!Thread.interrupted()) { 52 | try { 53 | http.executeMethod(getter) 54 | if (getter.getStatusCode() != HttpStatus.SC_OK) { 55 | throw new HttpException("HTTP status: " + getter.getStatusCode() + " " + getter.getStatusLine()) 56 | } 57 | this.process(getter.getResponseBodyAsStream()) 58 | } catch { 59 | case _ => { 60 | try { 61 | Thread.sleep(backoffTime) 62 | } catch { 63 | case e: InterruptedException => Thread.currentThread().interrupt() 64 | } 65 | backoffTime = backoffTime * 2; 66 | if (backoffTime > maxBackoffTime) { 67 | backoffTime = maxBackoffTime; 68 | } 69 | } 70 | } finally { 71 | getter.releaseConnection 72 | } 73 | } 74 | } 75 | } 76 | 77 | def getAuthScope(method: HttpMethod) = { 78 | val url = new HttpURL(method.getURI.toString) 79 | new AuthScope(url.getHost, url.getPort) 80 | } 81 | 82 | def process(is: InputStream) = { 83 | val reader: BufferedReader = new BufferedReader(new InputStreamReader(is, "UTF-8")) 84 | var line = reader.readLine() 85 | while (line != null) { 86 | queue.add(line) 87 | line = reader.readLine() 88 | } 89 | is.close 90 | } 91 | 92 | } 93 | --------------------------------------------------------------------------------