├── README.md ├── java-stackexchange ├── .gitignore ├── README.md ├── pom.xml └── src │ ├── main │ └── java │ │ └── org │ │ └── stackexchange │ │ └── api │ │ ├── client │ │ ├── ApiUris.java │ │ ├── HttpFactory.java │ │ ├── QuestionsApi.java │ │ └── RequestBuilder.java │ │ ├── constants │ │ ├── ApiConstants.java │ │ └── StackSite.java │ │ └── model │ │ └── Question.java │ └── test │ ├── java │ └── org │ │ └── stackexchange │ │ └── api │ │ └── client │ │ └── QuestionsApiLiveTest.java │ └── resources │ └── logback-test.xml ├── java-stackexchange2twitter ├── .gitignore ├── key.p12 ├── pom.xml └── src │ ├── main │ ├── java │ │ └── org │ │ │ ├── apache │ │ │ └── commons │ │ │ │ └── lang3 │ │ │ │ └── tuple │ │ │ │ ├── ImmutableTriple.java │ │ │ │ └── Triple.java │ │ │ ├── classification │ │ │ ├── data │ │ │ │ ├── ClassificationData.java │ │ │ │ ├── FromMultipleFilesDataLoadingStrategy.java │ │ │ │ ├── GenericClassificationDataUtil.java │ │ │ │ └── SimpleDataLoadingStrategy.java │ │ │ ├── service │ │ │ │ └── ClassificationService.java │ │ │ ├── spring │ │ │ │ └── ClassificationConfig.java │ │ │ └── util │ │ │ │ ├── ClassificationSettings.java │ │ │ │ ├── Classifiers.java │ │ │ │ └── GenericClassificationUtil.java │ │ │ ├── common │ │ │ ├── metrics │ │ │ │ ├── MetricRegistryFactoryBean.java │ │ │ │ └── MetricsUtil.java │ │ │ ├── persistence │ │ │ │ ├── IEntity.java │ │ │ │ ├── IOperations.java │ │ │ │ ├── IRawOperations.java │ │ │ │ ├── meta │ │ │ │ │ └── SearchField.java │ │ │ │ └── setup │ │ │ │ │ ├── AfterSetupEvent.java │ │ │ │ │ ├── BeforeSetupEvent.java │ │ │ │ │ ├── checks │ │ │ │ │ └── CredentialsForAllTwitterAccountsExistChecker.java │ │ │ │ │ └── upgrades │ │ │ │ │ ├── live │ │ │ │ │ ├── IRecreateMissingQuestionTweetsUpgrader.java │ │ │ │ │ ├── IRecreateMissingRetweetsUpgrader.java │ │ │ │ │ ├── IRemoveOrphanedRetweetsUpgrader.java │ │ │ │ │ ├── RecreateMissingQuestionTweetsUpgrader.java │ │ │ │ │ ├── RecreateMissingRetweetsUpgrader.java │ │ │ │ │ ├── RemoveOrphanedRetweetsUpgrader.java │ │ │ │ │ └── nolonger │ │ │ │ │ │ ├── AddTextToRetweetsUpgrader.java │ │ │ │ │ │ ├── FillInDatesOfRetweetsUpgrader.java │ │ │ │ │ │ ├── IAddTextToRetweetsUpgrader.java │ │ │ │ │ │ ├── IFillInDatesOfRetweetsUpgrader.java │ │ │ │ │ │ ├── IRemoveDuplicateRetweetsUpgrader.java │ │ │ │ │ │ └── RemoveDuplicateRetweetsUpgrader.java │ │ │ │ │ └── local │ │ │ │ │ ├── CleanTextOfRetweetsUpgrader.java │ │ │ │ │ ├── ICleanTextOfRetweetsUpgrader.java │ │ │ │ │ ├── IRemoveLocalDuplicateRetweetsUpgrader.java │ │ │ │ │ └── RemoveLocalDuplicateRetweetsUpgrader.java │ │ │ ├── service │ │ │ │ ├── AnalysisService.java │ │ │ │ ├── BaseTweetFromSourceLiveService.java │ │ │ │ ├── ContentExtractorLiveService.java │ │ │ │ ├── IAnalysisService.java │ │ │ │ ├── external │ │ │ │ │ └── bitly │ │ │ │ │ │ └── BitlyService.java │ │ │ │ └── live │ │ │ │ │ ├── HttpLiveService.java │ │ │ │ │ └── LinkLiveService.java │ │ │ ├── spring │ │ │ │ ├── CommonMetricsConfig.java │ │ │ │ ├── CommonPersistenceJPAConfig.java │ │ │ │ ├── CommonServiceConfig.java │ │ │ │ ├── MyApplicationContextInitializerProv.java │ │ │ │ ├── MyBaseApplicationContextInitializer.java │ │ │ │ └── PersistenceJPACommonConfig.java │ │ │ └── util │ │ │ │ └── order │ │ │ │ └── OrderById.java │ │ │ ├── gplus │ │ │ ├── service │ │ │ │ ├── ActivityHelper.java │ │ │ │ ├── GplusExtractorService.java │ │ │ │ └── GplusService.java │ │ │ ├── spring │ │ │ │ └── GplusContextConfig.java │ │ │ └── stackexchange │ │ │ │ ├── GplusApiFactoryBean.java │ │ │ │ └── GplusStackexchangeScheduler.java │ │ │ ├── keyval │ │ │ ├── persistence │ │ │ │ ├── dao │ │ │ │ │ └── IKeyValJpaDAO.java │ │ │ │ └── model │ │ │ │ │ └── KeyVal.java │ │ │ └── spring │ │ │ │ └── KeyValPersistenceJPAConfig.java │ │ │ ├── rss │ │ │ ├── RssScheduler.java │ │ │ ├── persistence │ │ │ │ ├── dao │ │ │ │ │ └── IRssEntryJpaDAO.java │ │ │ │ └── model │ │ │ │ │ └── RssEntry.java │ │ │ ├── service │ │ │ │ ├── RssService.java │ │ │ │ └── TweetRssLiveService.java │ │ │ └── spring │ │ │ │ ├── RssContextConfig.java │ │ │ │ └── RssPersistenceJPAConfig.java │ │ │ ├── stackexchange │ │ │ ├── TweetStackexchangeScheduler.java │ │ │ ├── component │ │ │ │ ├── MinStackScoreRetriever.java │ │ │ │ └── StackExchangePageStrategy.java │ │ │ ├── persistence │ │ │ │ ├── dao │ │ │ │ │ └── IQuestionTweetJpaDAO.java │ │ │ │ ├── model │ │ │ │ │ └── QuestionTweet.java │ │ │ │ └── setup │ │ │ │ │ ├── MainSetup.java │ │ │ │ │ └── TwitterAccountToStackAccount.java │ │ │ ├── service │ │ │ │ └── TweetStackexchangeLiveService.java │ │ │ ├── spring │ │ │ │ ├── StackexchangeConfig.java │ │ │ │ ├── StackexchangeContextConfig.java │ │ │ │ └── StackexchangePersistenceJPAConfig.java │ │ │ └── util │ │ │ │ ├── GenericUtil.java │ │ │ │ ├── IDUtil.java │ │ │ │ ├── StackTag.java │ │ │ │ ├── StackTagAdvanced.java │ │ │ │ ├── TwitterAccountEnum.java │ │ │ │ └── TwitterTag.java │ │ │ └── tweet │ │ │ ├── meta │ │ │ ├── FollowMetaScheduler.java │ │ │ ├── MetricsClearScheduler.java │ │ │ ├── NonTechnicalFollowMetaScheduler.java │ │ │ ├── NonTechnicalTweetMetaScheduler.java │ │ │ ├── TweetMetaScheduler.java │ │ │ ├── TwitterUserSnapshot.java │ │ │ ├── analysis │ │ │ │ ├── TweetPassesSet1ChecksPredicate.java │ │ │ │ ├── TweetPassesSet2ChecksPredicate.java │ │ │ │ └── TweetPassesSet3ChecksPredicate.java │ │ │ ├── component │ │ │ │ ├── InteractionHistoryPoller.java │ │ │ │ ├── MetaPoller.java │ │ │ │ ├── PredefinedAccountRetriever.java │ │ │ │ ├── TwitterInteractionValuesRetriever.java │ │ │ │ └── TwitterQuotaStatusPoller.java │ │ │ ├── persistence │ │ │ │ ├── dao │ │ │ │ │ └── IRetweetJpaDAO.java │ │ │ │ ├── model │ │ │ │ │ └── Retweet.java │ │ │ │ └── setup │ │ │ │ │ └── TwitterMetaSetupListener.java │ │ │ ├── service │ │ │ │ ├── AlreadyFollowedByPredicate.java │ │ │ │ ├── FollowLiveService.java │ │ │ │ ├── InteractionLiveService.java │ │ │ │ ├── IsMyselfPredicate.java │ │ │ │ ├── TweetByRtComparator.java │ │ │ │ ├── TweetContainsWordPredicate.java │ │ │ │ ├── TweetMetaLiveService.java │ │ │ │ └── TweetMetaLocalService.java │ │ │ ├── spring │ │ │ │ ├── TwitterMetaConfig.java │ │ │ │ └── TwitterMetaPersistenceJPAConfig.java │ │ │ └── util │ │ │ │ ├── TweetIsRetweetPredicate.java │ │ │ │ ├── TweetLinksToSePredicate.java │ │ │ │ └── TweetNotLinksToSePredicate.java │ │ │ ├── spring │ │ │ ├── TwitterConfig.java │ │ │ ├── TwitterLiveConfig.java │ │ │ └── util │ │ │ │ └── SpringProfileUtil.java │ │ │ └── twitter │ │ │ ├── component │ │ │ ├── DiscouragedExpressionRetriever.java │ │ │ ├── MinRtRetriever.java │ │ │ └── TwitterHashtagsRetriever.java │ │ │ ├── persistence │ │ │ └── model │ │ │ │ └── TwitterAccount.java │ │ │ ├── service │ │ │ ├── AdvancedTweetService.java │ │ │ ├── ErrorPoller.java │ │ │ ├── TagRetrieverService.java │ │ │ ├── TweetLiveService.java │ │ │ ├── TweetMentionService.java │ │ │ ├── TwitterTemplateCreator.java │ │ │ └── live │ │ │ │ ├── ITwitterWriteLiveService.java │ │ │ │ ├── TweetToStringFunction.java │ │ │ │ ├── TwitterAnalysisLiveService.java │ │ │ │ ├── TwitterReadLiveService.java │ │ │ │ ├── TwitterWriteDevLiveService.java │ │ │ │ ├── TwitterWriteLiveService.java │ │ │ │ └── UserLiveService.java │ │ │ └── util │ │ │ ├── TwitterInteraction.java │ │ │ └── TwitterInteractionWithValue.java │ ├── resources │ │ ├── backup.sh │ │ ├── classification │ │ │ ├── commercial │ │ │ │ ├── commercial.classif │ │ │ │ └── noncommercial.classif │ │ │ ├── jobs │ │ │ │ ├── jobs-core.classif │ │ │ │ ├── jobs-full.classif │ │ │ │ ├── jobs-onthefence.classif │ │ │ │ ├── jobs.classif │ │ │ │ ├── nonjobs-core.classif │ │ │ │ ├── nonjobs-full.classif │ │ │ │ └── nonjobs.classif │ │ │ ├── programming │ │ │ │ ├── nonprogramming.classif │ │ │ │ └── programming.classif │ │ │ ├── sample.classif │ │ │ └── test │ │ │ │ ├── commercial │ │ │ │ ├── commercial.classif │ │ │ │ └── noncommercial.classif │ │ │ │ ├── jobs │ │ │ │ ├── jobs.classif │ │ │ │ ├── nonjobs.classif │ │ │ │ └── note.info │ │ │ │ └── programming │ │ │ │ ├── nonprogramming.classif │ │ │ │ └── programming.classif │ │ ├── commonPersistenceConfig.xml │ │ ├── config-dev.properties │ │ ├── config-production.properties │ │ ├── env-dev.properties │ │ ├── env-production.properties │ │ ├── interaction.properties │ │ ├── logback.xml │ │ ├── metaPersistenceConfig.xml │ │ ├── notes │ │ │ ├── blockedBy.txt │ │ │ ├── blogs.txt │ │ │ ├── commercial │ │ │ │ ├── decided │ │ │ │ │ ├── rejected-giveaway.txt │ │ │ │ │ ├── rejected-lucky.txt │ │ │ │ │ └── rejected-promo.txt │ │ │ │ └── todecide-priced.txt │ │ │ ├── decided │ │ │ │ ├── need-toaccept.txt │ │ │ │ ├── need-toreject.txt │ │ │ │ ├── need-tosort.txt │ │ │ │ ├── rejected-dance-decided.txt │ │ │ │ ├── rejected-dumb.txt │ │ │ │ ├── rejected-policitcal.txt │ │ │ │ ├── rejected-remix.txt │ │ │ │ ├── rejected-snake.txt │ │ │ │ ├── rejected-trial.txt │ │ │ │ └── rejected-wife.txt │ │ │ ├── rejected.txt │ │ │ ├── test │ │ │ │ ├── analysis │ │ │ │ │ └── analysis-banned.txt │ │ │ │ ├── commercial │ │ │ │ │ ├── deal-toaccept.txt │ │ │ │ │ ├── deal-toreject.txt │ │ │ │ │ ├── deals-toaccept.txt │ │ │ │ │ ├── deals-toreject.txt │ │ │ │ │ ├── generic-commercial-toaccept.txt │ │ │ │ │ ├── generic-commercial-toreject.txt │ │ │ │ │ ├── win-toaccept.txt │ │ │ │ │ └── win-toreject.txt │ │ │ │ ├── generic-toaccept.txt │ │ │ │ ├── generic-toreject.txt │ │ │ │ ├── sandbox-toaccept.txt │ │ │ │ ├── sandbox-toreject.txt │ │ │ │ ├── tweeting │ │ │ │ │ ├── tweeting-banned.txt │ │ │ │ │ └── tweeting-personal-banned.txt │ │ │ │ └── undecided │ │ │ │ │ ├── commercial-general-tosort.txt │ │ │ │ │ ├── deal-tosort.txt │ │ │ │ │ ├── deal-undecided.txt │ │ │ │ │ ├── deals-random.txt │ │ │ │ │ ├── win-random.txt │ │ │ │ │ ├── win-tosort.txt │ │ │ │ │ └── win-undecided.txt │ │ │ ├── todecide-killed.txt │ │ │ └── todecide-racial.txt │ │ ├── persistence-dev.properties │ │ ├── persistence-prod.properties │ │ ├── persistence-setup.properties │ │ ├── persistence-test.properties │ │ ├── rssPersistenceConfig.xml │ │ ├── setup.properties │ │ ├── stackPersistenceConfig.xml │ │ ├── stackexchange.properties │ │ ├── twitter │ │ │ ├── interactions.txt │ │ │ └── schedule.txt │ │ └── twitterInternal.properties │ └── webapp │ │ ├── META-INF │ │ └── MANIFEST.MF │ │ └── WEB-INF │ │ └── web.xml │ └── test │ ├── java │ └── org │ │ ├── classification │ │ ├── data │ │ │ ├── ClassificationDataCleanupManualTest.java │ │ │ ├── ClassificationTestData.java │ │ │ └── SpecificClassificationDataUtilIntegrationTest.java │ │ └── service │ │ │ ├── ClassificationServiceIntegrationTest.java │ │ │ ├── ClassificationServiceLiveTest.java │ │ │ ├── ClassificationUnitTest.java │ │ │ └── accuracy │ │ │ ├── ClassificationAccuracyServiceLiveTest.java │ │ │ ├── ClassificationCommercialAccuracyTestService.java │ │ │ ├── ClassificationJobsAccuracyTestService.java │ │ │ └── ClassificationProgrammingAccuracyTestService.java │ │ ├── common │ │ ├── persistence │ │ │ ├── AbstractRawServicePersistenceIntegrationTest.java │ │ │ ├── IEntityOperations.java │ │ │ └── setup │ │ │ │ └── upgrades │ │ │ │ ├── CleanTextOfRetweetsUpgraderManualTest.java │ │ │ │ ├── RecreateMissingQuestionTweetsUpgraderLiveManualTest.java │ │ │ │ ├── RecreateMissingRetweetsUpgraderLiveManualTest.java │ │ │ │ ├── RemoveDuplicateRetweetsUpgraderLiveManualTest.java │ │ │ │ ├── RemoveLocalDuplicateRetweetsUpgraderManualTest.java │ │ │ │ ├── RemoveOrphanedRetweetsUpgraderLiveTest.java │ │ │ │ └── nolonger │ │ │ │ ├── AddTextToRetweetsUpgraderLiveManualTest.java │ │ │ │ └── FillInDatesOfRetweetsUpgraderLiveTest.java │ │ ├── service │ │ │ ├── ContentExtractorLiveServiceLiveTest.java │ │ │ ├── LinkServiceIntegrationTest.java │ │ │ ├── external │ │ │ │ └── bitly │ │ │ │ │ └── BitlyServiceLiveTest.java │ │ │ └── live │ │ │ │ ├── HttpLiveServiceLiveTest.java │ │ │ │ ├── LinkLiveServiceLiveTest.java │ │ │ │ └── RetweetScoresTuningLiveTest.java │ │ └── text │ │ │ ├── LinkUtilUnitTest.java │ │ │ └── TextUtilsUnitTest.java │ │ ├── gplus │ │ ├── service │ │ │ └── GplusExtractorServiceLiveTest.java │ │ └── stackexchange │ │ │ └── GPlusLiveTest.java │ │ ├── keyval │ │ └── persistence │ │ │ └── service │ │ │ ├── KeyValEntityOps.java │ │ │ └── KeyValPersistenceIntegrationTest.java │ │ ├── rss │ │ ├── persistence │ │ │ └── service │ │ │ │ └── RssEntryEntityOps.java │ │ └── service │ │ │ ├── RssEntryPersistenceIntegrationTest.java │ │ │ ├── RssServiceReadOnlyLiveTest.java │ │ │ └── TweetRssServiceLiveTest.java │ │ ├── stackexchange │ │ ├── ITwitterWriteLiveServiceLiveTest.java │ │ ├── TwitterClassificationDataLiveTest.java │ │ ├── TwitterReadLiveServiceReadOnlyLiveTest.java │ │ ├── UserLiveServiceReadOnlyLiveTest.java │ │ ├── component │ │ │ ├── MinStackScoreTagRetrieverIntegrationTest.java │ │ │ ├── StackExchangePageStrategyUnitTest.java │ │ │ └── StackTagsExistsIntegrationTest.java │ │ ├── gather │ │ │ ├── ClassificationDataFromTwitterManualTest.java │ │ │ ├── CleanupStringFunction.java │ │ │ └── ContainsKeywordPredicate.java │ │ ├── persistence │ │ │ └── service │ │ │ │ ├── QuestionTweetEntityOps.java │ │ │ │ └── QuestionTweetPersistenceIntegrationTest.java │ │ └── service │ │ │ └── TweetStackexchangeLiveServiceLiveTest.java │ │ └── tweet │ │ ├── meta │ │ ├── component │ │ │ ├── InteractionHistoryPollerLiveManualTest.java │ │ │ ├── MetaPollerLiveManualTest.java │ │ │ └── TwitterQuotaStatusPollerLiveTest.java │ │ ├── persistence │ │ │ └── service │ │ │ │ ├── RetweetEntityOps.java │ │ │ │ └── RetweetPersistenceIntegrationTest.java │ │ └── service │ │ │ ├── FollowLiveServiceLiveTest.java │ │ │ ├── InteractionLiveServiceLiveTest.java │ │ │ ├── InteractionLiveServiceMockUnitTest.java │ │ │ ├── InteractionLiveServiceUnitTest.java │ │ │ ├── TweetContainsWordPredicateUnitTest.java │ │ │ ├── TweetMetaLiveServiceLiveTest.java │ │ │ ├── TweetMetaLocalServiceIntegrationTest.java │ │ │ └── TweetMetaLocalServiceManualTest.java │ │ ├── spring │ │ ├── DevFullContextManualTest.java │ │ ├── ProdFullContextManualTest.java │ │ ├── SetupPersistenceTestConfig.java │ │ └── TestStackexchangePersistenceJPAConfig.java │ │ ├── test │ │ ├── FullTestSuite.java │ │ ├── IntegrationTestSuite.java │ │ ├── LiveReadOnlyTestSuite.java │ │ ├── ManualAcceptUnitTestSuite.java │ │ ├── ManualIntegrationTestSuite.java │ │ ├── ManualRejectUnitTestSuite.java │ │ ├── ManualUnitTestSuite.java │ │ ├── NonTechnicalPropertiesExistIntegrationTestSuite.java │ │ ├── PersistenceIntegrationTestSuite.java │ │ ├── PropertiesExistIntegrationTestSuite.java │ │ ├── TweetFixture.java │ │ └── UnitTestSuite.java │ │ └── twitter │ │ ├── component │ │ ├── MinRtScoreForTwitterTagsExistsIntegrationTest.java │ │ └── TwitterHashtagWordsIntegrationTest.java │ │ ├── service │ │ ├── ErrorPollerUnitTest.java │ │ ├── TagRetrieverServiceIntegrationTest.java │ │ ├── TweetMentionServiceIntegrationTest.java │ │ ├── TweetMentionServiceUnitTest.java │ │ ├── TweetServiceIntegrationTest.java │ │ ├── TweetServiceReadOnlyLiveTest.java │ │ ├── TweetServiceUnitTest.java │ │ ├── TwitterTemplateCreatorLiveTest.java │ │ └── live │ │ │ └── TwitterAnalysisLiveServiceLiveTest.java │ │ └── util │ │ ├── DataCleanupManualTest.java │ │ ├── HashtagUnitTest.java │ │ ├── PersonalTweetsBannedForTweetingManualTest.java │ │ ├── TwitterUtilBannedForAnalysisByRegexManualTest.java │ │ ├── TwitterUtilBannedForCommercialAnalysisIntegrationTest.java │ │ ├── TwitterUtilBannedForTweetingManualTest.java │ │ ├── TwitterUtilUnitTest.java │ │ ├── commercial │ │ ├── AcceptTweetsByWordDealManualTest.java │ │ ├── AcceptTweetsByWordDealsManualTest.java │ │ ├── AcceptTweetsByWordGenericCommercialManualTest.java │ │ ├── AcceptTweetsByWordWinManualTest.java │ │ ├── RejectTweetsByWordCommercialGenericManualTest.java │ │ ├── RejectTweetsByWordDealManualTest.java │ │ ├── RejectTweetsByWordDealsManualTest.java │ │ └── RejectTweetsByWordWinManualTest.java │ │ └── generic │ │ ├── AcceptTweetsByWordGenericManualTest.java │ │ ├── AcceptTweetsByWordSandboxManualTest.java │ │ ├── RejectTweetsByWordGenericManualTest.java │ │ └── RejectTweetsByWordSandboxManualTest.java │ └── resources │ ├── classification │ └── results │ │ ├── pass01_full.log │ │ └── pass01_good.log │ ├── logback-test.xml │ ├── setup-test.properties │ ├── tweetdata.properties │ └── tweetscores │ ├── current_noSE_scores.log │ └── current_rt_scores.log ├── pom.xml └── tweet-value ├── .gitignore ├── pom.xml └── src ├── main ├── java │ └── org │ │ ├── classification │ │ └── util │ │ │ └── TweetSettings.java │ │ ├── common │ │ ├── service │ │ │ └── LinkService.java │ │ └── util │ │ │ ├── LinkUtil.java │ │ │ └── TextUtil.java │ │ └── tweet │ │ └── twitter │ │ ├── evaluator │ │ ├── AbstractEvaluator.java │ │ ├── ChainingEvaluator.java │ │ ├── IEvaluator.java │ │ └── impl │ │ │ ├── ForAnalysisEvaluator.java │ │ │ ├── ForCommercialAnalysisEvaluator.java │ │ │ ├── ForNonTechnicalEvaluator.java │ │ │ └── ForTweetingEvaluator.java │ │ ├── service │ │ ├── TweetService.java │ │ └── TweetType.java │ │ └── util │ │ ├── ErrorUtil.java │ │ ├── HashtagWordFunction.java │ │ ├── RejectExpressionUtil.java │ │ ├── TweetUtil.java │ │ └── TwitterUtil.java ├── resources │ ├── backup.sh │ ├── classification │ │ ├── commercial │ │ │ ├── commercial.classif │ │ │ └── noncommercial.classif │ │ ├── jobs │ │ │ ├── jobs-core.classif │ │ │ ├── jobs-full.classif │ │ │ ├── jobs-onthefence.classif │ │ │ ├── jobs.classif │ │ │ ├── nonjobs-core.classif │ │ │ ├── nonjobs-full.classif │ │ │ └── nonjobs.classif │ │ ├── programming │ │ │ ├── nonprogramming.classif │ │ │ └── programming.classif │ │ ├── sample.classif │ │ └── test │ │ │ ├── commercial │ │ │ ├── commercial.classif │ │ │ └── noncommercial.classif │ │ │ ├── jobs │ │ │ ├── jobs.classif │ │ │ ├── nonjobs.classif │ │ │ └── note.info │ │ │ └── programming │ │ │ ├── nonprogramming.classif │ │ │ └── programming.classif │ ├── logback.xml.backup │ ├── twitter │ │ ├── interactions.txt │ │ └── schedule.txt │ └── twitterInternal.properties └── webapp │ ├── META-INF │ └── MANIFEST.MF │ └── WEB-INF │ └── web.xml └── test ├── java └── META-INF │ └── MANIFEST.MF └── resources ├── classification └── results │ ├── pass01_full.log │ └── pass01_good.log ├── logback-test.xml ├── tweetdata.properties └── tweetscores ├── current_noSE_scores.log └── current_rt_scores.log /README.md: -------------------------------------------------------------------------------- 1 | ===================== 2 | # Twitter Machine Learning Bot 3 | 4 | ## Overview 5 | 6 | **The Twitter Accounts managed by this project - [Twitter Accounts](https://github.com/eugenp/stackexchange2twitter/wiki/Twitter-Accounts)**. 7 | 8 | 9 | **Twitter Bot** that tweets from the following sources: 10 | - mostly the Top Questions **on StackOverflow** (and other [StackExchange Q&A Sites](http://stackexchange.com/sites#)) 11 | - Twitter #hashtags 12 | - Google+ 13 | 14 | 15 | ## Technologies used and Dependencies 16 | - [Spring Social](https://github.com/SpringSource/spring-social/wiki/Quick-Start) is used for integration with **Twitter** 17 | - [**Apache Mahout**](http://mahout.apache.org/) 18 | - the project is **not stand-alone** - it uses the [java-stackexchange](https://github.com/eugenp/java-stackexchange) project to consume the StackExchange REST API. 19 | 20 | 21 | ## Other Resources 22 | - [Deployment](https://github.com/eugenp/stackexchange2twitter/wiki/Deployment) 23 | - [Process - New Account](https://github.com/eugenp/stackexchange2twitter/wiki/Process---New-Account) 24 | - [Process - Setup](https://github.com/eugenp/stackexchange2twitter/wiki/Process---Setup) 25 | 26 | 27 | - [**the CI on cloudbees**](https://rest-security.ci.cloudbees.com/view/twitter/) 28 | 29 | ### Relevant Articles: 30 | - [Tweeting StackExchange Questions with Spring Social](https://www.baeldung.com/2013/02/25/tweeting-stackexchange-with-spring-social-part-1) 31 | - [Spring Social Twitter Setup](https://www.baeldung.com/spring-social-twitter-setup) 32 | -------------------------------------------------------------------------------- /java-stackexchange/.gitignore: -------------------------------------------------------------------------------- 1 | */bin/* 2 | 3 | *.class 4 | 5 | # Package Files # 6 | *.war 7 | *.ear 8 | 9 | 10 | # Eclipse 11 | .settings/ 12 | *.project 13 | *.classpath 14 | .prefs 15 | *.prefs 16 | .metadata/ 17 | 18 | # Intellij 19 | .idea/ 20 | *.iml 21 | *.iws 22 | 23 | # Mac 24 | .DS_Store 25 | 26 | # Maven 27 | log/ 28 | target/ -------------------------------------------------------------------------------- /java-stackexchange/README.md: -------------------------------------------------------------------------------- 1 | java-stackexchange 2 | ================== 3 | 4 | This is a simple Java Client for the Stackexchange 2.x API 5 | 6 | # Goals 7 | - interact with the Questions API of Stack Exchange 8 | - this interaction is read-only - it doesn't deal with the authentication and OAuth parts of the Stackexchange API 9 | 10 | ### Relevant Articles: 11 | 12 | - [StackExchange REST Client with Spring and RestTemplate](http://www.baeldung.com/tweeting-stackexchange-with-spring-social-part-1) 13 | 14 | -------------------------------------------------------------------------------- /java-stackexchange/src/main/java/org/stackexchange/api/client/ApiUris.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.api.client; 2 | 3 | import org.stackexchange.api.constants.ApiConstants.Questions; 4 | import org.stackexchange.api.constants.StackSite; 5 | 6 | public class ApiUris { 7 | private static final String API_2_1 = "https://api.stackexchange.com/2.1"; 8 | 9 | private ApiUris() { 10 | throw new AssertionError(); 11 | } 12 | 13 | // API 14 | 15 | public static String getQuestionsUri(final int min, final StackSite site) { 16 | return getQuestionsUri(min, site, 1); 17 | } 18 | 19 | public static String getQuestionsUri(final int min, final StackSite site, final int page) { 20 | return getMultipleUri(min, site, "/questions", page); 21 | } 22 | 23 | public static String getSingleQuestionUri(final StackSite site, final long id) { 24 | final String operation = "/questions/" + id; 25 | return getSingleUri(site, operation); 26 | } 27 | 28 | public static String getTagUri(final int min, final StackSite site, final String tag) { 29 | return getTagUri(min, site, tag, 1); 30 | } 31 | 32 | public static String getTagUri(final int min, final StackSite site, final String tag, final int page) { 33 | return getMultipleUri(min, site, "/tags/" + tag + "/faq", page); 34 | } 35 | 36 | // util 37 | 38 | static String getMultipleUri(final int min, final StackSite site, final String operation, final int page) { 39 | final String params = new RequestBuilder().add(Questions.order, "desc").add(Questions.sort, "votes").add(Questions.min, min).add(Questions.site, site).add(Questions.page, page).build(); 40 | return API_2_1 + operation + params; 41 | } 42 | 43 | static String getSingleUri(final StackSite site, final String operation) { 44 | final String params = new RequestBuilder().add(Questions.site, site).build(); 45 | return API_2_1 + operation + params; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /java-stackexchange/src/main/java/org/stackexchange/api/client/RequestBuilder.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.api.client; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | import java.util.Map.Entry; 6 | 7 | import org.apache.http.client.utils.URIBuilder; 8 | 9 | public class RequestBuilder { 10 | 11 | private Map parameters; 12 | 13 | public RequestBuilder() { 14 | this.parameters = new HashMap<>(); 15 | } 16 | 17 | // API 18 | 19 | public final RequestBuilder add(final String paramName, final Object paramValue) { 20 | this.parameters.put(paramName, paramValue); 21 | return this; 22 | } 23 | 24 | public final String build() { 25 | final URIBuilder uriBuilder = new URIBuilder(); 26 | for (final Entry param : this.parameters.entrySet()) { 27 | uriBuilder.addParameter(param.getKey(), param.getValue().toString()); 28 | } 29 | 30 | return uriBuilder.toString(); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /java-stackexchange/src/main/java/org/stackexchange/api/constants/ApiConstants.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.api.constants; 2 | 3 | public final class ApiConstants { 4 | 5 | private ApiConstants() { 6 | throw new AssertionError(); 7 | } 8 | 9 | public class Questions { 10 | public static final String order = "order"; 11 | public static final String sort = "sort"; 12 | public static final String min = "min"; 13 | public static final String max = "max"; 14 | public static final String site = "site"; 15 | public static final String page = "page"; 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /java-stackexchange/src/main/java/org/stackexchange/api/constants/StackSite.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.api.constants; 2 | 3 | public enum StackSite { 4 | StackOverflow, ServerFault, AskUbuntu, SuperUser 5 | } 6 | -------------------------------------------------------------------------------- /java-stackexchange/src/main/java/org/stackexchange/api/model/Question.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.api.model; 2 | 3 | import java.util.Date; 4 | 5 | public class Question { 6 | 7 | /* 8 | "question_id": 14841260, 9 | "last_edit_date": 1360701401, 10 | "creation_date": 1360701076, 11 | "last_activity_date": 1360701681, 12 | "score": 0, 13 | "answer_count": 2, 14 | "title": "AsyncTask process", 15 | "tags": [ 16 | "java", 17 | "android", 18 | "multithreading", 19 | "user-interface", 20 | "android-asynctask" 21 | ], 22 | "view_count": 16, 23 | "owner": { 24 | "user_id": 1204617, 25 | "display_name": "Mike", 26 | "reputation": 50, 27 | "user_type": "registered", 28 | "profile_image": "http://www.gravatar.com/avatar/419e0876a47cb109c149639f8ef3130f?d=identicon&r=PG", 29 | "link": "http://stackoverflow.com/users/1204617/mike", 30 | "accept_rate": 92 31 | }, 32 | "link": "http://stackoverflow.com/questions/14841260/asynctask-process", 33 | "is_answered": false 34 | */ 35 | 36 | private long questionId; 37 | private Date lastEditDate; 38 | private Date lastActivityDate; 39 | 40 | } 41 | -------------------------------------------------------------------------------- /java-stackexchange/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/.gitignore: -------------------------------------------------------------------------------- 1 | */bin/* 2 | 3 | *.class 4 | 5 | # Package Files # 6 | *.war 7 | *.ear 8 | 9 | 10 | # Eclipse 11 | .settings/ 12 | *.project 13 | *.classpath 14 | .prefs 15 | *.prefs 16 | .metadata/ 17 | 18 | # Intellij 19 | .idea/ 20 | *.iml 21 | *.iws 22 | 23 | # Mac 24 | .DS_Store 25 | 26 | # Maven 27 | log/ 28 | target/ -------------------------------------------------------------------------------- /java-stackexchange2twitter/key.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/java-stackexchange2twitter/key.p12 -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/classification/data/SimpleDataLoadingStrategy.java: -------------------------------------------------------------------------------- 1 | package org.classification.data; 2 | 3 | import static org.classification.data.GenericClassificationDataUtil.trainingData; 4 | 5 | import java.io.IOException; 6 | import java.util.List; 7 | 8 | import org.apache.commons.lang3.tuple.ImmutablePair; 9 | import org.apache.mahout.math.NamedVector; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import com.google.common.collect.Lists; 14 | 15 | public class SimpleDataLoadingStrategy { 16 | private final Logger logger = LoggerFactory.getLogger(getClass()); 17 | 18 | private String singlePath; 19 | private String type; 20 | 21 | public SimpleDataLoadingStrategy(final String singlePath, final String type) { 22 | super(); 23 | 24 | this.singlePath = singlePath; 25 | this.type = type; 26 | } 27 | 28 | // API 29 | 30 | public final List loadTrainData(final int probes, final int features) { 31 | return loadTrainDataInternal(probes, features); 32 | } 33 | 34 | public final List> loadTestData() { 35 | return loadTestDataInternal(); 36 | } 37 | 38 | private final List> loadTestDataInternal() { 39 | try { 40 | return GenericClassificationDataUtil.testData(singlePath, type); 41 | } catch (final IOException ioEx) { 42 | logger.error("Data could not be loaded", ioEx); 43 | return Lists.newArrayList(); 44 | } 45 | } 46 | 47 | private final List loadTrainDataInternal(final int probes, final int features) { 48 | try { 49 | return trainingData(singlePath, type, probes, features); 50 | } catch (final IOException ioEx) { 51 | logger.error("Data could not be loaded", ioEx); 52 | return Lists.newArrayList(); 53 | } 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/classification/spring/ClassificationConfig.java: -------------------------------------------------------------------------------- 1 | package org.classification.spring; 2 | 3 | import org.springframework.context.annotation.ComponentScan; 4 | import org.springframework.context.annotation.Configuration; 5 | 6 | @Configuration 7 | @ComponentScan({ "org.classification.service" }) 8 | public class ClassificationConfig { 9 | 10 | public ClassificationConfig() { 11 | super(); 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/classification/util/ClassificationSettings.java: -------------------------------------------------------------------------------- 1 | package org.classification.util; 2 | 3 | public final class ClassificationSettings { 4 | 5 | /** 6 | * - TODO: figure out what this is exactly 7 | */ 8 | public static final int PROBES_FOR_CONTENT_ENCODER_VECTOR = 8; 9 | public static final int LEARNERS_IN_THE_CLASSIFIER_POOL = 50; 10 | public static final int FEATURES = 7000; 11 | 12 | /* 13 | # Actual Scores (training data - core vs full) 14 | - 5000: 15 | -- 2 - 0.XXX 16 | -- 3 - 0.847 17 | -- 5 - 0.864 18 | -- 6 - 0.869 19 | - 6000: 20 | -- 7 - 0.889 21 | -- 8 - 0.903 22 | -- 9 - 0.903 23 | - 7000: 24 | -- 3 - 0.903 25 | -- 4 - 0.909 26 | -- 5 - 0.911 27 | -- 6 - 0.914 28 | -- 7 - 0.914 29 | -- 8 - 0.926,0.920 (0.910) 30 | -- 9 - 0.921 31 | - 8000 32 | -- 5 - 0.904 33 | -- 6 - 0.898 34 | -- 7 - 0.868 35 | -- 8 - 0.874 36 | -- 9 - 0.868 37 | */ 38 | 39 | /* 40 | # Vary the size of the pool 41 | - pool = 50 - 0.789 42 | - pool = 100 - 0.806 43 | - pool = 150 - 0.812,0.829 44 | - pool = 200 - 45 | * - note: it's not important how well they perform, just how much is the gap between 50-200 46 | */ 47 | 48 | /* 49 | # Vary the size of the runs 50 | - runs = 250 - 0.824 51 | - runs = 500 - 0.823 52 | - runs = 750 - 0.826 53 | - runs = 1000 - 0.826 54 | * - note: it's not important how well they perform, just how much is the gap between runs is 55 | */ 56 | 57 | private ClassificationSettings() { 58 | throw new AssertionError(); 59 | } 60 | 61 | // util 62 | 63 | } 64 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/metrics/MetricRegistryFactoryBean.java: -------------------------------------------------------------------------------- 1 | package org.common.metrics; 2 | 3 | import java.util.concurrent.TimeUnit; 4 | 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.beans.factory.FactoryBean; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.core.env.Environment; 9 | import org.springframework.stereotype.Component; 10 | 11 | import com.codahale.metrics.MetricRegistry; 12 | import com.codahale.metrics.Slf4jReporter; 13 | 14 | @Component 15 | public final class MetricRegistryFactoryBean implements FactoryBean { 16 | 17 | @Autowired 18 | private Environment env; 19 | 20 | public MetricRegistryFactoryBean() { 21 | super(); 22 | } 23 | 24 | // API 25 | 26 | @Override 27 | public final MetricRegistry getObject() { 28 | final MetricRegistry metricRegistry = new MetricRegistry(); 29 | 30 | final Slf4jReporter reporter = Slf4jReporter.forRegistry(metricRegistry).outputTo(LoggerFactory.getLogger("org.common.metrics")).convertRatesTo(TimeUnit.SECONDS).convertDurationsTo(TimeUnit.MILLISECONDS).build(); 31 | 32 | final Integer metricsLogRate = env.getProperty("metrics.log.rate", Integer.class); 33 | reporter.start(metricsLogRate, TimeUnit.MINUTES); 34 | 35 | return metricRegistry; 36 | } 37 | 38 | @Override 39 | public final Class getObjectType() { 40 | return MetricRegistry.class; 41 | } 42 | 43 | @Override 44 | public final boolean isSingleton() { 45 | return true; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/metrics/MetricsUtil.java: -------------------------------------------------------------------------------- 1 | package org.common.metrics; 2 | 3 | import org.tweet.meta.service.TweetMetaLiveService; 4 | import org.tweet.twitter.service.live.TwitterReadLiveService; 5 | import org.tweet.twitter.service.live.TwitterWriteLiveService; 6 | 7 | import com.codahale.metrics.MetricRegistry; 8 | 9 | public final class MetricsUtil { 10 | 11 | public static final class Meta { 12 | private static final String OK = "ok"; 13 | private static final String ERROR = "err"; 14 | 15 | private static final String READ = "read"; 16 | private static final String WRITE = "write"; 17 | 18 | public static final String RETWEET_ANY_ERROR = MetricRegistry.name(TweetMetaLiveService.class, "any", ERROR); 19 | public static final String RETWEET_ONE_ERROR = MetricRegistry.name(TweetMetaLiveService.class, "one", ERROR); 20 | 21 | public static final String TWITTER_READ_OK = MetricRegistry.name(TwitterReadLiveService.class, READ, OK); 22 | public static final String TWITTER_READ_ERR = MetricRegistry.name(TwitterReadLiveService.class, READ, ERROR); 23 | 24 | public static final String TWITTER_WRITE_OK = MetricRegistry.name(TwitterWriteLiveService.class, WRITE, OK); 25 | public static final String TWITTER_WRITE_ERR = MetricRegistry.name(TwitterWriteLiveService.class, WRITE, ERROR); 26 | 27 | public static final String HTTP_OK = MetricRegistry.name(TwitterWriteLiveService.class, WRITE, OK); 28 | public static final String HTTP_ERR = MetricRegistry.name(TwitterWriteLiveService.class, WRITE, "orr"); 29 | } 30 | 31 | private MetricsUtil() { 32 | throw new AssertionError(); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/persistence/IEntity.java: -------------------------------------------------------------------------------- 1 | package org.common.persistence; 2 | 3 | import java.io.Serializable; 4 | 5 | public interface IEntity extends Serializable { 6 | 7 | long getId(); 8 | 9 | void setId(long id); 10 | 11 | } 12 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/persistence/IOperations.java: -------------------------------------------------------------------------------- 1 | package org.common.persistence; 2 | 3 | public interface IOperations extends IRawOperations { 4 | // 5 | } 6 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/persistence/IRawOperations.java: -------------------------------------------------------------------------------- 1 | package org.common.persistence; 2 | 3 | import org.springframework.data.repository.CrudRepository; 4 | 5 | public interface IRawOperations extends CrudRepository { 6 | // 7 | } 8 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/persistence/meta/SearchField.java: -------------------------------------------------------------------------------- 1 | package org.common.persistence.meta; 2 | 3 | public enum SearchField { 4 | id, name, // common 5 | uuid, 6 | } 7 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/persistence/setup/AfterSetupEvent.java: -------------------------------------------------------------------------------- 1 | package org.common.persistence.setup; 2 | 3 | import org.springframework.context.ApplicationEvent; 4 | 5 | public final class AfterSetupEvent extends ApplicationEvent { 6 | 7 | public AfterSetupEvent(final Object sourceToSet) { 8 | super(sourceToSet); 9 | } 10 | 11 | // API 12 | 13 | } 14 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/persistence/setup/BeforeSetupEvent.java: -------------------------------------------------------------------------------- 1 | package org.common.persistence.setup; 2 | 3 | import org.springframework.context.ApplicationEvent; 4 | 5 | public final class BeforeSetupEvent extends ApplicationEvent { 6 | 7 | public BeforeSetupEvent(final Object sourceToSet) { 8 | super(sourceToSet); 9 | } 10 | 11 | // API 12 | 13 | } 14 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/persistence/setup/checks/CredentialsForAllTwitterAccountsExistChecker.java: -------------------------------------------------------------------------------- 1 | package org.common.persistence.setup.checks; 2 | 3 | import org.common.persistence.setup.AfterSetupEvent; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.context.ApplicationListener; 8 | import org.springframework.context.annotation.Profile; 9 | import org.springframework.core.env.Environment; 10 | import org.springframework.scheduling.annotation.Async; 11 | import org.springframework.stereotype.Component; 12 | import org.stackexchange.util.TwitterAccountEnum; 13 | import org.tweet.spring.util.SpringProfileUtil; 14 | import org.tweet.twitter.service.TwitterTemplateCreator; 15 | 16 | @Component 17 | @Profile(SpringProfileUtil.DEPLOYED) 18 | class CredentialsForAllTwitterAccountsExistChecker implements ApplicationListener { 19 | private final Logger logger = LoggerFactory.getLogger(getClass()); 20 | 21 | @Autowired 22 | private TwitterTemplateCreator twitterTemplateCreator; 23 | 24 | @Autowired 25 | private Environment env; 26 | 27 | public CredentialsForAllTwitterAccountsExistChecker() { 28 | super(); 29 | } 30 | 31 | // API 32 | 33 | @Override 34 | @Async 35 | public void onApplicationEvent(final AfterSetupEvent event) { 36 | logger.info("Starting to execute the CredentialsForAllTwitterAccountsExistChecker Checker"); 37 | 38 | for (final TwitterAccountEnum twitterAccount : TwitterAccountEnum.values()) { 39 | twitterTemplateCreator.createTwitterTemplate(twitterAccount.name()); 40 | } 41 | 42 | logger.info("Finished executing the CredentialsForAllTwitterAccountsExistChecker Checker"); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/persistence/setup/upgrades/live/IRecreateMissingQuestionTweetsUpgrader.java: -------------------------------------------------------------------------------- 1 | package org.common.persistence.setup.upgrades.live; 2 | 3 | import org.springframework.social.twitter.api.Tweet; 4 | 5 | public interface IRecreateMissingQuestionTweetsUpgrader { 6 | 7 | void recreateLocalQuestionTweetsFromLiveTweets(); 8 | 9 | void recreateLocalQuestionTweetsOnAccount(final String twitterAccount); 10 | 11 | String extractQuestionIdFromTweet(Tweet tweet); 12 | 13 | } 14 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/persistence/setup/upgrades/live/IRecreateMissingRetweetsUpgrader.java: -------------------------------------------------------------------------------- 1 | package org.common.persistence.setup.upgrades.live; 2 | 3 | public interface IRecreateMissingRetweetsUpgrader { 4 | 5 | void recreateLocalRetweetsFromLiveTweets(); 6 | 7 | void recreateLocalRetweetsFromLiveTweetsOnAccount(final String twitterAccount); 8 | 9 | } 10 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/persistence/setup/upgrades/live/IRemoveOrphanedRetweetsUpgrader.java: -------------------------------------------------------------------------------- 1 | package org.common.persistence.setup.upgrades.live; 2 | 3 | public interface IRemoveOrphanedRetweetsUpgrader { 4 | 5 | void removeOrphanedRetweets(); 6 | 7 | void removeOrphanedRetweetsOnAccount(final String twitterAccount); 8 | 9 | } 10 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/persistence/setup/upgrades/live/nolonger/IAddTextToRetweetsUpgrader.java: -------------------------------------------------------------------------------- 1 | package org.common.persistence.setup.upgrades.live.nolonger; 2 | 3 | /** 4 | * Iterates though all the local Retweets
5 | * Ads text (from live corresponding tweet) if the local retweet doesn't have text
6 | * - use: no longer - all Retweets now have text (plus constraint) 7 | */ 8 | public interface IAddTextToRetweetsUpgrader { 9 | 10 | void addTextOfRetweets(); 11 | 12 | boolean addTextOfRetweetsOnAccount(final String twitterAccount); 13 | 14 | } 15 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/persistence/setup/upgrades/live/nolonger/IFillInDatesOfRetweetsUpgrader.java: -------------------------------------------------------------------------------- 1 | package org.common.persistence.setup.upgrades.live.nolonger; 2 | 3 | /** 4 | * Iterates though all the live Tweets
5 | * Finds their corresponding local Retweet and adds the date to it (if it doesn't already have it)
6 | * - use: no longer - most Retweets now have dates - and the ones that do not, are orphans and need to be removed anyways 7 | */ 8 | public interface IFillInDatesOfRetweetsUpgrader { 9 | 10 | void fillInDatesOfRetweetsOfAllAccounts(); 11 | 12 | boolean fillInDatesOfRetweetsOfOneAccount(final String twitterAccount); 13 | 14 | } 15 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/persistence/setup/upgrades/live/nolonger/IRemoveDuplicateRetweetsUpgrader.java: -------------------------------------------------------------------------------- 1 | package org.common.persistence.setup.upgrades.live.nolonger; 2 | 3 | /** 4 | * Iterates though all the live Tweets
5 | * If any of these have more than one corresponding local Retweet - it removes the duplicates
6 | * - use: very unlikely - last pass it found a single occurrence, and that should be found by @IRemoveLocalDuplicateRetweetsUpgrader 7 | */ 8 | public interface IRemoveDuplicateRetweetsUpgrader { 9 | 10 | void removeDuplicateLocalRetweets(); 11 | 12 | boolean removeDuplicateRetweetsOnAccount(final String twitterAccount); 13 | 14 | } 15 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/persistence/setup/upgrades/local/ICleanTextOfRetweetsUpgrader.java: -------------------------------------------------------------------------------- 1 | package org.common.persistence.setup.upgrades.local; 2 | 3 | /** 4 | * Iterates though all the local Retweets
5 | * Cleans their text with the latest text cleanup rules (which are frequently updated)
6 | * - use: regularly 7 | */ 8 | public interface ICleanTextOfRetweetsUpgrader { 9 | 10 | void cleanTextOfRetweets(); 11 | 12 | boolean cleanTextOfRetweetsOnAccount(String twitterAccount); 13 | 14 | } 15 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/persistence/setup/upgrades/local/IRemoveLocalDuplicateRetweetsUpgrader.java: -------------------------------------------------------------------------------- 1 | package org.common.persistence.setup.upgrades.local; 2 | 3 | /** 4 | * Iterates though all the local Retweets
5 | * Removes exact duplicates (by text and account) for each of these
6 | * - use: maybe - but shouldn't really do anything
7 | * 8 | * - local 9 | */ 10 | public interface IRemoveLocalDuplicateRetweetsUpgrader { 11 | 12 | void removeLocalDuplicateRetweets(); 13 | 14 | boolean removeLocalDuplicateRetweetsOnAccount(final String twitterAccount); 15 | 16 | } 17 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/service/AnalysisService.java: -------------------------------------------------------------------------------- 1 | package org.common.service; 2 | 3 | import org.springframework.stereotype.Service; 4 | 5 | @Service 6 | public class AnalysisService implements IAnalysisService { 7 | 8 | public AnalysisService() { 9 | super(); 10 | } 11 | 12 | // API 13 | 14 | @Override 15 | public final double determineSimillarity(final String from, final String to) { 16 | return 0.0; 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/service/BaseTweetFromSourceLiveService.java: -------------------------------------------------------------------------------- 1 | package org.common.service; 2 | 3 | import java.util.Map; 4 | 5 | import org.common.persistence.IEntity; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.data.jpa.repository.JpaRepository; 8 | import org.tweet.twitter.service.AdvancedTweetService; 9 | import org.tweet.twitter.service.TweetLiveService; 10 | import org.tweet.twitter.service.TweetService; 11 | import org.tweet.twitter.service.live.ITwitterWriteLiveService; 12 | import org.tweet.twitter.service.live.TwitterReadLiveService; 13 | 14 | public abstract class BaseTweetFromSourceLiveService { 15 | 16 | @Autowired 17 | protected TwitterReadLiveService twitterReadLiveService; 18 | @Autowired 19 | protected ITwitterWriteLiveService twitterWriteLiveService; 20 | 21 | @Autowired 22 | protected TweetService tweetService; 23 | 24 | @Autowired 25 | protected AdvancedTweetService advancedTweetService; 26 | 27 | @Autowired 28 | protected TweetLiveService tweetLiveService; 29 | 30 | // API 31 | 32 | // TODO: make this consistent - url should probably be removed, the text should be clear - text only (without url) or full tweet 33 | protected abstract boolean tryTweetOne(final String text, final String url, final String twitterAccount, final Map customDetails); 34 | 35 | /** 36 | * - this should decide based also on the twitterAccount - currently, the implementations of T should actually contain the twitterAccount information 37 | */ 38 | protected abstract boolean hasThisAlreadyBeenTweetedById(final T entity); 39 | 40 | protected abstract void markDone(final T entity); 41 | 42 | protected abstract JpaRepository getApi(); 43 | 44 | } 45 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/service/IAnalysisService.java: -------------------------------------------------------------------------------- 1 | package org.common.service; 2 | 3 | public interface IAnalysisService { 4 | 5 | double determineSimillarity(final String from, final String to); 6 | 7 | } 8 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/service/external/bitly/BitlyService.java: -------------------------------------------------------------------------------- 1 | package org.common.service.external.bitly; 2 | 3 | import net.swisstech.bitly.BitlyClient; 4 | import net.swisstech.bitly.model.Response; 5 | import net.swisstech.bitly.model.v3.ShortenResponse; 6 | 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import org.springframework.stereotype.Service; 10 | 11 | @Service 12 | public class BitlyService { 13 | private final Logger logger = LoggerFactory.getLogger(getClass()); 14 | 15 | public BitlyService() { 16 | super(); 17 | } 18 | 19 | // API 20 | 21 | public final String shortenUrl(final String urlLarge) { 22 | try { 23 | return shortenUrlInternal(urlLarge); 24 | } catch (final RuntimeException runEx) { 25 | logger.error("Unable to shorten url= " + urlLarge + " via bitly!", runEx); 26 | return urlLarge; 27 | } 28 | } 29 | 30 | private final String shortenUrlInternal(final String urlLarge) { 31 | final String accessToken = "caf0026313bcf9709abfcc08a2270b55953bc95c"; 32 | final Response shortenResponse = new BitlyClient(accessToken).shorten().setLongUrl(urlLarge).call(); 33 | return shortenResponse.data.url; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/spring/CommonMetricsConfig.java: -------------------------------------------------------------------------------- 1 | package org.common.spring; 2 | 3 | import org.springframework.context.annotation.ComponentScan; 4 | import org.springframework.context.annotation.Configuration; 5 | 6 | @Configuration 7 | @ComponentScan({ "org.common.metrics" }) 8 | public class CommonMetricsConfig { 9 | 10 | public CommonMetricsConfig() { 11 | super(); 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/spring/CommonPersistenceJPAConfig.java: -------------------------------------------------------------------------------- 1 | package org.common.spring; 2 | 3 | import org.springframework.context.annotation.ComponentScan; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.context.annotation.Import; 6 | import org.springframework.context.annotation.ImportResource; 7 | import org.springframework.context.annotation.Profile; 8 | import org.springframework.scheduling.annotation.EnableAsync; 9 | import org.tweet.spring.util.SpringProfileUtil; 10 | 11 | @Configuration 12 | @Import(PersistenceJPACommonConfig.class) 13 | @ComponentScan({ "org.common.persistence" }) 14 | @ImportResource("classpath*:commonPersistenceConfig.xml") 15 | @EnableAsync 16 | // (proxyTargetClass = true) 17 | @Profile(SpringProfileUtil.PERSISTENCE) 18 | public class CommonPersistenceJPAConfig { 19 | 20 | public CommonPersistenceJPAConfig() { 21 | super(); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/spring/CommonServiceConfig.java: -------------------------------------------------------------------------------- 1 | package org.common.spring; 2 | 3 | import org.springframework.context.annotation.ComponentScan; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.context.annotation.Import; 6 | import org.springframework.context.annotation.PropertySource; 7 | 8 | @Configuration 9 | @ComponentScan({ "org.common.service" }) 10 | @Import(CommonMetricsConfig.class) 11 | @PropertySource({ "classpath:config-${envTarget:dev}.properties" }) 12 | public class CommonServiceConfig { 13 | 14 | public CommonServiceConfig() { 15 | super(); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/spring/MyApplicationContextInitializerProv.java: -------------------------------------------------------------------------------- 1 | package org.common.spring; 2 | 3 | import java.util.List; 4 | 5 | import com.google.common.collect.Lists; 6 | 7 | public class MyApplicationContextInitializerProv extends MyBaseApplicationContextInitializer { 8 | 9 | private static final String ENV_TARGET_KEY = "envTarget"; 10 | public static final String PERSISTENCE_TARGET_KEY = "persistenceTarget"; 11 | private static final String DEFAULT_VALUE = "production"; 12 | 13 | private final List validEnvTargets = Lists.newArrayList("dev", "production"); 14 | 15 | // template 16 | 17 | @Override 18 | protected String getEnvTargetKey() { 19 | return ENV_TARGET_KEY; 20 | } 21 | 22 | @Override 23 | protected String getEnvTargetVal() { 24 | return DEFAULT_VALUE; 25 | } 26 | 27 | @Override 28 | protected String getPersistenceTargetKey() { 29 | return PERSISTENCE_TARGET_KEY; 30 | } 31 | 32 | @Override 33 | protected List getValidEnvTargetValues() { 34 | return validEnvTargets; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/common/util/order/OrderById.java: -------------------------------------------------------------------------------- 1 | package org.common.util.order; 2 | 3 | import org.common.persistence.IEntity; 4 | 5 | import com.google.common.collect.Ordering; 6 | import com.google.common.primitives.Longs; 7 | 8 | public final class OrderById extends Ordering { 9 | 10 | public OrderById() { 11 | super(); 12 | } 13 | 14 | // API 15 | 16 | @Override 17 | public final int compare(final T left, final T right) { 18 | return Longs.compare(left.getId(), right.getId()); 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/gplus/spring/GplusContextConfig.java: -------------------------------------------------------------------------------- 1 | package org.gplus.spring; 2 | 3 | import org.springframework.context.annotation.ComponentScan; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.scheduling.annotation.EnableScheduling; 6 | 7 | @Configuration 8 | @EnableScheduling 9 | @ComponentScan({ "org.gplus" }) 10 | public class GplusContextConfig { 11 | 12 | public GplusContextConfig() { 13 | super(); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/gplus/stackexchange/GplusApiFactoryBean.java: -------------------------------------------------------------------------------- 1 | package org.gplus.stackexchange; 2 | 3 | import java.io.File; 4 | import java.util.Collections; 5 | 6 | import org.springframework.beans.factory.FactoryBean; 7 | import org.springframework.stereotype.Component; 8 | 9 | import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; 10 | import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; 11 | import com.google.api.client.http.HttpTransport; 12 | import com.google.api.client.json.JsonFactory; 13 | import com.google.api.client.json.jackson2.JacksonFactory; 14 | import com.google.api.services.plus.Plus; 15 | import com.google.api.services.plus.PlusScopes; 16 | 17 | @Component 18 | public class GplusApiFactoryBean implements FactoryBean { 19 | 20 | private HttpTransport httpTransport; 21 | 22 | private final JsonFactory jsonFactory = new JacksonFactory(); 23 | 24 | /** E-mail address of the service account. */ 25 | private final String serviceAccountEmail = "197652125488-ju7ev3nc4lcmdlhr83gjrhb7gk03g552@developer.gserviceaccount.com"; 26 | 27 | private final String APPLICATION_NAME = "BestScala"; 28 | 29 | // API 30 | 31 | @Override 32 | public Plus getObject() throws Exception { 33 | httpTransport = GoogleNetHttpTransport.newTrustedTransport(); 34 | // service account credential (uncomment setServiceAccountUser for domain-wide delegation) 35 | final GoogleCredential credential = new GoogleCredential.Builder().setTransport(httpTransport).setJsonFactory(jsonFactory).setServiceAccountId(serviceAccountEmail).setServiceAccountScopes(Collections.singleton(PlusScopes.PLUS_ME)) 36 | .setServiceAccountPrivateKeyFromP12File(new File("key.p12")) 37 | // .setServiceAccountUser("user@example.com") 38 | .build(); 39 | 40 | return new Plus.Builder(httpTransport, jsonFactory, credential).setApplicationName(APPLICATION_NAME).build(); 41 | } 42 | 43 | @Override 44 | public Class getObjectType() { 45 | return Plus.class; 46 | } 47 | 48 | @Override 49 | public boolean isSingleton() { 50 | return true; 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/keyval/persistence/dao/IKeyValJpaDAO.java: -------------------------------------------------------------------------------- 1 | package org.keyval.persistence.dao; 2 | 3 | import org.common.persistence.IOperations; 4 | import org.keyval.persistence.model.KeyVal; 5 | import org.springframework.data.jpa.repository.JpaRepository; 6 | 7 | public interface IKeyValJpaDAO extends JpaRepository, IOperations { 8 | 9 | KeyVal findByKey(final String key); 10 | 11 | } 12 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/keyval/spring/KeyValPersistenceJPAConfig.java: -------------------------------------------------------------------------------- 1 | package org.keyval.spring; 2 | 3 | import org.common.spring.PersistenceJPACommonConfig; 4 | import org.springframework.context.annotation.ComponentScan; 5 | import org.springframework.context.annotation.Configuration; 6 | import org.springframework.context.annotation.Import; 7 | import org.springframework.context.annotation.Profile; 8 | import org.springframework.data.jpa.repository.config.EnableJpaRepositories; 9 | import org.tweet.spring.util.SpringProfileUtil; 10 | 11 | @Configuration 12 | @Import(PersistenceJPACommonConfig.class) 13 | @ComponentScan({ "org.keyval.persistence" }) 14 | @EnableJpaRepositories("org.keyval.persistence.dao") 15 | @Profile(SpringProfileUtil.PERSISTENCE) 16 | public class KeyValPersistenceJPAConfig { 17 | 18 | public KeyValPersistenceJPAConfig() { 19 | super(); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/rss/RssScheduler.java: -------------------------------------------------------------------------------- 1 | package org.rss; 2 | 3 | import java.io.IOException; 4 | 5 | import org.rss.service.TweetRssLiveService; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.context.annotation.Profile; 10 | import org.springframework.core.env.Environment; 11 | import org.springframework.scheduling.annotation.Scheduled; 12 | import org.tweet.spring.util.SpringProfileUtil; 13 | 14 | import com.fasterxml.jackson.core.JsonProcessingException; 15 | 16 | // @Component 17 | @Profile(SpringProfileUtil.WRITE_PRODUCTION) 18 | public class RssScheduler { 19 | private final Logger logger = LoggerFactory.getLogger(getClass()); 20 | 21 | @Autowired 22 | private TweetRssLiveService service; 23 | 24 | @Autowired 25 | private Environment env; 26 | 27 | public RssScheduler() { 28 | super(); 29 | } 30 | 31 | // API 32 | @Scheduled(cron = "0 5 */1 * * *") 33 | public void tweetMeta1() throws JsonProcessingException, IOException { 34 | logger.info("Starting to execute scheduled RSS operations - 1"); 35 | 36 | if (env.getProperty("mode.maintainance.rss", Boolean.class)) { 37 | logger.warn("Maintainance Mode Active - skipping schedule"); 38 | return; 39 | } 40 | 41 | // service.tweetFromRss("http://feeds.feedburner.com/SpringSourceTeamBlog", TwitterAccountEnum.SpringTip.name(), "SpringSource"); 42 | // service.tweetFromRss("http://feeds.feedburner.com/springsource/OEVE", TwitterAccountEnum.SpringTip.name(), "SpringSource"); 43 | 44 | logger.info("Finished executing scheduled RSS operations - 1"); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/rss/persistence/dao/IRssEntryJpaDAO.java: -------------------------------------------------------------------------------- 1 | package org.rss.persistence.dao; 2 | 3 | import org.common.persistence.IOperations; 4 | import org.rss.persistence.model.RssEntry; 5 | import org.springframework.data.jpa.repository.JpaRepository; 6 | 7 | public interface IRssEntryJpaDAO extends JpaRepository, IOperations { 8 | 9 | RssEntry findOneByLinkAndTwitterAccount(final String link, final String twitterAccount); 10 | 11 | RssEntry findOneByLink(final String link); 12 | 13 | } 14 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/rss/spring/RssContextConfig.java: -------------------------------------------------------------------------------- 1 | package org.rss.spring; 2 | 3 | import org.springframework.context.annotation.ComponentScan; 4 | import org.springframework.context.annotation.Configuration; 5 | 6 | @Configuration 7 | @ComponentScan({ "org.rss" }) 8 | public class RssContextConfig { 9 | 10 | public RssContextConfig() { 11 | super(); 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/rss/spring/RssPersistenceJPAConfig.java: -------------------------------------------------------------------------------- 1 | package org.rss.spring; 2 | 3 | import org.common.spring.PersistenceJPACommonConfig; 4 | import org.springframework.context.annotation.ComponentScan; 5 | import org.springframework.context.annotation.Configuration; 6 | import org.springframework.context.annotation.Import; 7 | import org.springframework.context.annotation.ImportResource; 8 | import org.springframework.context.annotation.Profile; 9 | import org.springframework.context.annotation.PropertySource; 10 | import org.tweet.spring.util.SpringProfileUtil; 11 | 12 | @Configuration 13 | @Import(PersistenceJPACommonConfig.class) 14 | @ComponentScan({ "org.rss.persistence" }) 15 | @ImportResource("classpath*:rssPersistenceConfig.xml") 16 | @PropertySource({ "classpath:persistence-${persistenceTarget:prod}.properties" }) 17 | @Profile(SpringProfileUtil.PERSISTENCE) 18 | public class RssPersistenceJPAConfig { 19 | 20 | public RssPersistenceJPAConfig() { 21 | super(); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/stackexchange/component/MinStackScoreRetriever.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.component; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.core.env.Environment; 5 | import org.springframework.stereotype.Component; 6 | import org.stackexchange.api.constants.StackSite; 7 | 8 | import com.google.common.base.Preconditions; 9 | 10 | @Component 11 | public final class MinStackScoreRetriever { 12 | 13 | @Autowired 14 | private Environment env; 15 | 16 | public MinStackScoreRetriever() { 17 | super(); 18 | } 19 | 20 | // API 21 | 22 | // by tag and site 23 | 24 | public final int minScore(final String stackTag, final StackSite stackSite, final String twitterAccount) { 25 | Preconditions.checkNotNull(minScoreRaw(stackTag, stackSite), "Unable to find minscore by stackTag= " + stackTag + " and stackSite= " + stackSite + " on twitterAccount= " + twitterAccount); 26 | return env.getProperty(stackTag + "." + stackSite.name() + ".minscore", Integer.class); 27 | } 28 | 29 | final String minScoreRaw(final String stackTag, final StackSite stackSite) { 30 | return env.getProperty(stackTag + "." + stackSite.name() + ".minscore"); 31 | } 32 | 33 | // by site 34 | 35 | public final int minScoreByAccount(final String twitterAccount) { 36 | Preconditions.checkNotNull(minScoreRawByAccount(twitterAccount), "Unable to find minscore by twitterAccount= " + twitterAccount); 37 | return env.getProperty(twitterAccount + ".minscore", Integer.class); 38 | } 39 | 40 | final String minScoreRawByAccount(final String twitterAccount) { 41 | return env.getProperty(twitterAccount + ".minscore"); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/stackexchange/component/StackExchangePageStrategy.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.component; 2 | 3 | import org.apache.commons.lang.math.RandomUtils; 4 | import org.springframework.beans.factory.annotation.Autowired; 5 | import org.springframework.context.annotation.Profile; 6 | import org.springframework.stereotype.Component; 7 | import org.stackexchange.persistence.dao.IQuestionTweetJpaDAO; 8 | import org.tweet.spring.util.SpringProfileUtil; 9 | 10 | @Component 11 | @Profile(SpringProfileUtil.PERSISTENCE) 12 | public final class StackExchangePageStrategy { 13 | 14 | @Autowired 15 | private IQuestionTweetJpaDAO questionTweetApi; 16 | 17 | public StackExchangePageStrategy() { 18 | super(); 19 | } 20 | 21 | // API 22 | 23 | // TODO: implement and publish 24 | final int decidePage(final String twitterAccount, final String stackTag) { 25 | throw new UnsupportedOperationException(); 26 | } 27 | 28 | public final int decidePage(final String twitterAccount) { 29 | final int countAllByTwitterAccount = (int) questionTweetApi.countAllByTwitterAccount(twitterAccount); 30 | return decidePageInternal(countAllByTwitterAccount); 31 | } 32 | 33 | final int decidePageInternal(final int countAllByTwitterAccount) { 34 | final int page = countAllByTwitterAccount / 30; 35 | if (page < 3) { 36 | return page + 1; 37 | } 38 | 39 | // was + but, for a long time, a bug in the codebase meant that, from each of the resulted pages, only 1 question was actually tweeted, so now we're going back (for a while) 40 | // return page + RandomUtils.nextInt(4); 41 | return page - RandomUtils.nextInt(1); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/stackexchange/persistence/dao/IQuestionTweetJpaDAO.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.persistence.dao; 2 | 3 | import java.util.List; 4 | 5 | import org.common.persistence.IOperations; 6 | import org.springframework.data.jpa.repository.JpaRepository; 7 | import org.springframework.data.jpa.repository.Query; 8 | import org.stackexchange.persistence.model.QuestionTweet; 9 | 10 | public interface IQuestionTweetJpaDAO extends JpaRepository, IOperations { 11 | 12 | QuestionTweet findByQuestionId(final String questionId); 13 | 14 | QuestionTweet findByQuestionIdAndTwitterAccount(final String questionId, final String twitterAccount); 15 | 16 | List findAllByTwitterAccount(final String twitterAccount); 17 | 18 | @Query("select count(q) from QuestionTweet q where q.twitterAccount = ?1") 19 | long countAllByTwitterAccount(final String twitterAccount); 20 | 21 | } 22 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/stackexchange/spring/StackexchangeConfig.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.spring; 2 | 3 | import org.apache.http.client.HttpClient; 4 | import org.apache.http.impl.client.DecompressingHttpClient; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | import org.springframework.context.annotation.PropertySource; 8 | import org.stackexchange.api.client.HttpFactory; 9 | import org.stackexchange.api.client.QuestionsApi; 10 | 11 | @Configuration 12 | @PropertySource({ "classpath:stackexchange.properties" }) 13 | public class StackexchangeConfig { 14 | 15 | // 16 | 17 | @Bean 18 | public QuestionsApi questionsApi() { 19 | final HttpClient rawHttpClient = HttpFactory.httpClient(true); 20 | 21 | final DecompressingHttpClient httpClient = new DecompressingHttpClient(rawHttpClient); 22 | return new QuestionsApi(httpClient); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/stackexchange/spring/StackexchangeContextConfig.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.spring; 2 | 3 | import org.springframework.context.annotation.ComponentScan; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.scheduling.annotation.EnableScheduling; 6 | 7 | @Configuration 8 | @EnableScheduling 9 | @ComponentScan({ "org.stackexchange" }) 10 | public class StackexchangeContextConfig { 11 | 12 | public StackexchangeContextConfig() { 13 | super(); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/stackexchange/spring/StackexchangePersistenceJPAConfig.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.spring; 2 | 3 | import org.common.spring.PersistenceJPACommonConfig; 4 | import org.springframework.context.annotation.ComponentScan; 5 | import org.springframework.context.annotation.Configuration; 6 | import org.springframework.context.annotation.Import; 7 | import org.springframework.context.annotation.ImportResource; 8 | import org.springframework.context.annotation.Profile; 9 | import org.springframework.context.annotation.PropertySource; 10 | import org.tweet.spring.util.SpringProfileUtil; 11 | 12 | @Configuration 13 | @Import(PersistenceJPACommonConfig.class) 14 | @ComponentScan({ "org.stackexchange.persistence" }) 15 | @ImportResource("classpath*:stackPersistenceConfig.xml") 16 | @PropertySource({ "classpath:persistence-${persistenceTarget:prod}.properties", "classpath:setup.properties" }) 17 | @Profile(SpringProfileUtil.PERSISTENCE) 18 | public class StackexchangePersistenceJPAConfig { 19 | 20 | public StackexchangePersistenceJPAConfig() { 21 | super(); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/stackexchange/util/GenericUtil.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.util; 2 | 3 | import java.util.List; 4 | import java.util.Random; 5 | 6 | import com.google.common.base.Splitter; 7 | import com.google.common.collect.Lists; 8 | 9 | public final class GenericUtil { 10 | 11 | private GenericUtil() { 12 | throw new AssertionError(); 13 | } 14 | 15 | // API 16 | 17 | @SuppressWarnings("unchecked") 18 | public static T pickOneGeneric(final T... options) { 19 | return options[new Random().nextInt(options.length)]; 20 | } 21 | 22 | public static T pickOneGeneric(final List options) { 23 | return options.get(new Random().nextInt(options.size())); 24 | } 25 | 26 | /** 27 | * return - not null 28 | */ 29 | public static List breakApart(final String commaSeparated) { 30 | final Iterable split = Splitter.on(',').split(commaSeparated); 31 | return Lists.newArrayList(split); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/stackexchange/util/IDUtil.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.util; 2 | 3 | import java.util.Random; 4 | 5 | public final class IDUtil { 6 | 7 | private IDUtil() { 8 | throw new AssertionError(); 9 | } 10 | 11 | // API 12 | 13 | public final static String randomPositiveLongAsString() { 14 | return Long.toString(randomPositiveLong()); 15 | } 16 | 17 | public final static String randomNegativeLongAsString() { 18 | return Long.toString(randomNegativeLong()); 19 | } 20 | 21 | public final static long randomPositiveLong() { 22 | long id = new Random().nextLong() * 10000; 23 | id = (id < 0) ? (-1 * id) : id; 24 | return id; 25 | } 26 | 27 | public final static long randomNegativeLong() { 28 | long id = new Random().nextLong() * 10000; 29 | id = (id > 0) ? (-1 * id) : id; 30 | return id; 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/stackexchange/util/StackTag.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.util; 2 | 3 | public enum StackTag {// @formatter:off 4 | 5 | algorithm, 6 | apple, 7 | authentication, 8 | azure, 9 | android, 10 | akka, 11 | bash, 12 | clojure, 13 | css, 14 | css3, 15 | cloud, 16 | couchdb, 17 | cassandra, 18 | captcha, 19 | couchbase, 20 | // ddos, 21 | dynamobd, 22 | django, 23 | 24 | eclipse, 25 | facebook, 26 | git, 27 | google, 28 | gmail, 29 | hadoop, 30 | hbase, 31 | hibernate, 32 | 33 | ios, 34 | iphone, 35 | ipad, 36 | java, 37 | jquery, 38 | jpa, 39 | json, 40 | javascript, 41 | jvm, 42 | jsp, 43 | lisp, 44 | linux, 45 | 46 | mysql, 47 | maven, 48 | macbook, 49 | mongodb, 50 | multithreading, 51 | nosql, 52 | neo4j, 53 | parsing, 54 | passwords, 55 | php, 56 | python, 57 | perl, 58 | perl5, 59 | perl6, 60 | postgresql, 61 | 62 | rest, 63 | regex, 64 | ruby, 65 | riak, 66 | redis, 67 | 68 | scala, 69 | seo, 70 | security, 71 | sql, 72 | spring, 73 | springdata, 74 | springsocial, 75 | 76 | ubuntu, 77 | 78 | wordpress, 79 | 80 | xml, 81 | wpf 82 | }// @formatter:on 83 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/stackexchange/util/StackTagAdvanced.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.util; 2 | 3 | public final class StackTagAdvanced { 4 | public static final String amazons3 = "amazon-s3"; 5 | public static final String amazonec2 = "amazon-ec2"; 6 | public static final String amazonwebservices = "amazon-web-services"; 7 | 8 | public static final String aspnet = "asp.net"; 9 | 10 | public static final String rubyonrails = "ruby-on-rails"; 11 | public static final String objectivec = "objective-c"; 12 | public static final String java8 = "java-8"; 13 | public static final String java9 = "java-9"; 14 | 15 | private StackTagAdvanced() { 16 | throw new AssertionError(); 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/MetricsClearScheduler.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta; 2 | 3 | import java.io.IOException; 4 | 5 | import org.common.metrics.MetricsUtil; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.context.annotation.Profile; 8 | import org.springframework.scheduling.annotation.Scheduled; 9 | import org.springframework.stereotype.Component; 10 | import org.tweet.spring.util.SpringProfileUtil; 11 | 12 | import com.codahale.metrics.Counter; 13 | import com.codahale.metrics.MetricRegistry; 14 | import com.fasterxml.jackson.core.JsonProcessingException; 15 | 16 | @Component 17 | @Profile(SpringProfileUtil.WRITE_PRODUCTION) 18 | public class MetricsClearScheduler { 19 | // private final Logger logger = LoggerFactory.getLogger(getClass()); 20 | 21 | @Autowired 22 | private MetricRegistry metrics; 23 | 24 | public MetricsClearScheduler() { 25 | super(); 26 | } 27 | 28 | // API 29 | 30 | // git - not 100% sure that hashtag will only return relevant tweets - look into this further 31 | // for accounts - not yet: BestBash,EclipseFacts,BestGit,BestJPA,MavenFact,BestOfRuby,SpringTip,ServerFaultBest,JavaTopSO,RESTDaily 32 | 33 | @Scheduled(cron = "0 59 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23 * * *") 34 | public void tweetMeta1() throws JsonProcessingException, IOException { 35 | final Counter retweetAnyErrorCounter = metrics.counter(MetricsUtil.Meta.RETWEET_ANY_ERROR); 36 | retweetAnyErrorCounter.dec(retweetAnyErrorCounter.getCount()); 37 | 38 | final Counter retweetOneErrorCounter = metrics.counter(MetricsUtil.Meta.RETWEET_ONE_ERROR); 39 | retweetOneErrorCounter.dec(retweetOneErrorCounter.getCount()); 40 | 41 | final Counter httpErrorCounter = metrics.counter(MetricsUtil.Meta.HTTP_ERR); 42 | httpErrorCounter.dec(httpErrorCounter.getCount()); 43 | 44 | final Counter httpOkCounter = metrics.counter(MetricsUtil.Meta.HTTP_OK); 45 | httpOkCounter.dec(httpOkCounter.getCount()); 46 | 47 | final Counter twitterReadOkCounter = metrics.counter(MetricsUtil.Meta.TWITTER_READ_OK); 48 | twitterReadOkCounter.dec(twitterReadOkCounter.getCount()); 49 | 50 | final Counter twitterWriteOkCounter = metrics.counter(MetricsUtil.Meta.TWITTER_WRITE_OK); 51 | twitterWriteOkCounter.dec(twitterWriteOkCounter.getCount()); 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/NonTechnicalFollowMetaScheduler.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta; 2 | 3 | import java.io.IOException; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.context.annotation.Profile; 9 | import org.springframework.core.env.Environment; 10 | import org.springframework.scheduling.annotation.Scheduled; 11 | import org.springframework.stereotype.Component; 12 | import org.tweet.meta.service.FollowLiveService; 13 | import org.tweet.spring.util.SpringProfileUtil; 14 | 15 | import com.fasterxml.jackson.core.JsonProcessingException; 16 | 17 | @Component 18 | @Profile(SpringProfileUtil.WRITE_PRODUCTION) 19 | public class NonTechnicalFollowMetaScheduler { 20 | private static final String MODE_MAINTAINANCE_KEY = "mode.maintainance.rt"; 21 | 22 | private final Logger logger = LoggerFactory.getLogger(getClass()); 23 | 24 | @Autowired 25 | private FollowLiveService service; 26 | 27 | @Autowired 28 | private Environment env; 29 | 30 | public NonTechnicalFollowMetaScheduler() { 31 | super(); 32 | } 33 | 34 | // API 35 | 36 | @Scheduled(cron = "0 30 14,16,18,20,22 * * *") 37 | public void newScheduleFive() throws JsonProcessingException, IOException { 38 | logger.info("Starting new retweet schedule - five (non-tech)"); 39 | 40 | if (env.getProperty(MODE_MAINTAINANCE_KEY, Boolean.class)) { 41 | logger.warn("Maintainance Mode Active - skipping schedule"); 42 | return; 43 | } 44 | 45 | // service.followBestUser(TwitterAccountEnum.thedogbreeds.name(), TwitterTag.puppy.name()); 46 | 47 | logger.info("Finished new retweet schedule - five (non-tech)"); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/NonTechnicalTweetMetaScheduler.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta; 2 | 3 | import java.io.IOException; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.context.annotation.Profile; 9 | import org.springframework.core.env.Environment; 10 | import org.springframework.scheduling.annotation.Scheduled; 11 | import org.springframework.stereotype.Component; 12 | import org.tweet.meta.service.TweetMetaLiveService; 13 | import org.tweet.spring.util.SpringProfileUtil; 14 | 15 | import com.fasterxml.jackson.core.JsonProcessingException; 16 | 17 | @Component 18 | @Profile(SpringProfileUtil.WRITE_PRODUCTION) 19 | public class NonTechnicalTweetMetaScheduler { 20 | private static final String MODE_MAINTAINANCE_KEY = "mode.maintainance.rt"; 21 | 22 | private final Logger logger = LoggerFactory.getLogger(getClass()); 23 | 24 | @Autowired 25 | private TweetMetaLiveService service; 26 | 27 | @Autowired 28 | private Environment env; 29 | 30 | public NonTechnicalTweetMetaScheduler() { 31 | super(); 32 | } 33 | 34 | // API 35 | 36 | @Scheduled(cron = "0 10 13,14,15,16,17,18,19,20,21,22 * * *") 37 | public void newScheduleTen() throws JsonProcessingException, IOException { 38 | logger.info("Starting new retweet schedule - six (non-tech)"); 39 | 40 | if (env.getProperty(MODE_MAINTAINANCE_KEY, Boolean.class)) { 41 | logger.warn("Maintainance Mode Active - skipping schedule"); 42 | return; 43 | } 44 | 45 | // service.retweetAnyByWord(TwitterAccountEnum.thedogbreeds.name(), TweetType.NonTech); 46 | 47 | logger.info("Finished new retweet schedule - six (non-tech)"); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/analysis/TweetPassesSet1ChecksPredicate.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta.analysis; 2 | 3 | import org.springframework.social.twitter.api.Tweet; 4 | import org.tweet.twitter.service.TweetService; 5 | 6 | import com.google.common.base.Predicate; 7 | 8 | public final class TweetPassesSet1ChecksPredicate implements Predicate { 9 | 10 | private TweetService tweetService; 11 | 12 | public TweetPassesSet1ChecksPredicate(final TweetService tweetService) { 13 | this.tweetService = tweetService; 14 | } 15 | 16 | @Override 17 | public final boolean apply(final Tweet tweet) { 18 | return tweetService.passesSet1OfChecks(tweet, null); 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/analysis/TweetPassesSet2ChecksPredicate.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta.analysis; 2 | 3 | import org.springframework.social.twitter.api.Tweet; 4 | import org.tweet.twitter.service.TweetService; 5 | 6 | import com.google.common.base.Predicate; 7 | 8 | public final class TweetPassesSet2ChecksPredicate implements Predicate { 9 | 10 | private TweetService tweetService; 11 | 12 | public TweetPassesSet2ChecksPredicate(final TweetService tweetService) { 13 | this.tweetService = tweetService; 14 | } 15 | 16 | @Override 17 | public final boolean apply(final Tweet tweet) { 18 | return tweetService.passesLanguageChecksForAnalysis(tweet, null); 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/analysis/TweetPassesSet3ChecksPredicate.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta.analysis; 2 | 3 | import org.springframework.social.twitter.api.Tweet; 4 | import org.tweet.twitter.service.TweetService; 5 | import org.tweet.twitter.util.TweetUtil; 6 | 7 | import com.google.common.base.Predicate; 8 | 9 | public final class TweetPassesSet3ChecksPredicate implements Predicate { 10 | 11 | private TweetService tweetService; 12 | 13 | public TweetPassesSet3ChecksPredicate(final TweetService tweetService) { 14 | this.tweetService = tweetService; 15 | } 16 | 17 | @Override 18 | public final boolean apply(final Tweet tweet) { 19 | return tweetService.passesSet3OfChecksForAnalysis(TweetUtil.getText(tweet)); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/component/PredefinedAccountRetriever.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta.component; 2 | 3 | import java.util.List; 4 | 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.core.env.Environment; 7 | import org.springframework.stereotype.Component; 8 | import org.stackexchange.util.GenericUtil; 9 | 10 | import com.google.api.client.util.Preconditions; 11 | 12 | @Component 13 | public final class PredefinedAccountRetriever { 14 | 15 | @Autowired 16 | private Environment env; 17 | 18 | public PredefinedAccountRetriever() { 19 | super(); 20 | } 21 | 22 | // API 23 | 24 | public final List predefinedAccount(final String twitterAccount) { 25 | final String allPredefinedAccountsRaw = Preconditions.checkNotNull(predefinedAccountRaw(twitterAccount)); 26 | return GenericUtil.breakApart(allPredefinedAccountsRaw); 27 | } 28 | 29 | final String predefinedAccountRaw(final String twitterAccount) { 30 | return env.getProperty(twitterAccount + ".predefined"); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/persistence/dao/IRetweetJpaDAO.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta.persistence.dao; 2 | 3 | import java.util.List; 4 | 5 | import org.common.persistence.IOperations; 6 | import org.springframework.data.jpa.repository.JpaRepository; 7 | import org.springframework.data.jpa.repository.Query; 8 | import org.tweet.meta.persistence.model.Retweet; 9 | 10 | public interface IRetweetJpaDAO extends JpaRepository, IOperations { 11 | 12 | // find - one 13 | 14 | /*0*/Retweet findOneByTweetId(final long tweetId); 15 | 16 | /*1*/Retweet findOneByTweetIdAndTwitterAccount(final long tweetId, final String twitterAccount); 17 | 18 | Retweet findOneByTextAndTwitterAccount(final String text, final String twitterAccount); 19 | 20 | List findAllByTextAndTwitterAccount(final String text, final String twitterAccount); 21 | 22 | List findAllByTextEndsWithAndTwitterAccount(final String text, final String twitterAccount); 23 | 24 | List findAllByTextStartsWithAndTwitterAccount(final String text, final String twitterAccount); 25 | 26 | List findAllByTextContainsAndTwitterAccount(final String text, final String twitterAccount); 27 | 28 | Retweet findOneByTextEndsWithAndTwitterAccount(final String text, final String twitterAccount); 29 | 30 | Retweet findOneByTextStartsWithAndTwitterAccount(final String text, final String twitterAccount); 31 | 32 | // find - all 33 | 34 | /*0*/List findAllByTweetId(final long tweetId); 35 | 36 | List findAllByTwitterAccount(final String twitterAccount); 37 | 38 | // count 39 | 40 | @Query("select count(r) from Retweet r where r.twitterAccount = ?1") 41 | long countAllByTwitterAccount(final String twitterAccount); 42 | 43 | } 44 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/persistence/setup/TwitterMetaSetupListener.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta.persistence.setup; 2 | 3 | import org.common.persistence.setup.BeforeSetupEvent; 4 | import org.keyval.persistence.dao.IKeyValJpaDAO; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.context.ApplicationContext; 9 | import org.springframework.context.ApplicationListener; 10 | import org.springframework.context.annotation.Profile; 11 | import org.springframework.stereotype.Component; 12 | import org.tweet.spring.util.SpringProfileUtil; 13 | 14 | @Component 15 | @Profile(SpringProfileUtil.PERSISTENCE) 16 | public class TwitterMetaSetupListener implements ApplicationListener { 17 | private final Logger logger = LoggerFactory.getLogger(getClass()); 18 | 19 | @Autowired 20 | private ApplicationContext eventPublisher; 21 | 22 | @Autowired 23 | private IKeyValJpaDAO keyValApi; 24 | 25 | public TwitterMetaSetupListener() { 26 | super(); 27 | } 28 | 29 | // 30 | 31 | @Override 32 | public final void onApplicationEvent(final BeforeSetupEvent event) { 33 | setupRetweetThresholds(); 34 | } 35 | 36 | // util 37 | 38 | private void setupRetweetThresholds() { 39 | // keyValApi.save(new KeyVal(Tag.jquery.name(), 5d)); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/service/AlreadyFollowedByPredicate.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta.service; 2 | 3 | import java.util.Set; 4 | 5 | import javax.annotation.Nullable; 6 | 7 | import org.springframework.social.twitter.api.TwitterProfile; 8 | 9 | import com.google.api.client.util.Preconditions; 10 | import com.google.common.base.Predicate; 11 | 12 | public class AlreadyFollowedByPredicate implements Predicate { 13 | 14 | private final Set alreadyFollowedAccounts; 15 | 16 | public AlreadyFollowedByPredicate(final Set alreadyFollowedAccounts) { 17 | super(); 18 | 19 | this.alreadyFollowedAccounts = Preconditions.checkNotNull(alreadyFollowedAccounts); 20 | } 21 | 22 | // API 23 | 24 | @Override 25 | public final boolean apply(@Nullable final TwitterProfile input) { 26 | final boolean alreadyFollows = alreadyFollowedAccounts.contains(input.getId()); 27 | if (alreadyFollows) { 28 | return true; 29 | } 30 | 31 | return false; 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/service/IsMyselfPredicate.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta.service; 2 | 3 | import javax.annotation.Nullable; 4 | 5 | import org.springframework.social.twitter.api.TwitterProfile; 6 | 7 | import com.google.api.client.util.Preconditions; 8 | import com.google.common.base.Predicate; 9 | 10 | public class IsMyselfPredicate implements Predicate { 11 | 12 | private final String myAccount; 13 | 14 | public IsMyselfPredicate(final String myAccount) { 15 | super(); 16 | this.myAccount = Preconditions.checkNotNull(myAccount); 17 | } 18 | 19 | // API 20 | 21 | @Override 22 | public final boolean apply(@Nullable final TwitterProfile input) { 23 | if (input.getScreenName().equals(myAccount)) { 24 | return true; 25 | } 26 | return false; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/service/TweetByRtComparator.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta.service; 2 | 3 | import java.util.Comparator; 4 | 5 | import org.springframework.social.twitter.api.Tweet; 6 | 7 | public final class TweetByRtComparator implements Comparator { 8 | 9 | @Override 10 | public final int compare(final Tweet t1, final Tweet t2) { 11 | return t2.getRetweetCount().compareTo(t1.getRetweetCount()); 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/service/TweetContainsWordPredicate.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta.service; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.social.twitter.api.Tweet; 6 | 7 | import com.google.api.client.util.Preconditions; 8 | import com.google.common.base.Predicate; 9 | 10 | public final class TweetContainsWordPredicate implements Predicate { 11 | private final Logger logger = LoggerFactory.getLogger(getClass()); 12 | 13 | private final String word; 14 | 15 | public TweetContainsWordPredicate(final String word) { 16 | super(); 17 | 18 | this.word = Preconditions.checkNotNull(word); 19 | } 20 | 21 | // API 22 | 23 | @Override 24 | public final boolean apply(final Tweet input) { 25 | final String tweetTextLowerCase = input.getText().toLowerCase(); 26 | if (containsWord(tweetTextLowerCase)) { 27 | return true; 28 | } 29 | return false; 30 | } 31 | 32 | final boolean containsWord(final String tweetTextLowerCase) { 33 | final boolean contains = tweetTextLowerCase.matches(".*\\b" + word + "\\b.*"); 34 | if (!contains) { 35 | // logger.debug("Following tweet does not contain the word={}: \n{}", this.word, tweetTextLowerCase); 36 | } 37 | return contains; 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/spring/TwitterMetaConfig.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta.spring; 2 | 3 | import org.springframework.context.annotation.ComponentScan; 4 | import org.springframework.context.annotation.Configuration; 5 | 6 | @Configuration 7 | @ComponentScan({ "org.tweet.meta" }) 8 | public class TwitterMetaConfig { 9 | 10 | public TwitterMetaConfig() { 11 | super(); 12 | } 13 | 14 | // API 15 | 16 | } -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/spring/TwitterMetaPersistenceJPAConfig.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta.spring; 2 | 3 | import org.common.spring.PersistenceJPACommonConfig; 4 | import org.springframework.context.annotation.ComponentScan; 5 | import org.springframework.context.annotation.Configuration; 6 | import org.springframework.context.annotation.Import; 7 | import org.springframework.context.annotation.ImportResource; 8 | import org.springframework.context.annotation.Profile; 9 | import org.springframework.context.annotation.PropertySource; 10 | import org.tweet.spring.util.SpringProfileUtil; 11 | 12 | @Configuration 13 | @Import(PersistenceJPACommonConfig.class) 14 | @ComponentScan({ "org.tweet.meta.persistence" }) 15 | @ImportResource("classpath*:metaPersistenceConfig.xml") 16 | @PropertySource({ "classpath:persistence-${persistenceTarget:prod}.properties" }) 17 | @Profile(SpringProfileUtil.PERSISTENCE) 18 | public class TwitterMetaPersistenceJPAConfig { 19 | 20 | public TwitterMetaPersistenceJPAConfig() { 21 | super(); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/util/TweetIsRetweetPredicate.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta.util; 2 | 3 | import org.springframework.social.twitter.api.Tweet; 4 | 5 | import com.google.common.base.Predicate; 6 | 7 | public final class TweetIsRetweetPredicate implements Predicate { 8 | 9 | @Override 10 | public final boolean apply(final Tweet tweet) { 11 | return tweet.isRetweet(); 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/util/TweetLinksToSePredicate.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta.util; 2 | 3 | import org.common.service.live.LinkLiveService; 4 | import org.common.util.LinkUtil.Technical; 5 | import org.springframework.social.twitter.api.Tweet; 6 | 7 | import com.google.api.client.util.Preconditions; 8 | import com.google.common.base.Predicate; 9 | 10 | public final class TweetLinksToSePredicate implements Predicate { 11 | 12 | private LinkLiveService linkLiveService; 13 | 14 | public TweetLinksToSePredicate(final LinkLiveService linkLiveService) { 15 | this.linkLiveService = Preconditions.checkNotNull(linkLiveService); 16 | } 17 | 18 | @Override 19 | public final boolean apply(final Tweet tweet) { 20 | return linkLiveService.countLinksToAnyDomain(tweet, Technical.seDomains) > 0; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/meta/util/TweetNotLinksToSePredicate.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta.util; 2 | 3 | import org.common.service.live.LinkLiveService; 4 | import org.common.util.LinkUtil.Technical; 5 | import org.springframework.social.twitter.api.Tweet; 6 | 7 | import com.google.api.client.util.Preconditions; 8 | import com.google.common.base.Predicate; 9 | 10 | public final class TweetNotLinksToSePredicate implements Predicate { 11 | 12 | private LinkLiveService linkLiveService; 13 | 14 | public TweetNotLinksToSePredicate(final LinkLiveService linkLiveService) { 15 | this.linkLiveService = Preconditions.checkNotNull(linkLiveService); 16 | } 17 | 18 | @Override 19 | public final boolean apply(final Tweet tweet) { 20 | return linkLiveService.countLinksToAnyDomain(tweet, Technical.seDomains) == 0; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/spring/TwitterConfig.java: -------------------------------------------------------------------------------- 1 | package org.tweet.spring; 2 | 3 | import org.springframework.context.annotation.ComponentScan; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.context.annotation.PropertySource; 6 | 7 | @Configuration 8 | @ComponentScan({ "org.tweet.twitter" }) 9 | @PropertySource({ "classpath:twitterInternal.properties", "classpath:interaction.properties" }) 10 | public class TwitterConfig { 11 | 12 | public TwitterConfig() { 13 | super(); 14 | } 15 | 16 | // API 17 | 18 | } -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/spring/TwitterLiveConfig.java: -------------------------------------------------------------------------------- 1 | package org.tweet.spring; 2 | 3 | import org.springframework.context.annotation.Configuration; 4 | import org.springframework.context.annotation.PropertySource; 5 | 6 | @Configuration 7 | @PropertySource({ "file:///opt/stack/twitter.properties" }) 8 | public class TwitterLiveConfig { 9 | 10 | public TwitterLiveConfig() { 11 | super(); 12 | } 13 | 14 | // API 15 | 16 | } -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/spring/util/SpringProfileUtil.java: -------------------------------------------------------------------------------- 1 | package org.tweet.spring.util; 2 | 3 | public final class SpringProfileUtil { 4 | 5 | public static final class Live { 6 | public static final String ANALYSIS = "analysis"; 7 | } 8 | 9 | // these 3 profiles are modeling the environments - deployed is active for any deployment, dev and production for these specific environments 10 | public static final String DEPLOYED = "deployed"; 11 | public static final String DEPLOYED_POLLER = "deployed_poller"; 12 | /** 13 | * < WRITE < WRITE_PRODUCTION 14 | */ 15 | public static final String LIVE = "live"; 16 | public static final String DEV = "dev"; 17 | /** > WRITE > LIVE */ 18 | public static final String WRITE_PRODUCTION = "production"; 19 | /** > LIVE */ 20 | public static final String WRITE = "write"; 21 | 22 | // common 23 | public static final String TEST = "test"; 24 | public static final String CLIENT = "client"; 25 | public static final String PERSISTENCE = "persistence"; 26 | 27 | private SpringProfileUtil() { 28 | throw new AssertionError(); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/twitter/component/DiscouragedExpressionRetriever.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.component; 2 | 3 | import java.util.List; 4 | 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.core.env.Environment; 7 | import org.springframework.stereotype.Component; 8 | import org.stackexchange.util.GenericUtil; 9 | 10 | import com.google.common.collect.Lists; 11 | 12 | /** 13 | * - note: this is under .twitter (and not under .meta) because is core twitter functionality 14 | */ 15 | @Component 16 | public class DiscouragedExpressionRetriever { 17 | 18 | @Autowired 19 | private Environment env; 20 | 21 | public DiscouragedExpressionRetriever() { 22 | super(); 23 | } 24 | 25 | // API 26 | 27 | /** 28 | * return - not null 29 | */ 30 | public final List discouraged(final String twitterAccount) { 31 | final String discouragedRaw = discouragedRaw(twitterAccount); 32 | if (discouragedRaw == null) { 33 | return Lists.newArrayList(); 34 | } 35 | return GenericUtil.breakApart(discouragedRaw); 36 | } 37 | 38 | final String discouragedRaw(final String twitterAccount) { 39 | return env.getProperty(twitterAccount + ".discouraged"); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/twitter/component/MinRtRetriever.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.component; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.core.env.Environment; 5 | import org.springframework.stereotype.Component; 6 | 7 | import com.google.api.client.util.Preconditions; 8 | 9 | /** 10 | * - note: this is under .twitter (and not under .meta) because is core twitter functionality 11 | */ 12 | @Component 13 | public class MinRtRetriever { 14 | 15 | @Autowired 16 | private Environment env; 17 | 18 | public MinRtRetriever() { 19 | super(); 20 | } 21 | 22 | // API 23 | 24 | public final int minRt(final String twitterTag) { 25 | Preconditions.checkNotNull(minRtRaw(twitterTag), "Unable to find min rt value for twitterTag= " + twitterTag); 26 | return env.getProperty(twitterTag + ".minrt", Integer.class); 27 | } 28 | 29 | final String minRtRaw(final String twitterTag) { 30 | return env.getProperty(twitterTag + ".minrt"); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/twitter/component/TwitterHashtagsRetriever.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.component; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.core.env.Environment; 5 | import org.springframework.stereotype.Component; 6 | 7 | import com.google.common.base.Preconditions; 8 | 9 | @Component 10 | public final class TwitterHashtagsRetriever { 11 | 12 | @Autowired 13 | private Environment env; 14 | 15 | public TwitterHashtagsRetriever() { 16 | super(); 17 | } 18 | 19 | // API 20 | 21 | final String hashtagsRaw(final String twitterAccount) { 22 | return env.getProperty(twitterAccount + ".hash"); 23 | } 24 | 25 | public final String hashtags(final String twitterAccount) { 26 | return Preconditions.checkNotNull(hashtagsRaw(twitterAccount)); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/twitter/persistence/model/TwitterAccount.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.persistence.model; 2 | 3 | import javax.persistence.Column; 4 | import javax.persistence.GeneratedValue; 5 | import javax.persistence.GenerationType; 6 | import javax.persistence.Id; 7 | 8 | import org.common.persistence.IEntity; 9 | 10 | public class TwitterAccount implements IEntity { 11 | 12 | @Id 13 | @GeneratedValue(strategy = GenerationType.AUTO) 14 | @Column(name = "ACCOUNT_ID") 15 | private long id; 16 | 17 | public TwitterAccount() { 18 | super(); 19 | } 20 | 21 | // API 22 | 23 | @Override 24 | public long getId() { 25 | return id; 26 | } 27 | 28 | @Override 29 | public void setId(final long id) { 30 | this.id = id; 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/twitter/service/TweetLiveService.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.service; 2 | 3 | import org.common.service.LinkService; 4 | import org.common.service.live.HttpLiveService; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.context.annotation.Profile; 7 | import org.springframework.stereotype.Service; 8 | import org.tweet.spring.util.SpringProfileUtil; 9 | 10 | import com.google.common.base.Preconditions; 11 | 12 | @Service 13 | @Profile(SpringProfileUtil.LIVE) 14 | public class TweetLiveService { 15 | // private final Logger logger = LoggerFactory.getLogger(getClass()); 16 | 17 | @Autowired 18 | private HttpLiveService httpService; 19 | 20 | @Autowired 21 | private LinkService linkService; 22 | 23 | public TweetLiveService() { 24 | super(); 25 | } 26 | 27 | // API 28 | 29 | public final String constructTweetLive(final String text, final String url) { 30 | Preconditions.checkNotNull(text); 31 | Preconditions.checkNotNull(url); 32 | 33 | final String expandedUrl = Preconditions.checkNotNull(httpService.expand(url)); 34 | final String cleanExpandedUrl = linkService.removeUrlParameters(expandedUrl); 35 | 36 | final String textOfTweet = text; 37 | final String tweet = textOfTweet + " - " + cleanExpandedUrl; 38 | return tweet; 39 | } 40 | 41 | // util 42 | 43 | } 44 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/twitter/service/TwitterTemplateCreator.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.service; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.core.env.Environment; 5 | import org.springframework.social.twitter.api.Twitter; 6 | import org.springframework.social.twitter.api.impl.TwitterTemplate; 7 | import org.springframework.stereotype.Component; 8 | 9 | import com.google.common.base.Preconditions; 10 | 11 | @Component 12 | public class TwitterTemplateCreator { 13 | 14 | @Autowired 15 | private Environment env; 16 | 17 | public TwitterTemplateCreator() { 18 | super(); 19 | } 20 | 21 | // API 22 | 23 | public synchronized final Twitter createTwitterTemplate(final String twitterAccount) { 24 | final String consumerKey = Preconditions.checkNotNull(env.getProperty(twitterAccount + ".consumerKey"), "consumerKey not found for " + twitterAccount); 25 | final String consumerSecret = Preconditions.checkNotNull(env.getProperty(twitterAccount + ".consumerSecret"), "consumerSecret not found for " + twitterAccount); 26 | final String accessToken = Preconditions.checkNotNull(env.getProperty(twitterAccount + ".accessToken"), "accessToken not found for " + twitterAccount); 27 | final String accessTokenSecret = Preconditions.checkNotNull(env.getProperty(twitterAccount + ".accessTokenSecret"), "accessTokenSecret not found for " + twitterAccount); 28 | 29 | final TwitterTemplate twitterTemplate = new TwitterTemplate(consumerKey, consumerSecret, accessToken, accessTokenSecret); 30 | return twitterTemplate; 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/twitter/service/live/ITwitterWriteLiveService.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.service.live; 2 | 3 | import org.springframework.social.twitter.api.Tweet; 4 | 5 | public interface ITwitterWriteLiveService { 6 | 7 | boolean retweet(String twitterAccount, long tweetId); 8 | 9 | boolean tweet(String twitterAccount, String textToTweet); 10 | 11 | boolean tweet(String twitterAccount, String textToTweet, Tweet originalTweet); 12 | 13 | } 14 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/twitter/service/live/TweetToStringFunction.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.service.live; 2 | 3 | import org.springframework.social.twitter.api.Tweet; 4 | import org.tweet.twitter.util.TweetUtil; 5 | 6 | import com.google.common.base.Function; 7 | 8 | public final class TweetToStringFunction implements Function { 9 | 10 | @Override 11 | public final String apply(final Tweet input) { 12 | return TweetUtil.getText(input); 13 | } 14 | 15 | } -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/twitter/service/live/TwitterWriteDevLiveService.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.service.live; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.context.annotation.Profile; 6 | import org.springframework.social.twitter.api.Tweet; 7 | import org.springframework.stereotype.Service; 8 | import org.tweet.spring.util.SpringProfileUtil; 9 | 10 | @Service 11 | @Profile(SpringProfileUtil.DEV) 12 | public class TwitterWriteDevLiveService implements ITwitterWriteLiveService { 13 | private final Logger logger = LoggerFactory.getLogger(getClass()); 14 | 15 | public TwitterWriteDevLiveService() { 16 | super(); 17 | } 18 | 19 | // API 20 | 21 | // write 22 | 23 | @Override 24 | public boolean retweet(final String twitterAccount, final long tweetId) { 25 | logger.warn("Simmulating retweet on twitterAccount= {} or tweetId= {}", twitterAccount, tweetId); 26 | return true; 27 | } 28 | 29 | @Override 30 | public boolean tweet(final String twitterAccount, final String textToTweet) { 31 | logger.warn("Simmulating tweet on twitterAccount= {} or tweetText= {}", twitterAccount, textToTweet); 32 | return true; 33 | } 34 | 35 | @Override 36 | public boolean tweet(final String twitterAccount, final String textToTweet, final Tweet originalTweet) { 37 | logger.warn("Simmulating tweet on twitterAccount= {} or tweetText= {}", twitterAccount, textToTweet); 38 | return true; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/twitter/util/TwitterInteraction.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.util; 2 | 3 | public enum TwitterInteraction { 4 | 5 | /** 6 | * No value in interacting with the user 7 | */ 8 | None, 9 | 10 | /** 11 | * Best way to interact with this user is to mention it 12 | */ 13 | Mention, 14 | /** 15 | * Best way to interact with this user is to retweet it

16 | * - note: this is the fallback - if there is value in interacting with the user, the default is going to be the Retweet
17 | * that doesn't necessarily mean that the tweet itself should be retweeted if there's more value in tweeting it (may contain good mentions) 18 | */ 19 | Retweet 20 | 21 | } 22 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/java/org/tweet/twitter/util/TwitterInteractionWithValue.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.util; 2 | 3 | public final class TwitterInteractionWithValue { 4 | private final float val; 5 | private final TwitterInteraction twitterInteraction; 6 | 7 | public TwitterInteractionWithValue(final TwitterInteraction twitterInteraction, final float val) { 8 | super(); 9 | 10 | this.twitterInteraction = twitterInteraction; 11 | this.val = val; 12 | } 13 | 14 | // API 15 | 16 | public final float getVal() { 17 | return val; 18 | } 19 | 20 | public final TwitterInteraction getTwitterInteraction() { 21 | return twitterInteraction; 22 | } 23 | 24 | @Override 25 | public final String toString() { 26 | final StringBuilder builder = new StringBuilder(); 27 | builder.append("TwitterInteractionWithValue [val=").append(val).append(", twitterInteraction=").append(twitterInteraction).append("]"); 28 | return builder.toString(); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/backup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -x 4 | 5 | DATE=$(date +%Y%m%d) 6 | DATETIME=$(date +%Y%m%d-%H%m) 7 | BACKUP_DIRECTORY=/tmp/ 8 | DB=stackexchange_prod 9 | BACKUP_FILE=$BACKUP_DIRECTORY/${DATETIME}_${DB}.sql 10 | mysqldump -u stackUser -pstackmy5ql stackexchange_prod > $BACKUP_FILE 11 | 12 | S3_BUCKET_URL=s3://twitter_db_backup/$DATE/ 13 | s3cmd put ${BACKUP_FILE} $S3_BUCKET_URL 2>&1 14 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/classification/jobs/jobs-onthefence.classif: -------------------------------------------------------------------------------- 1 | 2 | - non-commercial (but rejected): 3 | We're giving out FREE #iPAD'S!!!! Get em' while they're hot ---> http://t.co/kfD8pizC3p RT! RT! RT! 4 | 5 | 6 | - commercial: 7 | 8 | 9 | - to sort: 10 | 11 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/classification/programming/nonprogramming.classif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/java-stackexchange2twitter/src/main/resources/classification/programming/nonprogramming.classif -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/classification/programming/programming.classif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/java-stackexchange2twitter/src/main/resources/classification/programming/programming.classif -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/classification/sample.classif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/java-stackexchange2twitter/src/main/resources/classification/sample.classif -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/classification/test/jobs/note.info: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/java-stackexchange2twitter/src/main/resources/classification/test/jobs/note.info -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/classification/test/programming/nonprogramming.classif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/java-stackexchange2twitter/src/main/resources/classification/test/programming/nonprogramming.classif -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/classification/test/programming/programming.classif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/java-stackexchange2twitter/src/main/resources/classification/test/programming/programming.classif -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/commonPersistenceConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/config-dev.properties: -------------------------------------------------------------------------------- 1 | metrics.log.rate=5 -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/config-production.properties: -------------------------------------------------------------------------------- 1 | metrics.log.rate=15 -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/env-dev.properties: -------------------------------------------------------------------------------- 1 | spring.profiles.active=deployed,live,dev,write,analysis,deployed_poller,persistence -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/env-production.properties: -------------------------------------------------------------------------------- 1 | spring.profiles.active=deployed,live,production,write,analysis,deployed_poller,persistence -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/interaction.properties: -------------------------------------------------------------------------------- 1 | 2 | twitter.value.pagestoanalyze=3 3 | twitter.value.largeaccount=500 4 | 5 | # twitter.value.user 6 | twitter.value.user.retweets.percentage.min=4 7 | twitter.value.user.retweetsandmentions.min=10 8 | twitter.value.user.smallaccountretweets.percentage.min=10 9 | twitter.value.user.followers.min=300 10 | twitter.value.user.tweets.min=500 11 | twitter.retweet.score.followers.percentage=50 12 | 13 | # twitter.value.tweet 14 | twitter.value.tweet.retweets.max=15 15 | 16 | 17 | # hashtags that should decrease interaction score (per account) 18 | BestOfCloud.discouraged=CloudExpo,ipexpo 19 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | 10 | /var/log/stackexchange/stack.log 11 | 12 | %date %level [%thread] %logger{10} [%file:%line] %msg%n 13 | 14 | 15 | 16 | 17 | 18 | 19 | ERROR 20 | 21 | email-smtp.us-east-1.amazonaws.com 22 | AKIAJIKXZAQFFJDXI4VQ 23 | AjYQNjXVTalbpfRL37RYqyXEkyQAvmKFcUFQllfAsBLh 24 | true 25 | javatotwitter@gmail.com 26 | javatotwitter@gmail.com 27 | stackexchange: %logger{20} - %m 28 | 29 | stackexchange - %date [%thread] %-5level %logger{36} - %message%n 30 | 31 | 32 | 1000 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/metaPersistenceConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/notes/blockedBy.txt: -------------------------------------------------------------------------------- 1 | 2 | # Not able to retweet tweets of: 3 | 4 | https://twitter.com/RainerJoswig 5 | https://twitter.com/danielwertheim 6 | 7 | TODO: soon, try to get the details, put them into the logged error and then clearly make error a warn - nothing I can do about it 8 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/notes/blogs.txt: -------------------------------------------------------------------------------- 1 | 2 | #clojure 3 | 4 | http://planet.clojure.in/ - http://planet.clojure.in/atom.xml 5 | http://clojure.livejournal.com/ - http://clojure.livejournal.com/data/rss 6 | http://mostlylazy.com/ - http://mostlylazy.com/feed 7 | http://blog.clojurewerkz.org/ - http://blog.clojurewerkz.org/atom.xml 8 | http://data-sorcery.org/ - http://data-sorcery.org/rss 9 | http://www.lispcast.com/ - http://lispcast.com/feed 10 | 11 | 12 | TODOs: 13 | 14 | Planet Lisp 15 | Planet Scala 16 | Planet Scheme 17 | Planet JDK 18 | Planet Haskell 19 | Planet Erlang 20 | Planet Python 21 | Planet Ruby 22 | Planet Perl 23 | 24 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/notes/decided/need-toreject.txt: -------------------------------------------------------------------------------- 1 | Created a dynamic (interactive) Twitter Mosaic http://bit.ly/89EL4 - need a name and logo; used #jquery, #bing and of \u2026 2 | #jQuery-Prototype Mobile Website Development by dsmadden: We need a mobile website & apps created. We're looki... http://t.co\u2026 3 | Need #WordPress website design? Let\u2019s talk! http://t.co/ztUViJhTgN 4 | If y'all not already. Y'all need to #follow @JackieEHill on #instagram #twitter #facebook #youtube and... http://t.co/Hp53ukauWj 5 | Can’t find the number to that pizza place? Zomato is all you need to look up menus & place orders within minutes! http://t.co/RQFqa1K9 6 | Managers often struggle to get the information they need from #HR. Fusion HCM changes all that. Here’s how;) http://t.co/laEqDlcFlt 7 | Need a website? Wordpress sites from $299 and up. #wordpress #website #plugins #html #css\u2026 http://t.co/ohJqzM0KSa 8 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/notes/decided/rejected-dance-decided.txt: -------------------------------------------------------------------------------- 1 | 2 | dance - to sort 3 | 4 | “@iOSdigest: Start Making Awesome Hip Hop, Rap, Dubstep, Dance, Trance, Trap Beats on iPhone/iPad https://t.co/70wFDDyKSk #ios 5 | “@novleg: Start Making Awesome Hip Hop, Rap, Dubstep, Dance, Trance, Trap Beats on iPhone/iPad https://t.co/YYkRDIISit #ios” 6 | Start Making Awesome Hip Hop, Rap, Dubstep, Dance, Trance, Trap Beats on iPhone/iPad https://t.co/70wFDDyKSk #ios 7 | LOL TROY #dance #math #justinbieber https://t.co/wCeYVM3H5p 8 | Come and dance with me! Have you tried the apps on iOs and Android? http://t.co/BbnnK0zI4H 9 | #neo4j can perform the waggle dance for graph marketing - @beehaus #graphconnect http://t.co/9zSzdO18bQ 10 | 11 | 12 | 13 | dance - OK for both 14 | 15 | The Intricate Dance Between Computing and Business Profit: By nature, I’m an eclectic thinker... http://t.co/3VsD9g6y #cloud #blogs #TCN 16 | “@CreateLex: Start Making Awesome Hip Hop, Rap, Dubstep, Dance, Trance, Reggae Beats on iPhone/iPad http://t.co/WJbCVCII5M #ios” 17 | John Boehner’s Obamacare victory dance - (Judson Phillips: Cold, Hard Truth) http://t.co/gX9eQgFr2y 18 | Yes, dear @BerlinGeekettes! I will be there for the exciting #hackathon in less than 2 weeks http://t.co/KJVjhKIq ♪ joy dance ♫ 19 | 20 | 21 | 22 | dance - OK for analysis but not for tweeting 23 | 24 | #llcoolj Tell'm his audience that I used to dance for him #backintheday. #fullcircle moment. It don't… http://t.co/ra4fgvjxQG 25 | RT “@ryanmstephen: Here's your chance to dance with Arjun Kapoor! Follow @Strut_Dance & enroll now! http://t.co/4qujlgkwND” 26 | My good mood ״ social emotional #songs for #parents and kids to sing and dance http://t.co/VKAgbXlnp4 + change mood 27 | @__NMJ R U remember when u dance ... LMAO 😂❤❤❤❤❤ http://t.co/hR3Jdolxfi 28 | You've got 5 more days to design the cover & visual identity for the DANCE @RED, SAVE LIVES² digital album. http://t.co/jPtznODrWB #makethis 29 | Track Of The Day: @yokoono – Moonbeams. Heavy art-dance number with guitars and screaming. It's addictive! http://t.co/pDRs02ByRg 30 | New Dance Video Coming Soon At 1 Hour! 😝 http://t.co/XkUtIEKjyr 31 | “@Bugalugx:Adore all your characters, but Prof Janine O'Dowd's wonderful world of ethnic dance is right up there http://t.co/iSoOjxsl”👍👍 32 | 33 | 34 | 35 | dance - not OK for either 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/notes/decided/rejected-dumb.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | dumb - OK 4 | So great: RT @spolsky: "I prevented a dumb patent from getting granted http://t.co/jfSv3z2Cr5” 5 | #NSA has an intelligence problem: It won’t admit how dumb it can be. http://t.co/5VI4U9i3rG @shanewharris breaks it down 6 | Musing over a dumb every-day little CSS issue we’ve probably all faced: the spacing on the bottom of modules. http://t.co/9beg7V42Lj 7 | So, offering a free online course on #a11y is great (thanks google), but why dumb it down to blind people only? » http://t.co/DzWR9rUPqe 8 | Audio: New track from Dumb Luck about insomnia. I made the beat, be sure to listen & share! Alive + Well... http://t.co/OxXtjPkCbw 9 | Here’s what embracing my inner dumb blonde taught me about situational leadership/being more authentic @hennainam http://t.co/aI1DRB7ENb 10 | Sorry, Walt’s plan was dumb and the finale’s whole plot was absurd: http://t.co/bNYcizHqvT 11 | 12 | dumb - not OK 13 | I fixed Twitter’s dumb chart: http://t.co/CHDTm4UW 14 | Lining up for a phone that hasn’t even been announced yet is just dumb. http://t.co/eUR5rytSmE 15 | 16 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/notes/decided/rejected-snake.txt: -------------------------------------------------------------------------------- 1 | snake - not OK mostly 2 | 3 | Celebrity Red Carpet Round Up – Emilia Fox wore our PERSIA snake jacquard shoes! http://t.co/NkNgxiPcEq #PaulAndrew http://t.co/UgE25E5U6L 4 | You’re kidding me. In the 2000 film “Python,” a snake terrorizes the town of “Ruby.” http://en.wikipedia.org/wiki/Python_(film) 5 | Just another wristband👌 Coolest snake ever. 🐍#snake #python #wristband #miami #heat #oldpic #snakeswag… http://t.co/IDOOx9o7E6 6 | ■■■■■■■■  <= core snake logic in 122 bytes, care of @attila_incze: http://t.co/XxWtoRXJ (demo: http://t.co/PLxGWr8a) 7 | Enjoy everything you’d expect from a 20-year old Nokia - except snake - with this DIY #Arduino cellphone: http://t.co/JP1qcDXFn1 #Atmel #TBT 8 | With a black headed python around my neck. #python #awesome #snake #BlackHeadedPython… http://t.co/4C3Mb4EnAC 9 | Snake Comes Out From Lady’s Genitals After Prayers (Viewer Discretion Advised) http://t.co/pLRPdATojr 10 | 11 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/notes/test/analysis/analysis-banned.txt: -------------------------------------------------------------------------------- 1 | RT @BETAwards: 3 Words: FREE. #IPAD. MINI ----> http://bet.us/14agHXR <--- RT if you want one! 2 | RT @BETAwards: some text - if you want one -RT 3 | 3 Words: FREE. #IPAD. MINI ----> http://bet.us/14agHXR <--- RT if you want one! 4 | 3 Words: Free. #IPAD. MINI ----> http://bet.us/14agHXR <--- RT if you want one! 5 | Your chance to win whateva... just Follow & RT 6 | Your chance to win whateva... just follow & RT 7 | Your chance to win whateva... just follow AND RT 8 | Follow & RT if you want loot 9 | FOLLOW @Harmini731 #HARMINI, #ANDYMINEO, #JSON & #YBSOD TURNT UP!!! Peep his Music Video \"Against The Ropes\" http://www.youtube.com/watch?v=arDCisqgdvM … 10 | // x% off 11 | You’ve Got 24 Hours - What Is Today's #Deal? - 97% Off - Web’s Best #Daily #Deals - #Free Shipping - http://t.co/jAKbaM8bfR - $AVE 12 | Get Learning Perl at 60% off today: Learning PerlWas: $31.99Now: $12.80(Save 60%) O’Reilly’s Cyber Monday Deals ... http://t.co/GWFDlLbd 13 | Get something at 5% off today: step right up -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/notes/test/commercial/generic-commercial-toaccept.txt: -------------------------------------------------------------------------------- 1 | “programmers must be skilled at interacting with people, and only incidentally at interacting with machines.” — @izs http://t.co/v8xpfn0wl0 2 | the only piece previewing apple’s 3rd-quarter earnings referencing elves, stolen tvs, and gene munster. guaranteed. http://t.co/9m9bk6fbqj 3 | Following Tumblr's sale to Yahoo, Wordpress' parent company raises money "just below" Tumblr's $1.1 billion pricetag. http://t.co/wab4Mja8NZ 4 | Still time left to get your @CloudDevConf tickets. Great opp to learn about #WindowsAzure, #AWS, #heroku & more. http://t.co/v003ovgaq3 5 | Hortonworks Raises $50M For Expansion And Development In Growing Hadoop Oriented Data Analytics M... http://t.co/5iNIhK05Df by @alexwilliams 6 | The cost difference between #windowsazure and #s3 is enormous, try storing and retrieving 1B objects, $30 vs $1100 http://t.co/2Ccpo2u6 7 | Microsoft team submits Redis patch to enable Windows support... http://t.co/IfxEa9IJ 8 | Congrats @aseemk! FiftyThree, Maker Of Drawing App Paper, Raises $15 Million From Andreessen Horowitz http://t.co/dkEr9JJd0S via @techcrunch 9 | Shareholders not showing confidence in financial well-being of healthy eaters' haven $WFM http://t.co/glH0InEwHI 10 | Did you know you could use #Hadoop on Windows? Hortonworks Data Platform 1.3 now on Windows! http://t.co/eaUJ0M5SJe 11 | Congrats to #WindowsAzure Accelerator Alums: @BuildersCloud raises $1.1M from super angel investors. http://t.co/Wyf59VdRia 12 | The photo was taken in 1946 to an orphaned child, showing his joy to receive a new pair of shoes as a birthday gift. http://t.co/qwbLWfvzwZ 13 | @joerogan San Francisco to test allowing citizens to vote online on how city budget is spent: http://t.co/aGWHW0477i 14 | Accel Leads $30M Round In Flash Sales Site For Wine And Gourmet Food Lot18 http://t.co/ykKC7ema by @leenarao 15 | Steam Autumn (Thanksgiving) and Winter (Holiday) Sales 2013 get leaked start dates - http://t.co/UHPe7M0vt5 #steam #sales ^ADo 16 | Kantar: Android Took 65% Of Sales Across Major Markets In Last Quarter; Windows Phone Grew… http://t.co/X3cWd6pPGL http://t.co/JgWqitmOdQ 17 | Can Salesforce Keep Its Cloud Business Growing? -by @ahess247 http://t.co/va87qBOugD 18 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/notes/test/generic-toaccept.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/java-stackexchange2twitter/src/main/resources/notes/test/generic-toaccept.txt -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/notes/test/generic-toreject.txt: -------------------------------------------------------------------------------- 1 | OMG MY DAD SAID IF I GET 2,000 RETWEETS I CAN GET A NEW PUPPY!!!! PLEASE RT OMG🙏🙏🙏🙏🙌🙌🙌🙌🙌🙌 pic.twitter.com/1ctT5aKQK5 2 | My boyfriend said if I get 2,000 Retweets he will get me a #puppy for Christmas. Everyone Help Me💞 @mm_brandon pic.twitter.com/BvvTzY2HJW 3 | A man from Indonesia left his 9 dogs unfed for 2 weeks. When he returned home his starving dogs attacked and... http://bit.ly/153DN5G 4 | #GIT UR DL "Official @ChoppaChavo Mixtape Talk of The Town Vol.2 They Still Talkin #RoxburyHippies #Bostonstrong http://www.datpiff.com/Hosted-By-Choppa-Chavo-Dinero-Family-Music-Organ-Talk-Of-The-Town-Vol2-They-Still-Talking-mixtape.532807.html … 5 | Happy Birthday Park Chanyeol Our #Puppy Our Love Our Happy Virus #ChanyeolVirusDay pic.twitter.com/ry9YdkBUfO - via @EXOFIC_THAILAND -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/notes/test/sandbox-toaccept.txt: -------------------------------------------------------------------------------- 1 | “@bahiajagdeep: why salesforce is winning the cloud platform wars http://t.co/rptvphcqjp #bluewolf” 2 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/notes/test/tweeting/tweeting-personal-banned.txt: -------------------------------------------------------------------------------- 1 | I will be presenting on #DDDesign with #Scala and #Akka @oredev next Thursday. Please join me: http://oredev.org/2013/wed-fri-conference/an-open-source-grid-based-actor-model … 2 | I just published an interesting interview on #BigData with @kocolosk, Founder & CTO of @cloudant : http://www.odbms.org/blog/ #odbms #nosql 3 | #lisp An older screencast I made: DSL in Lisp. Domain Specific Languages, development in Common #Lisp with LispWorks https://vimeo.com/77280671 4 | Working on my talk on 'The Road to #Akka Cluster, and Beyond...'. So much to talk about, hard to distill into 1 h. http://skillsmatter.com/podcast/scala/jonas-boner … 5 | If you like to discuss #LISP with me in Milano: ciscolivemilan.com/connect/sessio… #LISP #91 6 | 7 | Follow us on Linkedin - http://linkd.in/V4Fxa5 #Android #iOS #PS3 #Xbox360 #Apps #GameDev #IDRTG #Video #Game #Developer 8 | I'm broadcasting #LIVE on #HangWith for #iPhone! Come Hang w/souljaboy! http://bit.ly/hangwsocial 9 | Our Lead Developer @NicktheDev shares his experience using @ParseIt for push notifications in his #iOS app. Read here bit.ly/187uPV3 10 | My slides from #pgconfeu last week: using and writing #PostgreSQL Extensions, #pgloader for migrating from #MySQL, at http://tapoueh.org/blog/2013/11/05-back-from-dublin … 11 | Our very own (and brilliant) @costinl explains the beauty of #Elasticsearch and its powerful #Hadoop integration - http://www.infoq.com/articles/costin-elasticsearch-bigdata … -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/notes/test/undecided/deal-undecided.txt: -------------------------------------------------------------------------------- 1 | 2 | // by other keywords: 3 | Craigslist Guy Can’t Deal With His Roommate’s Opinions On Syria Anymore http://t.co/qFyIJLAr6Y via @miriamelder 4 | Photo: #latergram #shortrib & #pasta! #yum #dinner #delish #deal #scbd @nicolesten (at Nicole’s Ten... http://t.co/llboiA4fjh 5 | 6 | // by other keywords: buy 7 | #Exclusive: PWC #strikes #deal to buy BGT #Partners — #slideshow http://t.co/2OR3t2U0IZ #digitalmarketing 8 | Microsoft agrees deal to buy Nokia's mobile-phone business for 5.4bn euro ($7.2bn; £4.6bn) http://t.co/cAroa7M62i 9 | Telefónica confirms deal to buy E-Plus for $6.6 billion and create Germany’s largest operator http://t.co/OuQbXm9vWW by @jonrussell 10 | RT @gigastacey: Juniper to buy SDN startup Contrail in deal worth $176M http://t.co/EK9iR4P6 < /me whistles & looks around innocently :D 11 | 12 | // random - they may fall either way, but at the end of the day, it's perfectly fine to reject them 13 | Apple’s Mavericks off to strongest-ever OS X uptake start. Credit the free deal. http://t.co/9P35AnQZu9 -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/notes/test/undecided/deals-random.txt: -------------------------------------------------------------------------------- 1 | // false positive but not really - entirely irrelevant 2 | my third #coffee cup that goes with the third #meeting for #today ... #art #design #galleries #merchandising #deals pic.twitter.com/gBgGtccO 3 | No work today 👍👍 #Deals 4 | They are having us wear #backpacks at work today. #looklikeakid #backtoschool #deals #sale #new… http://instagram.com/p/cUThMiDWSz/ 5 | Another day another dollar. Got a call today. #vc #deals #work pic.twitter.com/qZk1Z7bfOK 6 | Got Mexican food for lunch and I get a discount because I work in the mall #Deals 7 | @oliviacanty Hahahaha I were stood in work today and actually started full on laughing about the shoe incident the other night #deals 8 | Today is my last day of relaxation b/4 the crazy week starts...to say Im excited is an understatement :) #deals #food #work #family #friends 9 | Good morning and L'shana Tova. Last day in Sychrov, back to Prague today and the UK on Tuesday. Let's go to work #deals 10 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/notes/test/undecided/win-random.txt: -------------------------------------------------------------------------------- 1 | // false positive, but entirely random and not relevant: 2 | catch the finale of #WIN this Friday...I love the show. which team will you vote for? pic.twitter.com/SxyD2YE3lF 3 | Most people get gift cards, or baskets from a happy customer . if u work in Harrison u get moonshine #Win pic.twitter.com/X0PcHXuyp1 4 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/persistence-dev.properties: -------------------------------------------------------------------------------- 1 | # Connection information 2 | jdbc.driverClassName=com.mysql.jdbc.Driver 3 | jdbc.url=jdbc:mysql://localhost:3306/stackexchange_dev?createDatabaseIfNotExist=true 4 | jdbc.username=stackUser 5 | jdbc.password=stackmy5ql 6 | 7 | # hibernate.X 8 | hibernate.dialect=org.hibernate.dialect.MySQL5Dialect 9 | hibernate.show_sql=false 10 | hibernate.hbm2ddl.auto=update 11 | 12 | # persistence.X 13 | jpa.generateDdl=true -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/persistence-prod.properties: -------------------------------------------------------------------------------- 1 | # Connection information 2 | jdbc.driverClassName=com.mysql.jdbc.Driver 3 | jdbc.url=jdbc:mysql://localhost:3306/stackexchange_prod?createDatabaseIfNotExist=true 4 | jdbc.username=stackUser 5 | jdbc.password=stackmy5ql 6 | 7 | # hibernate.X 8 | hibernate.dialect=org.hibernate.dialect.MySQL5Dialect 9 | hibernate.show_sql=false 10 | hibernate.hbm2ddl.auto=update 11 | 12 | # persistence.X 13 | jpa.generateDdl=true -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/persistence-setup.properties: -------------------------------------------------------------------------------- 1 | # Connection information 2 | jdbc.driverClassName=com.mysql.jdbc.Driver 3 | jdbc.url=jdbc:mysql://localhost:3306/stackexchange_prod 4 | jdbc.username=stackUser 5 | jdbc.password=stackmy5ql 6 | 7 | # hibernate.X 8 | hibernate.dialect=org.hibernate.dialect.MySQL5Dialect 9 | hibernate.show_sql=false 10 | hibernate.hbm2ddl.auto=update 11 | 12 | # persistence.X 13 | jpa.generateDdl=true -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/persistence-test.properties: -------------------------------------------------------------------------------- 1 | # Connection information 2 | jdbc.driverClassName=org.h2.Driver 3 | jdbc.url=jdbc:h2:mem:stackexchange_dev;DB_CLOSE_DELAY=-1 4 | jdbc.username=stackUser 5 | jdbc.password=stackmy5ql 6 | 7 | # hibernate.X 8 | hibernate.dialect=org.hibernate.dialect.H2Dialect 9 | hibernate.show_sql=false 10 | hibernate.hbm2ddl.auto=create-drop 11 | 12 | # persistence.X 13 | jpa.generateDdl=true -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/rssPersistenceConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/setup.properties: -------------------------------------------------------------------------------- 1 | mode.maintainance.se=false 2 | mode.maintainance.rt=false 3 | mode.maintainance.rss=false 4 | setup.do=false 5 | 6 | setup.upgrade.cleanretweettext.do=false 7 | setup.upgrade.retweettext.do=false 8 | setup.upgrade.retweets.removeduplicates.do=false 9 | setup.upgrade.retweets.local.removeduplicates.do=false 10 | setup.upgrade.remove.orphanedretweets.do=false 11 | 12 | setup.upgrade.retweetmissing.do=false 13 | setup.upgrade.stackquestionsmissing.do=false 14 | setup.upgrade.retweetdates.do=false 15 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/stackPersistenceConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/twitter/interactions.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 1. Deserves Freshness 4 | 5 | - call to join event 6 | DON'T MISS: Join @russmiles for a talk on #Clojure #Lisp & #Simplicity on THURSDAY @skillsmatter: http://ow.ly/nCZ5o #london 7 | 8 | - possible signals: `don't miss`, `talk`, `@skillsmatter`, location (#london) 9 | 10 | 11 | 12 | 2. Types of pre/post fixes: 13 | Useful stuff. 14 | Very cool. -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/resources/twitter/schedule.txt: -------------------------------------------------------------------------------- 1 | 2 | # The StackExchange API / TweetStackexchangeScheduler 3 | 4 | 10 - 5 | 11 - 6 | 12 - 8 7 | 13 - 8 8 | 14 - 8 9 | 15 - 8 10 | 16 - 7 11 | 17 - 8 12 | 18 - 8 13 | 19 - 8 14 | 20 - 7 15 | 21 - 8 16 | 22 - 17 | 23 - 18 | 19 | 20 | # The Twitter API (reads) / TweetMetaScheduler 21 | 22 | 9 - 7 23 | 10 - 7 24 | 11 - 7 25 | 12 - 7 26 | 13 - 7 27 | 14 - 7 28 | 15 - 7 29 | 16 - 7 30 | 17 - 7 31 | 18 - 7 32 | 19 - 7 33 | 20 - 7 34 | 21 - 7 35 | 22 - 7 36 | 23 - 7 37 | 38 | 39 | # RSS 40 | 41 | 9 - 42 | 10 - 43 | 11 - 44 | 12 - 45 | 13 - 46 | 14 - 47 | 15 - 48 | 16 - 49 | 17 - 50 | 18 - 51 | 19 - 52 | 20 - 53 | 21 - 54 | 22 - 55 | 23 - 56 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/webapp/META-INF/MANIFEST.MF: -------------------------------------------------------------------------------- 1 | Manifest-Version: 1.0 2 | Class-Path: 3 | 4 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/main/webapp/WEB-INF/web.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | stackexchange2twitter 9 | 10 | 11 | 12 | contextClass 13 | org.springframework.web.context.support.AnnotationConfigWebApplicationContext 14 | 15 | 16 | contextConfigLocation 17 | org.common.spring org.classification.spring org.keyval.spring org.tweet.spring org.stackexchange.spring org.tweet.meta.spring org.rss.spring 18 | 19 | 20 | 21 | org.springframework.web.context.ContextLoaderListener 22 | 23 | 24 | contextInitializerClasses 25 | org.common.spring.MyApplicationContextInitializerProv 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/classification/data/ClassificationDataCleanupManualTest.java: -------------------------------------------------------------------------------- 1 | package org.classification.data; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.File; 5 | import java.io.FileWriter; 6 | import java.io.IOException; 7 | import java.io.InputStream; 8 | import java.io.InputStreamReader; 9 | import java.util.List; 10 | 11 | import org.apache.commons.io.IOUtils; 12 | import org.classification.data.ClassificationData.CommercialDataApi; 13 | import org.classification.util.ClassificationSettings; 14 | import org.junit.Test; 15 | import org.stackexchange.gather.CleanupStringFunction; 16 | 17 | import com.google.common.collect.Lists; 18 | 19 | public class ClassificationDataCleanupManualTest { 20 | 21 | // tests 22 | 23 | @Test 24 | public final void whenLoadingTrainClassificationData_thenNoExceptions() throws IOException { 25 | CommercialDataApi.nonCommercialTrainingData(ClassificationSettings.PROBES_FOR_CONTENT_ENCODER_VECTOR, ClassificationSettings.FEATURES); 26 | } 27 | 28 | @Test 29 | public final void whenLoadingTestClassificationData_thenNoExceptions() throws IOException { 30 | CommercialDataApi.nonCommercialTestData(); 31 | } 32 | 33 | @Test 34 | public final void whenCleaningClassificationData_thenNoExceptions() throws IOException { 35 | final String path = ClassificationData.Other.SAMPLE; 36 | final InputStream is = ClassificationData.class.getResourceAsStream(path); 37 | final List tweetsForClassification = IOUtils.readLines(new BufferedReader(new InputStreamReader(is))); 38 | 39 | final List tweetsClean = Lists.transform(tweetsForClassification, new CleanupStringFunction()); 40 | final File file = new File("/opt/sandbox/classification_clean.classif"); 41 | final FileWriter fw = new FileWriter(file); // it creates the file writer and the actual file 42 | IOUtils.writeLines(tweetsClean, "\n", fw); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/classification/data/SpecificClassificationDataUtilIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package org.classification.data; 2 | 3 | import java.io.IOException; 4 | 5 | import org.classification.data.ClassificationData.JobsDataApi; 6 | import org.classification.util.ClassificationSettings; 7 | import org.junit.Test; 8 | 9 | public final class SpecificClassificationDataUtilIntegrationTest { 10 | 11 | @Test 12 | public final void whenTrainingDataIsCreated_thenNoExceptions() throws IOException { 13 | JobsDataApi.jobsCoreTrainingData(ClassificationSettings.PROBES_FOR_CONTENT_ENCODER_VECTOR, ClassificationSettings.FEATURES); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/classification/service/ClassificationServiceIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package org.classification.service; 2 | 3 | import static org.junit.Assert.assertTrue; 4 | 5 | import java.io.BufferedReader; 6 | import java.io.FileReader; 7 | import java.io.IOException; 8 | import java.util.List; 9 | 10 | import org.apache.commons.io.IOUtils; 11 | import org.classification.data.ClassificationData.Jobs; 12 | import org.classification.spring.ClassificationConfig; 13 | import org.common.spring.CommonServiceConfig; 14 | import org.gplus.spring.GplusContextConfig; 15 | import org.junit.Test; 16 | import org.junit.runner.RunWith; 17 | import org.springframework.beans.factory.annotation.Autowired; 18 | import org.springframework.test.context.ContextConfiguration; 19 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 20 | 21 | @RunWith(SpringJUnit4ClassRunner.class) 22 | @ContextConfiguration(classes = { CommonServiceConfig.class, ClassificationConfig.class, GplusContextConfig.class }) 23 | public class ClassificationServiceIntegrationTest { 24 | 25 | @Autowired 26 | private ClassificationService classificationService; 27 | 28 | // tests 29 | 30 | @Test 31 | public final void whenContextIsBootstrapped_thenNoException() { 32 | // 33 | } 34 | 35 | @Test 36 | public final void whenReadingClassificationTrainingFile_thenNoException() throws IOException { 37 | final List lines = IOUtils.readLines(new BufferedReader(new FileReader("src/main/resources" + Jobs.Training.JOBS))); 38 | System.out.println(lines); 39 | } 40 | 41 | @Test 42 | public final void whenTweetIsClassified_thenNoException() { 43 | final boolean isMatch = classificationService.isJobDefault("URGENT: Scala Developer | 3 Month Contract | Westminster | Immediate Requirement! #Scala #Freelance #Jobs #IT"); 44 | assertTrue(isMatch); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/classification/service/ClassificationServiceLiveTest.java: -------------------------------------------------------------------------------- 1 | package org.classification.service; 2 | 3 | import java.io.IOException; 4 | 5 | import org.classification.service.accuracy.ClassificationJobsAccuracyTestService; 6 | import org.classification.spring.ClassificationConfig; 7 | import org.common.spring.CommonServiceConfig; 8 | import org.gplus.spring.GplusContextConfig; 9 | import org.junit.Ignore; 10 | import org.junit.Test; 11 | import org.junit.runner.RunWith; 12 | import org.springframework.beans.factory.annotation.Autowired; 13 | import org.springframework.test.context.ActiveProfiles; 14 | import org.springframework.test.context.ContextConfiguration; 15 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 16 | import org.tweet.spring.util.SpringProfileUtil; 17 | 18 | /** 19 | * - note: this test is not actually live, just long running 20 | */ 21 | @RunWith(SpringJUnit4ClassRunner.class) 22 | @ContextConfiguration(classes = { CommonServiceConfig.class, ClassificationConfig.class, GplusContextConfig.class }) 23 | @ActiveProfiles(SpringProfileUtil.LIVE) 24 | public class ClassificationServiceLiveTest { 25 | 26 | @Autowired 27 | private ClassificationJobsAccuracyTestService classificationAccuracyService; 28 | 29 | // tests 30 | 31 | // 5000 features: 32 | // 10000 features: 33 | /** 34 | * - note: the data to be classified has EMPTY type information included in the encoded vector
35 | * - so the results are production-like, but not excellent 36 | */ 37 | @Test 38 | @Ignore("long running - ignored by default") 39 | public final void givenClassifierWasTrained_whenClassifyingTestDataWithoutTypeInfo_thenResultsAreGood() throws IOException { 40 | // final int runs = 750; 41 | final int runs = 300; 42 | // final double mean = classificationAccuracyService.calculateJobsClassifierAccuracyWithCoreTrainingDataDefault(runs); 43 | final double mean = classificationAccuracyService.calculateJobsClassifierAccuracyWithFullTrainingDataDefault(runs); 44 | System.out.println("Average Success Rate: " + mean); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/common/persistence/IEntityOperations.java: -------------------------------------------------------------------------------- 1 | package org.common.persistence; 2 | 3 | public interface IEntityOperations { 4 | 5 | T createNewEntity(); 6 | 7 | void invalidate(final T entity); 8 | 9 | void change(final T resource); 10 | 11 | } 12 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/common/persistence/setup/upgrades/nolonger/AddTextToRetweetsUpgraderLiveManualTest.java: -------------------------------------------------------------------------------- 1 | package org.common.persistence.setup.upgrades.nolonger; 2 | 3 | import org.common.persistence.setup.upgrades.live.nolonger.IAddTextToRetweetsUpgrader; 4 | import org.common.spring.CommonPersistenceJPAConfig; 5 | import org.common.spring.CommonServiceConfig; 6 | import org.common.spring.MyApplicationContextInitializerProv; 7 | import org.junit.Test; 8 | import org.junit.runner.RunWith; 9 | import org.keyval.spring.KeyValPersistenceJPAConfig; 10 | import org.springframework.beans.factory.annotation.Autowired; 11 | import org.springframework.test.context.ActiveProfiles; 12 | import org.springframework.test.context.ContextConfiguration; 13 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 14 | import org.stackexchange.util.TwitterAccountEnum; 15 | import org.tweet.meta.spring.TwitterMetaPersistenceJPAConfig; 16 | import org.tweet.spring.TwitterConfig; 17 | import org.tweet.spring.TwitterLiveConfig; 18 | import org.tweet.spring.util.SpringProfileUtil; 19 | 20 | @RunWith(SpringJUnit4ClassRunner.class) 21 | @ContextConfiguration(classes = { CommonServiceConfig.class, CommonPersistenceJPAConfig.class, KeyValPersistenceJPAConfig.class, TwitterConfig.class, TwitterLiveConfig.class, TwitterMetaPersistenceJPAConfig.class }) 22 | @ActiveProfiles({ SpringProfileUtil.DEPLOYED, SpringProfileUtil.LIVE }) 23 | public class AddTextToRetweetsUpgraderLiveManualTest { 24 | 25 | static { 26 | System.setProperty(MyApplicationContextInitializerProv.PERSISTENCE_TARGET_KEY, "prod"); 27 | } 28 | 29 | @Autowired 30 | private IAddTextToRetweetsUpgrader addTextToRetweetsUpgrader; 31 | 32 | // fixtures 33 | 34 | // tests 35 | 36 | @Test 37 | public final void whenRecreatingTheTweetedQuestions_thenNoExceptions() { 38 | addTextToRetweetsUpgrader.addTextOfRetweets(); 39 | } 40 | 41 | @Test 42 | public final void whenRecreatingTheTweetedQuestionsOnSpecificAccount_thenNoExceptions() { 43 | addTextToRetweetsUpgrader.addTextOfRetweetsOnAccount(TwitterAccountEnum.ScalaFact.name()); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/common/service/ContentExtractorLiveServiceLiveTest.java: -------------------------------------------------------------------------------- 1 | package org.common.service; 2 | 3 | import static org.hamcrest.Matchers.equalTo; 4 | import static org.junit.Assert.assertThat; 5 | 6 | import java.io.IOException; 7 | import java.net.MalformedURLException; 8 | 9 | import org.common.spring.CommonServiceConfig; 10 | import org.junit.Test; 11 | import org.junit.runner.RunWith; 12 | import org.springframework.beans.factory.annotation.Autowired; 13 | import org.springframework.test.context.ActiveProfiles; 14 | import org.springframework.test.context.ContextConfiguration; 15 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 16 | import org.tweet.spring.util.SpringProfileUtil; 17 | 18 | @RunWith(SpringJUnit4ClassRunner.class) 19 | @ContextConfiguration(classes = { CommonServiceConfig.class }) 20 | @ActiveProfiles(SpringProfileUtil.LIVE) 21 | public class ContentExtractorLiveServiceLiveTest { 22 | 23 | @Autowired 24 | private ContentExtractorLiveService contentExtractorService; 25 | 26 | // tests 27 | 28 | @Test 29 | public final void whenContextIsInitialized_thenNoExceptions() { 30 | // 31 | } 32 | 33 | @Test 34 | public final void whenContentIsExtractedFromUrl_thenNoExceptions() throws MalformedURLException, IOException { 35 | assertThat(contentExtractorService.extractTitle("http://www.google.com"), equalTo("Google")); 36 | } 37 | 38 | // util 39 | 40 | } 41 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/common/service/external/bitly/BitlyServiceLiveTest.java: -------------------------------------------------------------------------------- 1 | package org.common.service.external.bitly; 2 | 3 | import static org.hamcrest.Matchers.startsWith; 4 | import static org.junit.Assert.assertNotNull; 5 | import static org.junit.Assert.assertThat; 6 | 7 | import org.common.spring.CommonServiceConfig; 8 | import org.junit.Test; 9 | import org.junit.runner.RunWith; 10 | import org.springframework.beans.factory.annotation.Autowired; 11 | import org.springframework.test.context.ContextConfiguration; 12 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 13 | 14 | @RunWith(SpringJUnit4ClassRunner.class) 15 | @ContextConfiguration(classes = {// @formatter:off 16 | CommonServiceConfig.class 17 | }) // @formatter:on 18 | public class BitlyServiceLiveTest { 19 | 20 | @Autowired 21 | private BitlyService instance; 22 | 23 | // tests 24 | 25 | @Test 26 | public final void whenContextIsBootstrapped_thenNoException() { 27 | assertNotNull(instance); 28 | } 29 | 30 | @Test 31 | public final void whenUrlIsShortened_thenCorrect() { 32 | final String shortenedUrl = instance.shortenUrl("http://www.baeldung.com"); 33 | assertNotNull(shortenedUrl); 34 | assertThat(shortenedUrl, startsWith("http://bit.ly/")); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/keyval/persistence/service/KeyValEntityOps.java: -------------------------------------------------------------------------------- 1 | package org.keyval.persistence.service; 2 | 3 | import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; 4 | 5 | import org.common.persistence.IEntityOperations; 6 | import org.keyval.persistence.model.KeyVal; 7 | import org.springframework.stereotype.Component; 8 | 9 | @Component 10 | public final class KeyValEntityOps implements IEntityOperations { 11 | 12 | public KeyValEntityOps() { 13 | super(); 14 | } 15 | 16 | // API 17 | 18 | // template method 19 | 20 | @Override 21 | public final KeyVal createNewEntity() { 22 | return new KeyVal(randomAlphabetic(6), randomAlphabetic(6)); 23 | } 24 | 25 | @Override 26 | public final void invalidate(final KeyVal entity) { 27 | entity.setKey(null); 28 | entity.setValue(null); 29 | } 30 | 31 | @Override 32 | public final void change(final KeyVal entity) { 33 | entity.setKey(randomAlphabetic(6)); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/keyval/persistence/service/KeyValPersistenceIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package org.keyval.persistence.service; 2 | 3 | import org.common.persistence.AbstractRawServicePersistenceIntegrationTest; 4 | import org.common.persistence.IEntityOperations; 5 | import org.junit.Test; 6 | import org.junit.runner.RunWith; 7 | import org.keyval.persistence.dao.IKeyValJpaDAO; 8 | import org.keyval.persistence.model.KeyVal; 9 | import org.keyval.spring.KeyValPersistenceJPAConfig; 10 | import org.springframework.beans.factory.annotation.Autowired; 11 | import org.springframework.test.context.ContextConfiguration; 12 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 13 | import org.tweet.spring.TestStackexchangePersistenceJPAConfig; 14 | 15 | @RunWith(SpringJUnit4ClassRunner.class) 16 | @ContextConfiguration(classes = { KeyValPersistenceJPAConfig.class, TestStackexchangePersistenceJPAConfig.class }) 17 | public class KeyValPersistenceIntegrationTest extends AbstractRawServicePersistenceIntegrationTest { 18 | 19 | @Autowired 20 | private IKeyValJpaDAO api; 21 | 22 | @Autowired 23 | private KeyValEntityOps entityOps; 24 | 25 | // tests 26 | 27 | // find - one - by name 28 | 29 | // find - all 30 | 31 | // count - all 32 | 33 | @Test 34 | /**/public void whenAllResourcesAreCounted_thenNoExceptions() { 35 | getApi().count(); 36 | } 37 | 38 | // create 39 | 40 | // template method 41 | 42 | @Override 43 | protected IKeyValJpaDAO getApi() { 44 | return api; 45 | } 46 | 47 | @Override 48 | protected IEntityOperations getEntityOps() { 49 | return entityOps; 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/rss/persistence/service/RssEntryEntityOps.java: -------------------------------------------------------------------------------- 1 | package org.rss.persistence.service; 2 | 3 | import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; 4 | 5 | import java.util.Date; 6 | 7 | import org.common.persistence.IEntityOperations; 8 | import org.rss.persistence.model.RssEntry; 9 | import org.springframework.stereotype.Component; 10 | 11 | @Component 12 | public final class RssEntryEntityOps implements IEntityOperations { 13 | 14 | public RssEntryEntityOps() { 15 | super(); 16 | } 17 | 18 | // API 19 | 20 | // template method 21 | 22 | @Override 23 | public final RssEntry createNewEntity() { 24 | return new RssEntry(randomAlphabetic(6), randomAlphabetic(6), randomAlphabetic(6), new Date(), new Date()); 25 | } 26 | 27 | @Override 28 | public final void invalidate(final RssEntry entity) { 29 | entity.setTitle(null); 30 | entity.setTwitterAccount(null); 31 | } 32 | 33 | @Override 34 | public final void change(final RssEntry entity) { 35 | entity.setTitle(randomAlphabetic(6)); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/rss/service/RssEntryPersistenceIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package org.rss.service; 2 | 3 | import org.common.persistence.AbstractRawServicePersistenceIntegrationTest; 4 | import org.common.persistence.IEntityOperations; 5 | import org.common.spring.MyApplicationContextInitializerProv; 6 | import org.junit.Test; 7 | import org.junit.runner.RunWith; 8 | import org.rss.persistence.dao.IRssEntryJpaDAO; 9 | import org.rss.persistence.model.RssEntry; 10 | import org.rss.persistence.service.RssEntryEntityOps; 11 | import org.rss.spring.RssPersistenceJPAConfig; 12 | import org.springframework.beans.factory.annotation.Autowired; 13 | import org.springframework.test.context.ContextConfiguration; 14 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 15 | 16 | @RunWith(SpringJUnit4ClassRunner.class) 17 | @ContextConfiguration(classes = { RssPersistenceJPAConfig.class }) 18 | public class RssEntryPersistenceIntegrationTest extends AbstractRawServicePersistenceIntegrationTest { 19 | 20 | static { 21 | System.setProperty(MyApplicationContextInitializerProv.PERSISTENCE_TARGET_KEY, "test"); 22 | } 23 | 24 | @Autowired 25 | private IRssEntryJpaDAO api; 26 | 27 | @Autowired 28 | private RssEntryEntityOps entityOps; 29 | 30 | // tests 31 | 32 | // find - one - by name 33 | 34 | // find - all 35 | 36 | // count - all 37 | 38 | @Test 39 | /**/public void whenAllResourcesAreCounted_thenNoExceptions() { 40 | getApi().count(); 41 | } 42 | 43 | // create 44 | 45 | // update 46 | 47 | // template method 48 | 49 | @Override 50 | protected IRssEntryJpaDAO getApi() { 51 | return api; 52 | } 53 | 54 | @Override 55 | protected final IEntityOperations getEntityOps() { 56 | return entityOps; 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/rss/service/RssServiceReadOnlyLiveTest.java: -------------------------------------------------------------------------------- 1 | package org.rss.service; 2 | 3 | import static org.hamcrest.Matchers.empty; 4 | import static org.hamcrest.Matchers.not; 5 | import static org.hamcrest.Matchers.notNullValue; 6 | import static org.junit.Assert.assertThat; 7 | 8 | import java.io.IOException; 9 | import java.util.List; 10 | 11 | import org.junit.Test; 12 | import org.junit.runner.RunWith; 13 | import org.rss.persistence.model.RssEntry; 14 | import org.rss.spring.RssContextConfig; 15 | import org.springframework.beans.factory.annotation.Autowired; 16 | import org.springframework.test.context.ActiveProfiles; 17 | import org.springframework.test.context.ContextConfiguration; 18 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 19 | import org.tweet.spring.util.SpringProfileUtil; 20 | 21 | import com.sun.syndication.io.FeedException; 22 | 23 | @RunWith(SpringJUnit4ClassRunner.class) 24 | @ContextConfiguration(classes = { RssContextConfig.class }) 25 | @ActiveProfiles(SpringProfileUtil.LIVE) 26 | public class RssServiceReadOnlyLiveTest { 27 | 28 | @Autowired 29 | private RssService rssService; 30 | 31 | // tests 32 | 33 | @Test 34 | public final void whenConsumingAnRssFeed_theenFeedsAreExtracted() throws IOException, IllegalArgumentException, FeedException { 35 | final List feeds = rssService.extractTitlesAndLinks("http://feeds.feedburner.com/baeldung"); 36 | assertThat(feeds, notNullValue()); 37 | assertThat(feeds, not(empty())); 38 | 39 | for (final RssEntry feed : feeds) { 40 | System.out.println(feed); 41 | } 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/stackexchange/ITwitterWriteLiveServiceLiveTest.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange; 2 | 3 | import org.junit.Ignore; 4 | import org.junit.Test; 5 | import org.junit.runner.RunWith; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.test.context.ActiveProfiles; 8 | import org.springframework.test.context.ContextConfiguration; 9 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 10 | import org.stackexchange.spring.StackexchangeContextConfig; 11 | import org.stackexchange.util.TwitterAccountEnum; 12 | import org.tweet.spring.TwitterConfig; 13 | import org.tweet.spring.util.SpringProfileUtil; 14 | import org.tweet.twitter.service.TwitterTemplateCreator; 15 | import org.tweet.twitter.service.live.ITwitterWriteLiveService; 16 | import org.tweet.twitter.service.live.TwitterReadLiveService; 17 | 18 | @RunWith(SpringJUnit4ClassRunner.class) 19 | @ContextConfiguration(classes = { TwitterConfig.class, StackexchangeContextConfig.class }) 20 | @ActiveProfiles({ SpringProfileUtil.LIVE, SpringProfileUtil.WRITE_PRODUCTION }) 21 | @Ignore("by default, there should be no component that is not deployed in production, configured to tweet") 22 | public class ITwitterWriteLiveServiceLiveTest { 23 | 24 | @Autowired 25 | private TwitterReadLiveService twitterReadLiveService; 26 | @Autowired 27 | protected ITwitterWriteLiveService twitterWriteLiveService; 28 | 29 | @Autowired 30 | private TwitterTemplateCreator twitterCreator; 31 | 32 | // tests 33 | 34 | @Test 35 | public final void whenTweeting_thenNoExceptions() { 36 | twitterWriteLiveService.tweet(TwitterAccountEnum.BestBash.name(), "What are Unity's keyboard and mouse shortcuts?"); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/stackexchange/component/MinStackScoreTagRetrieverIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.component; 2 | 3 | import static org.junit.Assert.assertNotNull; 4 | 5 | import org.common.spring.CommonServiceConfig; 6 | import org.junit.Test; 7 | import org.junit.runner.RunWith; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.test.context.ContextConfiguration; 10 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 11 | import org.stackexchange.api.constants.StackSite; 12 | import org.stackexchange.spring.StackexchangeConfig; 13 | import org.stackexchange.spring.StackexchangeContextConfig; 14 | import org.stackexchange.util.StackTag; 15 | import org.tweet.spring.TwitterConfig; 16 | 17 | @RunWith(SpringJUnit4ClassRunner.class) 18 | @ContextConfiguration(classes = {// @formatter:off 19 | CommonServiceConfig.class, 20 | TwitterConfig.class, 21 | StackexchangeConfig.class, 22 | StackexchangeContextConfig.class 23 | })// @formatter:off 24 | // @ActiveProfiles(SpringProfileUtil.LIVE) 25 | public class MinStackScoreTagRetrieverIntegrationTest { 26 | 27 | @Autowired 28 | private MinStackScoreRetriever minStackScoreRetriever; 29 | 30 | // API 31 | 32 | /** 33 | * twitterInternal.properties 34 | */ 35 | @Test 36 | public final void givenOnStackOverFlowOnly_whenRetrievingMinStackExchangeScoreForTag_thenFound() { 37 | for (final StackTag stackTag : StackTag.values()) { 38 | assertNotNull("No min score for tag " + stackTag, minStackScoreRetriever.minScoreRaw(stackTag.name(), StackSite.StackOverflow)); 39 | } 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/stackexchange/component/StackExchangePageStrategyUnitTest.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.component; 2 | 3 | import static org.hamcrest.Matchers.equalTo; 4 | import static org.hamcrest.Matchers.lessThan; 5 | import static org.junit.Assert.assertThat; 6 | 7 | import org.junit.Test; 8 | 9 | public final class StackExchangePageStrategyUnitTest { 10 | 11 | // tests 12 | 13 | @Test 14 | public final void whenDecidingCorrectPage_thenNoExceptions() { 15 | new StackExchangePageStrategy().decidePageInternal(1); 16 | } 17 | 18 | @Test 19 | public final void givenPageIs1_whenDecidingCorrectPage_thenPageIs1() { 20 | final int decidedPage = new StackExchangePageStrategy().decidePageInternal(5); 21 | assertThat(decidedPage, equalTo(1)); 22 | } 23 | 24 | @Test 25 | public final void givenPageIs2_whenDecidingCorrectPage_thenPageIs2() { 26 | final int decidedPage = new StackExchangePageStrategy().decidePageInternal(35); 27 | assertThat(decidedPage, equalTo(2)); 28 | } 29 | 30 | @Test 31 | public final void givenPageIs3_whenDecidingCorrectPage_thenPageIs3() { 32 | final int decidedPage = new StackExchangePageStrategy().decidePageInternal(65); 33 | assertThat(decidedPage, equalTo(3)); 34 | } 35 | 36 | @Test 37 | public final void givenPageIsOver3_whenDecidingCorrectPage_thenCorrect() { 38 | final int decidedPage = new StackExchangePageStrategy().decidePageInternal(155); 39 | assertThat(decidedPage, lessThan(9)); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/stackexchange/gather/CleanupStringFunction.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.gather; 2 | 3 | import org.common.util.TextUtil; 4 | 5 | import com.google.common.base.Function; 6 | 7 | public class CleanupStringFunction implements Function { 8 | 9 | @Override 10 | public final String apply(final String input) { 11 | String cleanText = TextUtil.cleanupInvalidCharacters(input); 12 | cleanText = cleanText.replaceAll(" ", " "); 13 | cleanText = cleanText.trim(); 14 | if (cleanText.matches(".*&\\S*;.*")) { 15 | return ""; 16 | } 17 | return cleanText; 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/stackexchange/gather/ContainsKeywordPredicate.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.gather; 2 | 3 | import java.util.List; 4 | 5 | import org.classification.util.TweetSettings; 6 | 7 | import com.google.api.client.util.Preconditions; 8 | import com.google.common.base.CharMatcher; 9 | import com.google.common.base.Predicate; 10 | import com.google.common.base.Splitter; 11 | import com.google.common.collect.Lists; 12 | 13 | public class ContainsKeywordPredicate implements Predicate { 14 | 15 | private final List keywords; 16 | 17 | public ContainsKeywordPredicate(final List keywords) { 18 | super(); 19 | this.keywords = Preconditions.checkNotNull(keywords); 20 | } 21 | 22 | // API 23 | 24 | @Override 25 | public final boolean apply(final String input) { 26 | final String inputInternal = input.toLowerCase(); 27 | final List tweetTokens = Lists.newArrayList(Splitter.on(CharMatcher.anyOf(TweetSettings.TWEET_TOKENIZER + "#[]")).split(inputInternal)); 28 | for (final String token : tweetTokens) { 29 | if (keywords.contains(token)) { 30 | return true; 31 | } 32 | } 33 | 34 | return false; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/stackexchange/persistence/service/QuestionTweetEntityOps.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.persistence.service; 2 | 3 | import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; 4 | 5 | import java.util.Date; 6 | 7 | import org.common.persistence.IEntityOperations; 8 | import org.springframework.stereotype.Component; 9 | import org.stackexchange.persistence.model.QuestionTweet; 10 | 11 | @Component 12 | public final class QuestionTweetEntityOps implements IEntityOperations { 13 | 14 | public QuestionTweetEntityOps() { 15 | super(); 16 | } 17 | 18 | // API 19 | 20 | // template method 21 | 22 | @Override 23 | public final QuestionTweet createNewEntity() { 24 | return new QuestionTweet(randomAlphabetic(6), randomAlphabetic(6), randomAlphabetic(6), new Date()); 25 | } 26 | 27 | @Override 28 | public final void invalidate(final QuestionTweet entity) { 29 | entity.setQuestionId(null); 30 | entity.setTwitterAccount(null); 31 | } 32 | 33 | @Override 34 | public final void change(final QuestionTweet entity) { 35 | entity.setQuestionId(randomAlphabetic(6)); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/stackexchange/persistence/service/QuestionTweetPersistenceIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package org.stackexchange.persistence.service; 2 | 3 | import org.common.persistence.AbstractRawServicePersistenceIntegrationTest; 4 | import org.common.persistence.IEntityOperations; 5 | import org.junit.Test; 6 | import org.junit.runner.RunWith; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.test.context.ContextConfiguration; 9 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 10 | import org.stackexchange.persistence.dao.IQuestionTweetJpaDAO; 11 | import org.stackexchange.persistence.model.QuestionTweet; 12 | import org.stackexchange.util.TwitterAccountEnum; 13 | import org.tweet.spring.TestStackexchangePersistenceJPAConfig; 14 | 15 | @RunWith(SpringJUnit4ClassRunner.class) 16 | @ContextConfiguration(classes = { TestStackexchangePersistenceJPAConfig.class }) 17 | public class QuestionTweetPersistenceIntegrationTest extends AbstractRawServicePersistenceIntegrationTest { 18 | 19 | @Autowired 20 | private IQuestionTweetJpaDAO api; 21 | 22 | @Autowired 23 | private QuestionTweetEntityOps entityOps; 24 | 25 | // tests 26 | 27 | // find - one - by name 28 | 29 | // find - all 30 | 31 | // count - all 32 | 33 | @Test 34 | /**/public void whenAllResourcesAreCounted_thenNoExceptions() { 35 | getApi().count(); 36 | } 37 | 38 | @Test 39 | /**/public void whenAllResourcesAreCountedByTwitterAccount_thenNoExceptions() { 40 | final long countAllByTwitterAccount = getApi().countAllByTwitterAccount(TwitterAccountEnum.AskUbuntuBest.name()); 41 | System.out.println(countAllByTwitterAccount); 42 | } 43 | 44 | // create 45 | 46 | // template method 47 | 48 | @Override 49 | protected IQuestionTweetJpaDAO getApi() { 50 | return api; 51 | } 52 | 53 | @Override 54 | protected IEntityOperations getEntityOps() { 55 | return entityOps; 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/meta/persistence/service/RetweetEntityOps.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta.persistence.service; 2 | 3 | import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; 4 | 5 | import java.util.Date; 6 | 7 | import org.common.persistence.IEntityOperations; 8 | import org.springframework.stereotype.Component; 9 | import org.stackexchange.util.IDUtil; 10 | import org.tweet.meta.persistence.model.Retweet; 11 | 12 | @Component 13 | public final class RetweetEntityOps implements IEntityOperations { 14 | 15 | public RetweetEntityOps() { 16 | super(); 17 | } 18 | 19 | // API 20 | 21 | // template method 22 | 23 | @Override 24 | public final Retweet createNewEntity() { 25 | return new Retweet(IDUtil.randomPositiveLong(), randomAlphabetic(6), randomAlphabetic(6), new Date()); 26 | } 27 | 28 | public final Retweet createNewEntity(final String twitterAccount) { 29 | return new Retweet(IDUtil.randomPositiveLong(), twitterAccount, randomAlphabetic(6), new Date()); 30 | } 31 | 32 | @Override 33 | public final void invalidate(final Retweet entity) { 34 | entity.setText(null); 35 | entity.setTwitterAccount(null); 36 | } 37 | 38 | @Override 39 | public final void change(final Retweet entity) { 40 | entity.setText(randomAlphabetic(6)); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/meta/service/InteractionLiveServiceMockUnitTest.java: -------------------------------------------------------------------------------- 1 | package org.tweet.meta.service; 2 | 3 | import static org.mockito.Matchers.any; 4 | import static org.mockito.Matchers.anyString; 5 | import static org.mockito.Mockito.mock; 6 | import static org.mockito.Mockito.when; 7 | 8 | import org.junit.Before; 9 | import org.junit.Test; 10 | import org.slf4j.Logger; 11 | import org.springframework.social.twitter.api.TwitterProfile; 12 | import org.tweet.meta.component.TwitterInteractionValuesRetriever; 13 | import org.tweet.twitter.service.TweetMentionService; 14 | import org.tweet.twitter.service.TweetService; 15 | import org.tweet.twitter.service.live.TwitterReadLiveService; 16 | import org.tweet.twitter.service.live.UserLiveService; 17 | import org.tweet.twitter.util.TwitterInteraction; 18 | import org.tweet.twitter.util.TwitterInteractionWithValue; 19 | 20 | public final class InteractionLiveServiceMockUnitTest { 21 | 22 | private InteractionLiveService instance; 23 | 24 | // fixtures 25 | 26 | @Before 27 | public final void before() { 28 | this.instance = mock(InteractionLiveService.class); 29 | this.instance.logger = mock(Logger.class); 30 | this.instance.twitterInteractionValuesRetriever = mock(TwitterInteractionValuesRetriever.class); 31 | this.instance.twitterReadLiveService = mock(TwitterReadLiveService.class); 32 | this.instance.tweetMentionService = mock(TweetMentionService.class); 33 | this.instance.userLiveService = mock(UserLiveService.class); 34 | this.instance.tweetService = mock(TweetService.class); 35 | 36 | when(this.instance.twitterInteractionValuesRetriever.getPagesToAnalyze()).thenReturn(1); 37 | when(this.instance.twitterInteractionValuesRetriever.getMaxRetweetsForTweet()).thenReturn(15); 38 | when(this.instance.determineBestInteractionWithAuthorLive(any(TwitterProfile.class), anyString(), anyString())).thenReturn(new TwitterInteractionWithValue(TwitterInteraction.None, 0)); 39 | } 40 | 41 | // tests 42 | 43 | // determine best interaction 44 | 45 | // - tweet itself has mention value 46 | 47 | @Test 48 | public final void when_thenNoExceptions() { 49 | // 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/spring/SetupPersistenceTestConfig.java: -------------------------------------------------------------------------------- 1 | package org.tweet.spring; 2 | 3 | import javax.sql.DataSource; 4 | 5 | import org.apache.tomcat.dbcp.dbcp.BasicDataSource; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | import org.springframework.context.annotation.Profile; 10 | import org.springframework.context.annotation.PropertySource; 11 | import org.springframework.core.env.Environment; 12 | import org.springframework.jdbc.core.JdbcTemplate; 13 | import org.tweet.spring.util.SpringProfileUtil; 14 | 15 | @Configuration 16 | @PropertySource({ "classpath:persistence-${persistenceTarget:setup}.properties" }) 17 | @Profile(SpringProfileUtil.PERSISTENCE) 18 | public class SetupPersistenceTestConfig { 19 | 20 | @Autowired 21 | private Environment env; 22 | 23 | public SetupPersistenceTestConfig() { 24 | super(); 25 | } 26 | 27 | // 28 | 29 | @Bean 30 | public DataSource restDataSource() { 31 | final BasicDataSource dataSource = new BasicDataSource(); 32 | dataSource.setDriverClassName(env.getProperty("jdbc.driverClassName")); 33 | dataSource.setUrl(env.getProperty("jdbc.url")); 34 | dataSource.setUsername(env.getProperty("jdbc.username")); 35 | dataSource.setPassword(env.getProperty("jdbc.password")); 36 | return dataSource; 37 | } 38 | 39 | @Bean 40 | public JdbcTemplate jdbcTemplate() { 41 | return new JdbcTemplate(restDataSource()); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/spring/TestStackexchangePersistenceJPAConfig.java: -------------------------------------------------------------------------------- 1 | package org.tweet.spring; 2 | 3 | import org.common.spring.PersistenceJPACommonConfig; 4 | import org.springframework.context.annotation.ComponentScan; 5 | import org.springframework.context.annotation.Configuration; 6 | import org.springframework.context.annotation.Import; 7 | import org.springframework.context.annotation.ImportResource; 8 | import org.springframework.context.annotation.Profile; 9 | import org.springframework.context.annotation.PropertySource; 10 | import org.tweet.spring.util.SpringProfileUtil; 11 | 12 | @Configuration 13 | @Import(PersistenceJPACommonConfig.class) 14 | @ComponentScan({ "org.stackexchange.persistence" }) 15 | @ImportResource({ "classpath*:stackPersistenceConfig.xml" }) 16 | @PropertySource({ "classpath:persistence-${persistenceTarget:test}.properties", "classpath:setup-test.properties" }) 17 | @Profile(SpringProfileUtil.PERSISTENCE) 18 | public class TestStackexchangePersistenceJPAConfig { 19 | 20 | public TestStackexchangePersistenceJPAConfig() { 21 | super(); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/test/FullTestSuite.java: -------------------------------------------------------------------------------- 1 | package org.tweet.test; 2 | 3 | import org.junit.runner.RunWith; 4 | import org.junit.runners.Suite; 5 | import org.junit.runners.Suite.SuiteClasses; 6 | 7 | @RunWith(Suite.class) 8 | @SuiteClasses({ UnitTestSuite.class, LiveReadOnlyTestSuite.class, IntegrationTestSuite.class }) 9 | public final class FullTestSuite { 10 | // 11 | } 12 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/test/IntegrationTestSuite.java: -------------------------------------------------------------------------------- 1 | package org.tweet.test; 2 | 3 | import org.classification.data.SpecificClassificationDataUtilIntegrationTest; 4 | import org.classification.service.ClassificationServiceIntegrationTest; 5 | import org.common.service.LinkServiceIntegrationTest; 6 | import org.junit.runner.RunWith; 7 | import org.junit.runners.Suite; 8 | import org.junit.runners.Suite.SuiteClasses; 9 | import org.tweet.meta.service.TweetMetaLocalServiceIntegrationTest; 10 | import org.tweet.twitter.service.TweetMentionServiceIntegrationTest; 11 | import org.tweet.twitter.service.TweetServiceIntegrationTest; 12 | 13 | @RunWith(Suite.class) 14 | @SuiteClasses({// @formatter:off 15 | PersistenceIntegrationTestSuite.class, 16 | PropertiesExistIntegrationTestSuite.class, 17 | 18 | // classif 19 | ClassificationServiceIntegrationTest.class, 20 | SpecificClassificationDataUtilIntegrationTest.class, 21 | 22 | // tweet meta 23 | TweetMetaLocalServiceIntegrationTest.class, 24 | 25 | // tweet 26 | TweetServiceIntegrationTest.class, 27 | TweetMentionServiceIntegrationTest.class, 28 | 29 | // stack 30 | 31 | // common 32 | LinkServiceIntegrationTest.class 33 | }) // @formatter:on 34 | // SetupBackupIntegrationTest - not to be included 35 | public final class IntegrationTestSuite { 36 | // 37 | } 38 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/test/LiveReadOnlyTestSuite.java: -------------------------------------------------------------------------------- 1 | package org.tweet.test; 2 | 3 | import org.classification.service.ClassificationServiceLiveTest; 4 | import org.common.service.ContentExtractorLiveServiceLiveTest; 5 | import org.common.service.live.HttpLiveServiceLiveTest; 6 | import org.common.service.live.LinkLiveServiceLiveTest; 7 | import org.gplus.service.GplusExtractorServiceLiveTest; 8 | import org.gplus.stackexchange.GPlusLiveTest; 9 | import org.junit.runner.RunWith; 10 | import org.junit.runners.Suite; 11 | import org.junit.runners.Suite.SuiteClasses; 12 | import org.rss.service.RssServiceReadOnlyLiveTest; 13 | import org.stackexchange.TwitterReadLiveServiceReadOnlyLiveTest; 14 | import org.stackexchange.UserLiveServiceReadOnlyLiveTest; 15 | import org.tweet.meta.service.InteractionLiveServiceLiveTest; 16 | import org.tweet.twitter.service.TweetServiceReadOnlyLiveTest; 17 | import org.tweet.twitter.service.TwitterTemplateCreatorLiveTest; 18 | 19 | @RunWith(Suite.class) 20 | @SuiteClasses({// @formatter:off 21 | 22 | // common 23 | ContentExtractorLiveServiceLiveTest.class, 24 | HttpLiveServiceLiveTest.class, 25 | LinkLiveServiceLiveTest.class, 26 | 27 | // twitter 28 | TwitterTemplateCreatorLiveTest.class, 29 | TweetServiceReadOnlyLiveTest.class, 30 | 31 | // twitter meta 32 | InteractionLiveServiceLiveTest.class, 33 | 34 | // gplus 35 | GplusExtractorServiceLiveTest.class, 36 | GPlusLiveTest.class, 37 | 38 | // classif 39 | ClassificationServiceLiveTest.class, 40 | 41 | // rss 42 | RssServiceReadOnlyLiveTest.class, 43 | 44 | // stack 45 | TwitterReadLiveServiceReadOnlyLiveTest.class, 46 | UserLiveServiceReadOnlyLiveTest.class 47 | }) // @formatter:on 48 | // QuestionsApiLiveTest.class - not in the jar 49 | public final class LiveReadOnlyTestSuite { 50 | // 51 | } 52 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/test/ManualAcceptUnitTestSuite.java: -------------------------------------------------------------------------------- 1 | package org.tweet.test; 2 | 3 | import org.junit.runner.RunWith; 4 | import org.junit.runners.Suite; 5 | import org.junit.runners.Suite.SuiteClasses; 6 | import org.tweet.twitter.util.commercial.AcceptTweetsByWordDealManualTest; 7 | import org.tweet.twitter.util.commercial.AcceptTweetsByWordDealsManualTest; 8 | import org.tweet.twitter.util.commercial.AcceptTweetsByWordGenericCommercialManualTest; 9 | import org.tweet.twitter.util.commercial.AcceptTweetsByWordWinManualTest; 10 | import org.tweet.twitter.util.generic.AcceptTweetsByWordGenericManualTest; 11 | 12 | @RunWith(Suite.class) 13 | @SuiteClasses({//@formatter:off 14 | 15 | // common 16 | 17 | // twitter 18 | AcceptTweetsByWordWinManualTest.class, 19 | 20 | AcceptTweetsByWordGenericManualTest.class, 21 | AcceptTweetsByWordGenericCommercialManualTest.class, 22 | 23 | AcceptTweetsByWordDealManualTest.class, 24 | AcceptTweetsByWordDealsManualTest.class, 25 | }) //@formatter:off 26 | public final class ManualAcceptUnitTestSuite { // 119 / 1094 27 | // 28 | } 29 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/test/ManualIntegrationTestSuite.java: -------------------------------------------------------------------------------- 1 | package org.tweet.test; 2 | 3 | import org.junit.runner.RunWith; 4 | import org.junit.runners.Suite; 5 | import org.junit.runners.Suite.SuiteClasses; 6 | import org.tweet.meta.service.TweetMetaLocalServiceManualTest; 7 | 8 | @RunWith(Suite.class) 9 | @SuiteClasses({//@formatter:off 10 | 11 | TweetMetaLocalServiceManualTest.class 12 | 13 | }) //@formatter:off 14 | public final class ManualIntegrationTestSuite { 15 | // 16 | } 17 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/test/ManualRejectUnitTestSuite.java: -------------------------------------------------------------------------------- 1 | package org.tweet.test; 2 | 3 | import org.junit.runner.RunWith; 4 | import org.junit.runners.Suite; 5 | import org.junit.runners.Suite.SuiteClasses; 6 | import org.tweet.twitter.util.commercial.RejectTweetsByWordCommercialGenericManualTest; 7 | import org.tweet.twitter.util.commercial.RejectTweetsByWordDealManualTest; 8 | import org.tweet.twitter.util.commercial.RejectTweetsByWordDealsManualTest; 9 | import org.tweet.twitter.util.commercial.RejectTweetsByWordWinManualTest; 10 | import org.tweet.twitter.util.generic.RejectTweetsByWordGenericManualTest; 11 | 12 | @RunWith(Suite.class) 13 | @SuiteClasses({//@formatter:off 14 | 15 | // common 16 | 17 | // twitter 18 | RejectTweetsByWordWinManualTest.class, 19 | 20 | RejectTweetsByWordGenericManualTest.class, 21 | RejectTweetsByWordCommercialGenericManualTest.class, 22 | 23 | RejectTweetsByWordDealManualTest.class, 24 | RejectTweetsByWordDealsManualTest.class, 25 | }) //@formatter:off 26 | public final class ManualRejectUnitTestSuite { // 289 / 2870 27 | // 28 | } 29 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/test/ManualUnitTestSuite.java: -------------------------------------------------------------------------------- 1 | package org.tweet.test; 2 | 3 | import org.junit.runner.RunWith; 4 | import org.junit.runners.Suite; 5 | import org.junit.runners.Suite.SuiteClasses; 6 | import org.tweet.twitter.util.TwitterUtilBannedForAnalysisByRegexManualTest; 7 | import org.tweet.twitter.util.TwitterUtilBannedForTweetingManualTest; 8 | 9 | @RunWith(Suite.class) 10 | @SuiteClasses({//@formatter:off 11 | 12 | // common 13 | 14 | // twitter 15 | ManualRejectUnitTestSuite.class, 16 | ManualAcceptUnitTestSuite.class, 17 | 18 | TwitterUtilBannedForAnalysisByRegexManualTest.class, 19 | TwitterUtilBannedForTweetingManualTest.class 20 | }) //@formatter:off 21 | public final class ManualUnitTestSuite { 22 | // 23 | } 24 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/test/NonTechnicalPropertiesExistIntegrationTestSuite.java: -------------------------------------------------------------------------------- 1 | package org.tweet.test; 2 | 3 | import org.junit.runner.RunWith; 4 | import org.junit.runners.Suite; 5 | import org.junit.runners.Suite.SuiteClasses; 6 | import org.stackexchange.component.MinStackScoreTagRetrieverIntegrationTest; 7 | import org.tweet.twitter.component.MinRtScoreForTwitterTagsExistsIntegrationTest; 8 | import org.tweet.twitter.component.TwitterHashtagWordsIntegrationTest; 9 | import org.tweet.twitter.service.TagRetrieverServiceIntegrationTest; 10 | 11 | @RunWith(Suite.class) 12 | @SuiteClasses({// @formatter:off 13 | TwitterHashtagWordsIntegrationTest.class, 14 | MinStackScoreTagRetrieverIntegrationTest.class, 15 | MinRtScoreForTwitterTagsExistsIntegrationTest.class, 16 | TagRetrieverServiceIntegrationTest.class 17 | }) // @formatter:on 18 | // SetupBackupIntegrationTest - not to be included 19 | public final class NonTechnicalPropertiesExistIntegrationTestSuite { 20 | // 21 | } 22 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/test/PersistenceIntegrationTestSuite.java: -------------------------------------------------------------------------------- 1 | package org.tweet.test; 2 | 3 | import org.junit.runner.RunWith; 4 | import org.junit.runners.Suite; 5 | import org.junit.runners.Suite.SuiteClasses; 6 | import org.keyval.persistence.service.KeyValPersistenceIntegrationTest; 7 | import org.rss.service.RssEntryPersistenceIntegrationTest; 8 | import org.stackexchange.persistence.service.QuestionTweetPersistenceIntegrationTest; 9 | import org.tweet.meta.persistence.service.RetweetPersistenceIntegrationTest; 10 | 11 | @RunWith(Suite.class) 12 | @SuiteClasses({// @formatter:off 13 | QuestionTweetPersistenceIntegrationTest.class, 14 | RssEntryPersistenceIntegrationTest.class, 15 | RetweetPersistenceIntegrationTest.class, 16 | KeyValPersistenceIntegrationTest.class 17 | }) // @formatter:on 18 | // SetupBackupIntegrationTest - not to be included 19 | public final class PersistenceIntegrationTestSuite { 20 | // 21 | } 22 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/test/PropertiesExistIntegrationTestSuite.java: -------------------------------------------------------------------------------- 1 | package org.tweet.test; 2 | 3 | import org.junit.runner.RunWith; 4 | import org.junit.runners.Suite; 5 | import org.junit.runners.Suite.SuiteClasses; 6 | import org.stackexchange.component.MinStackScoreTagRetrieverIntegrationTest; 7 | import org.stackexchange.component.StackTagsExistsIntegrationTest; 8 | import org.tweet.twitter.component.TwitterHashtagWordsIntegrationTest; 9 | import org.tweet.twitter.component.MinRtScoreForTwitterTagsExistsIntegrationTest; 10 | import org.tweet.twitter.service.TagRetrieverServiceIntegrationTest; 11 | 12 | @RunWith(Suite.class) 13 | @SuiteClasses({// @formatter:off 14 | TwitterHashtagWordsIntegrationTest.class, 15 | MinStackScoreTagRetrieverIntegrationTest.class, 16 | MinRtScoreForTwitterTagsExistsIntegrationTest.class, 17 | StackTagsExistsIntegrationTest.class, 18 | TagRetrieverServiceIntegrationTest.class 19 | }) // @formatter:on 20 | // SetupBackupIntegrationTest - not to be included 21 | public final class PropertiesExistIntegrationTestSuite { 22 | // 23 | } 24 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/test/TweetFixture.java: -------------------------------------------------------------------------------- 1 | package org.tweet.test; 2 | 3 | import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; 4 | 5 | import java.util.Date; 6 | 7 | import org.springframework.social.twitter.api.Tweet; 8 | import org.springframework.social.twitter.api.TwitterProfile; 9 | import org.springframework.test.util.ReflectionTestUtils; 10 | 11 | public final class TweetFixture { 12 | 13 | private TweetFixture() { 14 | throw new AssertionError(); 15 | } 16 | 17 | // API 18 | 19 | public static Tweet createTweet(final int retweetCount) { 20 | return createTweet(retweetCount, "en", "en"); 21 | } 22 | 23 | public static Tweet createTweet(final int retweetCount, final String userLanguage, final String tweetLanguage) { 24 | final Tweet tweet = new Tweet(0l, randomAlphabetic(6), new Date(), null, null, null, 0l, tweetLanguage, null); 25 | final TwitterProfile user = createTwitterProfile(); 26 | ReflectionTestUtils.setField(user, "language", userLanguage); 27 | ReflectionTestUtils.setField(user, "screenName", randomAlphabetic(6)); 28 | tweet.setUser(user); 29 | tweet.setFromUser(user.getName()); 30 | tweet.setRetweetCount(retweetCount); 31 | return tweet; 32 | } 33 | 34 | public static TwitterProfile createTwitterProfile() { 35 | final TwitterProfile twitterProfile = new TwitterProfile(0l, randomAlphabetic(6), randomAlphabetic(6), null, null, null, null, new Date()); 36 | ReflectionTestUtils.setField(twitterProfile, "followersCount", 500); 37 | return twitterProfile; 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/test/UnitTestSuite.java: -------------------------------------------------------------------------------- 1 | package org.tweet.test; 2 | 3 | import org.classification.service.ClassificationUnitTest; 4 | import org.common.text.LinkUtilUnitTest; 5 | import org.common.text.TextUtilsUnitTest; 6 | import org.junit.runner.RunWith; 7 | import org.junit.runners.Suite; 8 | import org.junit.runners.Suite.SuiteClasses; 9 | import org.stackexchange.component.StackExchangePageStrategyUnitTest; 10 | import org.tweet.meta.service.InteractionLiveServiceMockUnitTest; 11 | import org.tweet.meta.service.InteractionLiveServiceUnitTest; 12 | import org.tweet.meta.service.TweetContainsWordPredicateUnitTest; 13 | import org.tweet.twitter.service.TweetMentionServiceUnitTest; 14 | import org.tweet.twitter.service.TweetServiceUnitTest; 15 | import org.tweet.twitter.util.HashtagUnitTest; 16 | import org.tweet.twitter.util.TwitterUtilUnitTest; 17 | 18 | @RunWith(Suite.class) 19 | @SuiteClasses({//@formatter:off 20 | 21 | // common 22 | TextUtilsUnitTest.class, 23 | LinkUtilUnitTest.class, 24 | 25 | // twitter 26 | HashtagUnitTest.class, 27 | TwitterUtilUnitTest.class, 28 | TweetServiceUnitTest.class, 29 | TweetMentionServiceUnitTest.class, 30 | 31 | // stack 32 | StackExchangePageStrategyUnitTest.class, 33 | 34 | // classif 35 | ClassificationUnitTest.class, 36 | 37 | // meta 38 | InteractionLiveServiceMockUnitTest.class, 39 | InteractionLiveServiceUnitTest.class, 40 | TweetContainsWordPredicateUnitTest.class 41 | 42 | }) //@formatter:off 43 | public final class UnitTestSuite { 44 | // 45 | } 46 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/twitter/component/MinRtScoreForTwitterTagsExistsIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.component; 2 | 3 | import static org.junit.Assert.assertNotNull; 4 | 5 | import org.common.spring.CommonServiceConfig; 6 | import org.junit.Test; 7 | import org.junit.runner.RunWith; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.test.context.ContextConfiguration; 10 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 11 | import org.stackexchange.util.TwitterTag; 12 | import org.tweet.spring.TwitterConfig; 13 | 14 | @RunWith(SpringJUnit4ClassRunner.class) 15 | @ContextConfiguration(classes = { CommonServiceConfig.class, TwitterConfig.class }) 16 | // @ActiveProfiles(SpringProfileUtil.LIVE) 17 | public class MinRtScoreForTwitterTagsExistsIntegrationTest { 18 | 19 | @Autowired 20 | private MinRtRetriever minRtRetriever; 21 | 22 | // API 23 | 24 | @Test 25 | public final void whenRetrievingMinRtScoresForTag_thenFound() { 26 | for (final TwitterTag tag : TwitterTag.values()) { 27 | assertNotNull("No minrt for tag " + tag, minRtRetriever.minRtRaw(tag.name())); 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/twitter/component/TwitterHashtagWordsIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.component; 2 | 3 | import static org.junit.Assert.assertNotNull; 4 | 5 | import org.common.spring.CommonServiceConfig; 6 | import org.junit.Test; 7 | import org.junit.runner.RunWith; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.test.context.ContextConfiguration; 10 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 11 | import org.stackexchange.util.TwitterAccountEnum; 12 | import org.tweet.spring.TwitterConfig; 13 | 14 | @RunWith(SpringJUnit4ClassRunner.class) 15 | @ContextConfiguration(classes = { CommonServiceConfig.class, TwitterConfig.class }) 16 | // @ActiveProfiles(SpringProfileUtil.LIVE) 17 | public class TwitterHashtagWordsIntegrationTest { 18 | 19 | @Autowired 20 | private TwitterHashtagsRetriever twitterHashtagsRetriever; 21 | 22 | // API 23 | 24 | @Test 25 | public final void whenRetrievingWordsToHashForAccount_thenFound() { 26 | for (final TwitterAccountEnum twitterAccount : TwitterAccountEnum.values()) { 27 | assertNotNull("No words to hash for twitterAccount " + twitterAccount, twitterHashtagsRetriever.hashtagsRaw(twitterAccount.name())); 28 | } 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/twitter/service/ErrorPollerUnitTest.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.service; 2 | 3 | import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; 4 | 5 | import org.junit.Before; 6 | import org.junit.Test; 7 | import org.tweet.twitter.util.ErrorUtil; 8 | 9 | public final class ErrorPollerUnitTest { 10 | 11 | private ErrorPoller instance; 12 | 13 | @Before 14 | public final void before() { 15 | instance = new ErrorPoller(); 16 | } 17 | 18 | // tests 19 | 20 | @Test 21 | public final void when_thenNoExceptions() { 22 | ErrorUtil.registerError(ErrorUtil.bannedRegExesMaybeErrors, randomAlphabetic(5) + "win" + randomAlphabetic(5), randomAlphabetic(20)); 23 | ErrorUtil.registerError(ErrorUtil.bannedRegExesMaybeErrors, randomAlphabetic(5) + "only" + randomAlphabetic(5), randomAlphabetic(20)); 24 | ErrorUtil.registerError(ErrorUtil.bannedRegExesMaybeErrors, randomAlphabetic(5) + "win" + randomAlphabetic(5), randomAlphabetic(20)); 25 | ErrorUtil.registerError(ErrorUtil.bannedRegExesMaybeErrors, randomAlphabetic(5) + "deal" + randomAlphabetic(5), randomAlphabetic(20)); 26 | 27 | instance.logBannedRegExesMaybeErrors(); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/twitter/service/TagRetrieverServiceIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.service; 2 | 3 | import static org.junit.Assert.assertNotNull; 4 | 5 | import java.util.List; 6 | 7 | import org.common.spring.CommonServiceConfig; 8 | import org.junit.Test; 9 | import org.junit.runner.RunWith; 10 | import org.springframework.beans.factory.annotation.Autowired; 11 | import org.springframework.test.context.ContextConfiguration; 12 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 13 | import org.stackexchange.util.TwitterAccountEnum; 14 | import org.stackexchange.util.TwitterTag; 15 | import org.tweet.spring.TwitterConfig; 16 | 17 | @RunWith(SpringJUnit4ClassRunner.class) 18 | @ContextConfiguration(classes = { CommonServiceConfig.class, TwitterConfig.class }) 19 | public class TagRetrieverServiceIntegrationTest { 20 | 21 | @Autowired 22 | private TagRetrieverService tagRetrieverService; 23 | 24 | // API 25 | 26 | @Test 27 | public final void whenAllTwitterTagsAreRetrieved_thenTwitterTagsExistForAllAccounts() { 28 | for (final TwitterAccountEnum twitterAccount : TwitterAccountEnum.values()) { 29 | assertNotNull(tagRetrieverService.twitterTagsRaw(twitterAccount.name())); 30 | } 31 | } 32 | 33 | @Test 34 | public final void givenAllTwitterTagsAreRetrieved_whenCheckingIfEachIsDefinedAsTag_thenYes() { 35 | for (final TwitterAccountEnum twitterAccount : TwitterAccountEnum.values()) { 36 | final List twitterTags = tagRetrieverService.twitterTags(twitterAccount.name()); 37 | for (final String twitterTag : twitterTags) { 38 | if (!twitterTag.isEmpty()) { 39 | assertNotNull(TwitterTag.valueOf(twitterTag)); 40 | } 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/twitter/service/TweetMentionServiceIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.service; 2 | 3 | import static org.hamcrest.Matchers.containsString; 4 | import static org.hamcrest.Matchers.equalTo; 5 | import static org.junit.Assert.assertThat; 6 | 7 | import org.apache.commons.lang3.StringUtils; 8 | import org.common.spring.CommonServiceConfig; 9 | import org.junit.Test; 10 | import org.junit.runner.RunWith; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.test.context.ContextConfiguration; 13 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 14 | import org.tweet.spring.TwitterConfig; 15 | 16 | @RunWith(SpringJUnit4ClassRunner.class) 17 | @ContextConfiguration(classes = { CommonServiceConfig.class, TwitterConfig.class }) 18 | // @ActiveProfiles(SpringProfileUtil.LIVE) 19 | public class TweetMentionServiceIntegrationTest { 20 | 21 | @Autowired 22 | private TweetMentionService instance; 23 | 24 | // tests 25 | 26 | @Test 27 | public final void whenContextIsInitialized_thenNoExceptions() { 28 | // 29 | } 30 | 31 | @Test 32 | public final void whenAddingMentionToTweet1_thenCorrect() { 33 | final String actual = instance.addMention("user1", "some text here in the tweet"); 34 | assertThat(actual, containsString("@user1")); 35 | } 36 | 37 | @Test 38 | public final void whenAddingMentionToTweet2_thenCorrect() { 39 | final String actual = instance.addMention("BethMassi", "Very cool walkthrough on building a #LightSwitch #HTML5 scheduling/calendar app from @ADefWebserver - http://t.co/U5hKuLAfxN"); 40 | assertThat(actual, containsString("@BethMassi")); 41 | } 42 | 43 | @Test 44 | public final void givenMentionAlreadyExists_whenAddingSameMentionToTweet1_thenCorrect() { 45 | final String actual = instance.addMention("ManningBooks", "Hey #graphconnect! Save 45% on #Neo4J in Action and other eBooks with code graphcf at http://manning.com . @manningbooks"); 46 | assertThat(StringUtils.countMatches(actual.toLowerCase(), "manningbooks"), equalTo(1)); 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/twitter/service/TweetMentionServiceUnitTest.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.service; 2 | 3 | import static org.hamcrest.Matchers.hasItem; 4 | import static org.hamcrest.Matchers.hasItems; 5 | import static org.junit.Assert.assertThat; 6 | 7 | import java.util.List; 8 | 9 | import org.junit.Before; 10 | import org.junit.Test; 11 | 12 | public class TweetMentionServiceUnitTest { 13 | 14 | private TweetMentionService instance; 15 | 16 | @Before 17 | public final void before() { 18 | instance = new TweetMentionService(); 19 | } 20 | 21 | // tests 22 | 23 | // extract mentions 24 | 25 | @Test 26 | public final void whenExtractingMentionsFromTweetText1_thenCorrect() { 27 | final List mentions = instance.extractMentions("@fmueller_bln When you are talking about layer, do you mean the XML change sets of Liquibase?"); 28 | assertThat(mentions, hasItem("@fmueller_bln")); 29 | } 30 | 31 | @Test 32 | public final void whenExtractingMentionsFromTweetText2_thenCorrect() { 33 | final List mentions = instance.extractMentions("DON'T MISS: Join @russmiles for a talk on #Clojure #Lisp & #Simplicity on THURSDAY @skillsmatter: http://ow.ly/nCZ5o #london"); 34 | assertThat(mentions, hasItems("@russmiles", "@skillsmatter")); 35 | } 36 | 37 | @Test 38 | public final void whenExtractingMentionsFromTweetText3_thenCorrect() { 39 | final List mentions = instance.extractMentions("@AmitSin94906455 I participated to this course: https://www.coursera.org/course/progfun The http://scala-lang.org has great docs: http://www.scala-lang.org/documentation/"); 40 | assertThat(mentions, hasItems("@AmitSin94906455")); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/twitter/service/TwitterTemplateCreatorLiveTest.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.service; 2 | 3 | import static org.apache.commons.lang3.RandomStringUtils.randomAlphabetic; 4 | 5 | import org.common.spring.CommonServiceConfig; 6 | import org.junit.Test; 7 | import org.junit.runner.RunWith; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.test.context.ContextConfiguration; 10 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 11 | import org.stackexchange.util.TwitterAccountEnum; 12 | import org.tweet.spring.TwitterConfig; 13 | import org.tweet.spring.TwitterLiveConfig; 14 | 15 | @RunWith(SpringJUnit4ClassRunner.class) 16 | @ContextConfiguration(classes = { CommonServiceConfig.class, TwitterConfig.class, TwitterLiveConfig.class }) 17 | public class TwitterTemplateCreatorLiveTest { 18 | 19 | @Autowired 20 | private TwitterTemplateCreator twitterTemplateCreator; 21 | 22 | // API 23 | 24 | @Test(expected = RuntimeException.class) 25 | public final void givenInvalidAccount_whenRetrievingTwitterClient_thenException() { 26 | twitterTemplateCreator.createTwitterTemplate(randomAlphabetic(6)); 27 | } 28 | 29 | @Test 30 | public final void givenValidAccount_whenRetrievingTwitterClient_thenNoException() { 31 | for (final TwitterAccountEnum twitterAccount : TwitterAccountEnum.values()) { 32 | twitterTemplateCreator.createTwitterTemplate(twitterAccount.name()); 33 | } 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/twitter/service/live/TwitterAnalysisLiveServiceLiveTest.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.service.live; 2 | 3 | import org.common.spring.CommonServiceConfig; 4 | import org.junit.Ignore; 5 | import org.junit.Test; 6 | import org.junit.runner.RunWith; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.test.context.ActiveProfiles; 9 | import org.springframework.test.context.ContextConfiguration; 10 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 11 | import org.stackexchange.util.TwitterAccountEnum; 12 | import org.tweet.spring.TwitterConfig; 13 | import org.tweet.spring.TwitterLiveConfig; 14 | import org.tweet.spring.util.SpringProfileUtil; 15 | import org.tweet.twitter.service.live.TwitterAnalysisLiveService; 16 | 17 | @RunWith(SpringJUnit4ClassRunner.class) 18 | @ContextConfiguration(classes = { CommonServiceConfig.class, TwitterConfig.class, TwitterLiveConfig.class }) 19 | @ActiveProfiles(SpringProfileUtil.LIVE) 20 | @Ignore("only dev - to many requests against the twitter APIs") 21 | public class TwitterAnalysisLiveServiceLiveTest { 22 | 23 | @Autowired 24 | private TwitterAnalysisLiveService twitterAnalysisLiveService; 25 | 26 | // API 27 | 28 | @Test 29 | public final void whenAnalyzingAccount_thenNoExceptions() { 30 | twitterAnalysisLiveService.calculateLiveStatisticsForAccount(TwitterAccountEnum.jQueryDaily.name()); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/twitter/util/HashtagUnitTest.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.util; 2 | 3 | import static org.hamcrest.Matchers.equalTo; 4 | import static org.junit.Assert.assertThat; 5 | 6 | import org.junit.Test; 7 | 8 | import com.google.common.collect.Lists; 9 | 10 | public final class HashtagUnitTest { 11 | 12 | // tests 13 | 14 | // isTweetTextValid 15 | 16 | @Test 17 | public final void givenWordShouldBeHashtag_whenTransformingToHashtag_thenTransformed() { 18 | final String twitterTag = new HashtagWordFunction(Lists.newArrayList("java")).apply("Java?"); 19 | assertThat(twitterTag, equalTo("#Java?")); 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/java/org/tweet/twitter/util/TwitterUtilBannedForCommercialAnalysisIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.util; 2 | 3 | import static org.junit.Assert.assertFalse; 4 | import static org.junit.Assert.assertTrue; 5 | 6 | import org.junit.Ignore; 7 | import org.junit.Test; 8 | 9 | public final class TwitterUtilBannedForCommercialAnalysisIntegrationTest { 10 | 11 | // tests - should not be banned 12 | 13 | @Test 14 | public final void givenTweetIsOK_whenCheckingIfIsBannedForCommercialAnalysis1_thenNo() { 15 | final String tweet = "I heard 2 vets talking about Star Trek. I asked how they became Trekkies. Answer: \"Only final natural after serving final on the USS Enterprise.\" #win"; 16 | assertFalse(TwitterUtil.isTweetBannedForCommercialAnalysis(tweet)); 17 | } 18 | 19 | // tests - should be banned 20 | 21 | @Test 22 | @Ignore 23 | public final void givenTweetIsNotOK_whenCheckingIfIsBannedForCommercialAnalysis1_thenYes() { 24 | final String tweet = ""; 25 | assertTrue(TwitterUtil.isTweetBannedForCommercialAnalysis(tweet)); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/resources/classification/results/pass01_good.log: -------------------------------------------------------------------------------- 1 | 2013-07-20 01:01:17,050 WARN [main] o.c.s.ClassificationAccuracyServiceLiveTest [ClassificationAccuracyServiceLiveTest.java:48] For features= 5000 and probes= 3 result is= 0.8985925925925926 2 | 2013-07-20 01:09:33,411 WARN [main] o.c.s.ClassificationAccuracyServiceLiveTest [ClassificationAccuracyServiceLiveTest.java:48] For features= 5000 and probes= 4 result is= 0.8978703703703703 3 | 4 | 2013-07-20 07:29:23,506 WARN [main] o.c.s.ClassificationAccuracyServiceLiveTest [ClassificationAccuracyServiceLiveTest.java:48] For features= 7000 and probes= 10 result is= 0.8894814814814814 5 | 2013-07-20 10:34:51,990 WARN [main] o.c.s.ClassificationAccuracyServiceLiveTest [ClassificationAccuracyServiceLiveTest.java:48] For features= 9000 and probes= 1 result is= 0.8893148148148148 6 | 2013-07-20 12:16:49,028 WARN [main] o.c.s.ClassificationAccuracyServiceLiveTest [ClassificationAccuracyServiceLiveTest.java:48] For features= 9000 and probes= 7 result is= 0.8856296296296297 7 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | 10 | /var/log/stackexchange/stack.log 11 | 12 | %date %level [%thread] %logger{10} [%file:%line] %msg%n 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/resources/setup-test.properties: -------------------------------------------------------------------------------- 1 | mode.maintainance.se=false 2 | mode.maintainance.rt=false 3 | mode.maintainance.rss=false 4 | setup.do=false 5 | 6 | setup.upgrade.retweettext.do=false 7 | setup.upgrade.cleanretweettext.do=false 8 | setup.upgrade.retweets.removeduplicates.do=false 9 | setup.upgrade.retweets.local.removeduplicates.do=false 10 | setup.upgrade.remove.orphanedretweets.do=false 11 | 12 | setup.upgrade.retweetmissing.do=false 13 | setup.upgrade.stackquestionsmissing.do=false 14 | setup.upgrade.retweetdates.do=false 15 | -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/resources/tweetdata.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/java-stackexchange2twitter/src/test/resources/tweetdata.properties -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/resources/tweetscores/current_noSE_scores.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/java-stackexchange2twitter/src/test/resources/tweetscores/current_noSE_scores.log -------------------------------------------------------------------------------- /java-stackexchange2twitter/src/test/resources/tweetscores/current_rt_scores.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/java-stackexchange2twitter/src/test/resources/tweetscores/current_rt_scores.log -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.baeldung 5 | stackexchange2twitter 6 | 0.6.0-SNAPSHOT 7 | pom 8 | stackexchange2twitter 9 | https://github.com/eugenp/stackexchange2twitter 10 | Twitter bot posting Top Questions from StackExchange API 11 | 12 | 13 | java-stackexchange 14 | java-stackexchange2twitter 15 | tweet-value 16 | 17 | 18 | 19 | https://github.com/eugenp/stackexchange2twitter/issues 20 | GitHub Issues 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /tweet-value/.gitignore: -------------------------------------------------------------------------------- 1 | */bin/* 2 | 3 | *.class 4 | 5 | # Package Files # 6 | *.war 7 | *.ear 8 | 9 | 10 | # Eclipse 11 | .settings/ 12 | *.project 13 | *.classpath 14 | .prefs 15 | *.prefs 16 | .metadata/ 17 | 18 | # Intellij 19 | .idea/ 20 | *.iml 21 | *.iws 22 | 23 | # Mac 24 | .DS_Store 25 | 26 | # Maven 27 | log/ 28 | target/ -------------------------------------------------------------------------------- /tweet-value/src/main/java/org/classification/util/TweetSettings.java: -------------------------------------------------------------------------------- 1 | package org.classification.util; 2 | 3 | public final class TweetSettings { 4 | 5 | /** 6 | * can be included as well 7 | * - TODO: potential: 8 | * `/` with = 0.9xx; without = 0.9xx 9 | * `+` with = ~0.9x; without = 0.907 - data shows it should be kept out - counterexample: C++ 10 | * `_` with = ~0.908; without = 0.900 - data shows that it should be in 11 | */ 12 | public static final String TWEET_TOKENIZER = " ,.!?\":()|-&/_"; 13 | 14 | // @ is an important signal - do not add to the tokenizer 15 | 16 | private TweetSettings() { 17 | throw new AssertionError(); 18 | } 19 | 20 | // util 21 | 22 | } 23 | -------------------------------------------------------------------------------- /tweet-value/src/main/java/org/tweet/twitter/evaluator/ChainingEvaluator.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.evaluator; 2 | 3 | import java.util.List; 4 | 5 | import com.google.common.base.Preconditions; 6 | 7 | public class ChainingEvaluator implements IEvaluator { 8 | 9 | private final IEvaluator[] evaluators; 10 | 11 | public ChainingEvaluator(final IEvaluator... evaluators) { 12 | this.evaluators = Preconditions.checkNotNull(evaluators); 13 | } 14 | 15 | // API 16 | 17 | @Override 18 | public final boolean isRejectedByBannedRegexExpressions(final String tweet) { 19 | for (final IEvaluator evaluator : evaluators) { 20 | if (evaluator.isRejectedByBannedRegexExpressions(tweet)) { 21 | return true; 22 | } 23 | } 24 | 25 | return false; 26 | } 27 | 28 | @Override 29 | public final boolean isRejectedByContainsKeywordMaybe(final List tweetTokens, final String tweet) { 30 | for (final IEvaluator evaluator : evaluators) { 31 | if (evaluator.isRejectedByContainsKeywordMaybe(tweetTokens, tweet)) { 32 | return true; 33 | } 34 | } 35 | 36 | return false; 37 | } 38 | 39 | @Override 40 | public final boolean isTweetBanned(final String originalTweet) { 41 | for (final IEvaluator evaluator : evaluators) { 42 | if (evaluator.isTweetBanned(originalTweet)) { 43 | return true; 44 | } 45 | } 46 | 47 | return false; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /tweet-value/src/main/java/org/tweet/twitter/evaluator/IEvaluator.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.evaluator; 2 | 3 | import java.util.List; 4 | 5 | public interface IEvaluator { 6 | 7 | boolean isRejectedByBannedRegexExpressions(final String tweet); 8 | 9 | boolean isRejectedByContainsKeywordMaybe(final List tweetTokens, final String tweet); 10 | 11 | boolean isTweetBanned(final String originalTweet); 12 | 13 | } 14 | -------------------------------------------------------------------------------- /tweet-value/src/main/java/org/tweet/twitter/evaluator/impl/ForAnalysisEvaluator.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.evaluator.impl; 2 | 3 | import java.util.List; 4 | 5 | import org.tweet.twitter.evaluator.AbstractEvaluator; 6 | import org.tweet.twitter.util.ErrorUtil; 7 | import org.tweet.twitter.util.TwitterUtil.ForAnalysis; 8 | 9 | public class ForAnalysisEvaluator extends AbstractEvaluator { 10 | 11 | public ForAnalysisEvaluator() { 12 | super(); 13 | } 14 | 15 | // API 16 | 17 | @Override 18 | public final boolean isRejectedByBannedRegexExpressions(final String tweet) { 19 | configureForRegexExpression(ForAnalysis.acceptedRegExes, ForAnalysis.bannedRegExesMaybe, ForAnalysis.bannedRegExes, "analysis-generic", ErrorUtil.bannedRegExesMaybeErrorsAnalysis); 20 | return super.isRejectedByBannedRegexExpressions(tweet); 21 | } 22 | 23 | @Override 24 | public final boolean isRejectedByContainsKeywordMaybe(final List tweetTokens, final String tweet) { 25 | configureForContainsKeyword(ForAnalysis.bannedContainsKeywordsMaybe, ForAnalysis.acceptedContainsKeywordsOverrides, ErrorUtil.bannedContainsMaybeErrorsForAnalysis); 26 | return super.isRejectedByContainsKeywordMaybe(tweetTokens, tweet); 27 | } 28 | 29 | @Override 30 | public final boolean isTweetBanned(final String originalTweet) { 31 | configure(ForAnalysis.bannedExpressionsMaybe, ForAnalysis.bannedExpressions, ForAnalysis.bannedContainsKeywords, ForAnalysis.bannedStartsWithExprs); 32 | return super.isTweetBanned(originalTweet); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /tweet-value/src/main/java/org/tweet/twitter/evaluator/impl/ForCommercialAnalysisEvaluator.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.evaluator.impl; 2 | 3 | import java.util.List; 4 | 5 | import org.tweet.twitter.evaluator.AbstractEvaluator; 6 | import org.tweet.twitter.util.ErrorUtil; 7 | import org.tweet.twitter.util.TwitterUtil.ForAnalysis; 8 | 9 | public class ForCommercialAnalysisEvaluator extends AbstractEvaluator { 10 | 11 | public ForCommercialAnalysisEvaluator() { 12 | super(); 13 | } 14 | 15 | // API 16 | 17 | @Override 18 | public final boolean isRejectedByBannedRegexExpressions(final String tweet) { 19 | configureForRegexExpression(ForAnalysis.Commercial.acceptedRegExes, ForAnalysis.Commercial.bannedRegExesMaybe, ForAnalysis.Commercial.bannedRegExes, "analysis-commercial", ErrorUtil.bannedRegExesMaybeErrorsAnalysisCommercial); 20 | return super.isRejectedByBannedRegexExpressions(tweet); 21 | } 22 | 23 | @Override 24 | public final boolean isRejectedByContainsKeywordMaybe(final List tweetTokens, final String tweet) { 25 | configureForContainsKeyword(ForAnalysis.Commercial.bannedContainsKeywordsMaybe, ForAnalysis.Commercial.acceptedContainsKeywordsOverrides, ErrorUtil.bannedContainsMaybeErrorsForAnalysisCommercial); 26 | return super.isRejectedByContainsKeywordMaybe(tweetTokens, tweet); 27 | } 28 | 29 | @Override 30 | public final boolean isTweetBanned(final String originalTweet) { 31 | configure(ForAnalysis.Commercial.bannedExpressionsMaybe, ForAnalysis.Commercial.bannedExpressions, ForAnalysis.Commercial.bannedContainsKeywords, ForAnalysis.Commercial.bannedStartsWithExprs); 32 | return super.isTweetBanned(originalTweet); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /tweet-value/src/main/java/org/tweet/twitter/evaluator/impl/ForNonTechnicalEvaluator.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.evaluator.impl; 2 | 3 | import java.util.List; 4 | 5 | import org.tweet.twitter.evaluator.AbstractEvaluator; 6 | import org.tweet.twitter.util.ErrorUtil; 7 | import org.tweet.twitter.util.TwitterUtil.ForTweeting.NonTechnicalOnly; 8 | 9 | public class ForNonTechnicalEvaluator extends AbstractEvaluator { 10 | 11 | public ForNonTechnicalEvaluator() { 12 | super(); 13 | } 14 | 15 | // API 16 | 17 | @Override 18 | public final boolean isRejectedByBannedRegexExpressions(final String tweet) { 19 | configureForRegexExpression(NonTechnicalOnly.acceptedRegExes, NonTechnicalOnly.bannedRegExesMaybe, NonTechnicalOnly.bannedRegExes, "analysis-non-tech", ErrorUtil.bannedRegExesMaybeErrorsNonTech); 20 | return super.isRejectedByBannedRegexExpressions(tweet); 21 | } 22 | 23 | @Override 24 | public final boolean isRejectedByContainsKeywordMaybe(final List tweetTokens, final String tweet) { 25 | configureForContainsKeyword(NonTechnicalOnly.bannedContainsKeywordsMaybe, NonTechnicalOnly.acceptedContainsKeywordsOverrides, ErrorUtil.bannedContainsMaybeErrorsForNonTech); 26 | return super.isRejectedByContainsKeywordMaybe(tweetTokens, tweet); 27 | } 28 | 29 | @Override 30 | public final boolean isTweetBanned(final String originalTweet) { 31 | configure(NonTechnicalOnly.bannedExpressionsMaybe, NonTechnicalOnly.bannedExpressions, NonTechnicalOnly.bannedContainsKeywords, NonTechnicalOnly.bannedStartsWithExprs); 32 | return super.isTweetBanned(originalTweet); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /tweet-value/src/main/java/org/tweet/twitter/evaluator/impl/ForTweetingEvaluator.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.evaluator.impl; 2 | 3 | import java.util.List; 4 | 5 | import org.tweet.twitter.evaluator.AbstractEvaluator; 6 | import org.tweet.twitter.util.ErrorUtil; 7 | import org.tweet.twitter.util.TwitterUtil.ForTweeting; 8 | 9 | public class ForTweetingEvaluator extends AbstractEvaluator { 10 | 11 | public ForTweetingEvaluator() { 12 | super(); 13 | } 14 | 15 | // API 16 | 17 | @Override 18 | public final boolean isRejectedByBannedRegexExpressions(final String tweet) { 19 | configureForRegexExpression(ForTweeting.acceptedRegExes, ForTweeting.bannedRegExesMaybe, ForTweeting.bannedRegExes, "tweeting-generic", ErrorUtil.bannedRegExesMaybeErrorsTweeting); 20 | return super.isRejectedByBannedRegexExpressions(tweet); 21 | } 22 | 23 | @Override 24 | public final boolean isRejectedByContainsKeywordMaybe(final List tweetTokens, final String tweet) { 25 | configureForContainsKeyword(ForTweeting.bannedContainsKeywordsMaybe, ForTweeting.acceptedContainsKeywordsOverrides, ErrorUtil.bannedContainsMaybeErrorsForTweeting); 26 | return super.isRejectedByContainsKeywordMaybe(tweetTokens, tweet); 27 | } 28 | 29 | @Override 30 | public final boolean isTweetBanned(final String originalTweet) { 31 | configure(ForTweeting.bannedExpressionsMaybe, ForTweeting.bannedExpressions, ForTweeting.bannedContainsKeywords, ForTweeting.bannedStartsWithExprs); 32 | return super.isTweetBanned(originalTweet); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /tweet-value/src/main/java/org/tweet/twitter/service/TweetType.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.service; 2 | 3 | public enum TweetType { 4 | 5 | Standard, NonTech 6 | 7 | } 8 | -------------------------------------------------------------------------------- /tweet-value/src/main/java/org/tweet/twitter/util/ErrorUtil.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.util; 2 | 3 | import java.util.Map; 4 | import java.util.Set; 5 | 6 | import com.google.common.base.Preconditions; 7 | import com.google.common.collect.Maps; 8 | import com.google.common.collect.Sets; 9 | 10 | public final class ErrorUtil { 11 | 12 | public static final Map> bannedRegExesMaybeErrors = Maps.newConcurrentMap(); 13 | 14 | public static final Map> bannedRegExesMaybeErrorsTweeting = Maps.newConcurrentMap(); 15 | public static final Map> bannedRegExesMaybeErrorsAnalysis = Maps.newConcurrentMap(); 16 | public static final Map> bannedRegExesMaybeErrorsAnalysisCommercial = Maps.newConcurrentMap(); 17 | public static final Map> bannedRegExesMaybeErrorsNonTech = Maps.newConcurrentMap(); 18 | 19 | public static final Map> bannedContainsMaybeErrorsForAnalysisCommercial = Maps.newConcurrentMap(); 20 | public static final Map> bannedContainsMaybeErrorsForAnalysis = Maps.newConcurrentMap(); 21 | public static final Map> bannedContainsMaybeErrorsForTweeting = Maps.newConcurrentMap(); 22 | public static final Map> bannedContainsMaybeErrorsForNonTech = Maps.newConcurrentMap(); 23 | 24 | public static final Set rejectedByClassifierJob = Sets.newConcurrentHashSet(); 25 | 26 | private ErrorUtil() { 27 | throw new AssertionError(); 28 | } 29 | 30 | // API 31 | 32 | public static final void registerError(final Set collector, final String error) { 33 | Preconditions.checkNotNull(collector); 34 | Preconditions.checkNotNull(error); 35 | 36 | collector.add(error); 37 | } 38 | 39 | public static final void registerError(final Map> collector, final String key, final String error) { 40 | Preconditions.checkNotNull(collector); 41 | Preconditions.checkNotNull(error); 42 | 43 | Set existingListForRegex = collector.get(key); 44 | if (existingListForRegex == null) { 45 | existingListForRegex = Sets.newConcurrentHashSet(); 46 | collector.put(key, existingListForRegex); 47 | } 48 | existingListForRegex.add(error); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /tweet-value/src/main/java/org/tweet/twitter/util/HashtagWordFunction.java: -------------------------------------------------------------------------------- 1 | package org.tweet.twitter.util; 2 | 3 | import java.util.List; 4 | 5 | import com.google.common.base.Function; 6 | 7 | public class HashtagWordFunction implements Function { 8 | private final List wordsToHash; 9 | private int transformationsDone = 0; 10 | 11 | public HashtagWordFunction(final List wordsToHash) { 12 | this.wordsToHash = wordsToHash; 13 | } 14 | 15 | // API 16 | 17 | @Override 18 | public final String apply(final String input) { 19 | return wordToHashtag(input); 20 | } 21 | 22 | public int getTransformationsDone() { 23 | return transformationsDone; 24 | } 25 | 26 | // util 27 | 28 | private final String wordToHashtag(final String originalCandidate) { 29 | String candidate = originalCandidate; 30 | if (candidate.endsWith("?") || candidate.endsWith("!") || candidate.endsWith(",")) { 31 | candidate = originalCandidate.substring(0, originalCandidate.length() - 1); 32 | } 33 | for (final String wordToHash : wordsToHash) { 34 | if (wordToHash.equals(candidate.toLowerCase())) { 35 | transformationsDone++; 36 | return "#" + originalCandidate; 37 | } 38 | } 39 | return originalCandidate; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /tweet-value/src/main/resources/backup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -x 4 | 5 | DATE=$(date +%Y%m%d) 6 | DATETIME=$(date +%Y%m%d-%H%m) 7 | BACKUP_DIRECTORY=/tmp/ 8 | DB=stackexchange_prod 9 | BACKUP_FILE=$BACKUP_DIRECTORY/${DATETIME}_${DB}.sql 10 | mysqldump -u stackUser -pstackmy5ql stackexchange_prod > $BACKUP_FILE 11 | 12 | S3_BUCKET_URL=s3://twitter_db_backup/$DATE/ 13 | s3cmd put ${BACKUP_FILE} $S3_BUCKET_URL 2>&1 14 | -------------------------------------------------------------------------------- /tweet-value/src/main/resources/classification/jobs/jobs-onthefence.classif: -------------------------------------------------------------------------------- 1 | 2 | - non-commercial (but rejected): 3 | We're giving out FREE #iPAD'S!!!! Get em' while they're hot ---> http://t.co/kfD8pizC3p RT! RT! RT! 4 | 5 | 6 | - commercial: 7 | 8 | 9 | - to sort: 10 | 11 | -------------------------------------------------------------------------------- /tweet-value/src/main/resources/classification/programming/nonprogramming.classif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/tweet-value/src/main/resources/classification/programming/nonprogramming.classif -------------------------------------------------------------------------------- /tweet-value/src/main/resources/classification/programming/programming.classif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/tweet-value/src/main/resources/classification/programming/programming.classif -------------------------------------------------------------------------------- /tweet-value/src/main/resources/classification/sample.classif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/tweet-value/src/main/resources/classification/sample.classif -------------------------------------------------------------------------------- /tweet-value/src/main/resources/classification/test/jobs/note.info: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/tweet-value/src/main/resources/classification/test/jobs/note.info -------------------------------------------------------------------------------- /tweet-value/src/main/resources/classification/test/programming/nonprogramming.classif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/tweet-value/src/main/resources/classification/test/programming/nonprogramming.classif -------------------------------------------------------------------------------- /tweet-value/src/main/resources/classification/test/programming/programming.classif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/tweet-value/src/main/resources/classification/test/programming/programming.classif -------------------------------------------------------------------------------- /tweet-value/src/main/resources/logback.xml.backup: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | 10 | /var/log/stackexchange/stack.log 11 | 12 | %date %level [%thread] %logger{10} [%file:%line] %msg%n 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /tweet-value/src/main/resources/twitter/interactions.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 1. Deserves Freshness 4 | 5 | - call to join event 6 | DON'T MISS: Join @russmiles for a talk on #Clojure #Lisp & #Simplicity on THURSDAY @skillsmatter: http://ow.ly/nCZ5o #london 7 | 8 | - possible signals: `don't miss`, `talk`, `@skillsmatter`, location (#london) 9 | 10 | 11 | 12 | 2. Types of pre/post fixes: 13 | Useful stuff. 14 | Very cool. -------------------------------------------------------------------------------- /tweet-value/src/main/resources/twitter/schedule.txt: -------------------------------------------------------------------------------- 1 | 2 | # The StackExchange API / TweetStackexchangeScheduler 3 | 4 | 10 - 5 | 11 - 6 | 12 - 8 7 | 13 - 8 8 | 14 - 8 9 | 15 - 8 10 | 16 - 7 11 | 17 - 8 12 | 18 - 8 13 | 19 - 8 14 | 20 - 7 15 | 21 - 8 16 | 22 - 17 | 23 - 18 | 19 | 20 | # The Twitter API (reads) / TweetMetaScheduler 21 | 22 | 9 - 7 23 | 10 - 7 24 | 11 - 7 25 | 12 - 7 26 | 13 - 7 27 | 14 - 7 28 | 15 - 7 29 | 16 - 7 30 | 17 - 7 31 | 18 - 7 32 | 19 - 7 33 | 20 - 7 34 | 21 - 7 35 | 22 - 7 36 | 23 - 7 37 | 38 | 39 | # RSS 40 | 41 | 9 - 42 | 10 - 43 | 11 - 44 | 12 - 45 | 13 - 46 | 14 - 47 | 15 - 48 | 16 - 49 | 17 - 50 | 18 - 51 | 19 - 52 | 20 - 53 | 21 - 54 | 22 - 55 | 23 - 56 | -------------------------------------------------------------------------------- /tweet-value/src/main/webapp/META-INF/MANIFEST.MF: -------------------------------------------------------------------------------- 1 | Manifest-Version: 1.0 2 | Class-Path: 3 | 4 | -------------------------------------------------------------------------------- /tweet-value/src/main/webapp/WEB-INF/web.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | stackexchange2twitter 9 | 10 | 11 | 12 | contextClass 13 | org.springframework.web.context.support.AnnotationConfigWebApplicationContext 14 | 15 | 16 | contextConfigLocation 17 | org.common.spring org.classification.spring org.keyval.spring org.tweet.spring org.stackexchange.spring org.tweet.meta.spring org.rss.spring 18 | 19 | 20 | 21 | org.springframework.web.context.ContextLoaderListener 22 | 23 | 24 | contextInitializerClasses 25 | org.common.spring.MyApplicationContextInitializerProv 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /tweet-value/src/test/java/META-INF/MANIFEST.MF: -------------------------------------------------------------------------------- 1 | Manifest-Version: 1.0 2 | Class-Path: 3 | 4 | -------------------------------------------------------------------------------- /tweet-value/src/test/resources/classification/results/pass01_good.log: -------------------------------------------------------------------------------- 1 | 2013-07-20 01:01:17,050 WARN [main] o.c.s.ClassificationAccuracyServiceLiveTest [ClassificationAccuracyServiceLiveTest.java:48] For features= 5000 and probes= 3 result is= 0.8985925925925926 2 | 2013-07-20 01:09:33,411 WARN [main] o.c.s.ClassificationAccuracyServiceLiveTest [ClassificationAccuracyServiceLiveTest.java:48] For features= 5000 and probes= 4 result is= 0.8978703703703703 3 | 4 | 2013-07-20 07:29:23,506 WARN [main] o.c.s.ClassificationAccuracyServiceLiveTest [ClassificationAccuracyServiceLiveTest.java:48] For features= 7000 and probes= 10 result is= 0.8894814814814814 5 | 2013-07-20 10:34:51,990 WARN [main] o.c.s.ClassificationAccuracyServiceLiveTest [ClassificationAccuracyServiceLiveTest.java:48] For features= 9000 and probes= 1 result is= 0.8893148148148148 6 | 2013-07-20 12:16:49,028 WARN [main] o.c.s.ClassificationAccuracyServiceLiveTest [ClassificationAccuracyServiceLiveTest.java:48] For features= 9000 and probes= 7 result is= 0.8856296296296297 7 | -------------------------------------------------------------------------------- /tweet-value/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | 10 | /var/log/stackexchange/stack.log 11 | 12 | %date %level [%thread] %logger{10} [%file:%line] %msg%n 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /tweet-value/src/test/resources/tweetdata.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/tweet-value/src/test/resources/tweetdata.properties -------------------------------------------------------------------------------- /tweet-value/src/test/resources/tweetscores/current_noSE_scores.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/tweet-value/src/test/resources/tweetscores/current_noSE_scores.log -------------------------------------------------------------------------------- /tweet-value/src/test/resources/tweetscores/current_rt_scores.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eugenp/stackexchange2twitter/00c489a6fb10ebda4388c7145c9982ce81155680/tweet-value/src/test/resources/tweetscores/current_rt_scores.log --------------------------------------------------------------------------------