├── .gitignore ├── .swift-version ├── .travis.yml ├── HNScraper.podspec ├── HNScraper.xcodeproj ├── project.pbxproj └── project.xcworkspace │ ├── contents.xcworkspacedata │ └── xcshareddata │ └── IDEWorkspaceChecks.plist ├── HNScraper ├── HNComment.swift ├── HNLogin.swift ├── HNParseConfig.swift ├── HNPost.swift ├── HNScraper.swift ├── HNUser.swift ├── Helpers │ ├── RessourceFetcher.swift │ └── Scanner+ScanBetweenString.swift └── Info.plist ├── HNScraperTests ├── HNLoginTest.swift ├── HNPostTest.swift ├── HNScraperTest.swift ├── Info.plist └── RessourceFetcherTest.swift ├── LICENSE ├── README.md └── hn.json /.gitignore: -------------------------------------------------------------------------------- 1 | ## mac os 2 | .DS_Store 3 | 4 | ## User settings 5 | xcuserdata/ 6 | 7 | ## compatibility with Xcode 8 and earlier (ignoring not required starting Xcode 9) 8 | *.xcscmblueprint 9 | *.xccheckout -------------------------------------------------------------------------------- /.swift-version: -------------------------------------------------------------------------------- 1 | 4.0 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: objective-c 2 | osx_image: xcode9.4 3 | script: 4 | - xcodebuild -project 'HNScraper.xcodeproj' -scheme 'HNScraper' -sdk iphonesimulator build -------------------------------------------------------------------------------- /HNScraper.podspec: -------------------------------------------------------------------------------- 1 | Pod::Spec.new do |s| 2 | s.name = 'HNScraper' 3 | s.version = '0.2.2' 4 | s.summary = 'Scraper for hackernews written in swift' 5 | 6 | s.description = <<-DESC 7 | Scraper for hackernews written in swift. Supports grabbing posts, comments & user data as well as logging in, voting and favouriting items. 8 | DESC 9 | 10 | s.homepage = 'https://github.com/tsucres/HNScraper' 11 | s.license = { :type => 'MIT', :file => 'LICENSE' } 12 | s.author = { 'Stéphane Sercu' => 'stefsercu@gmail.com' } 13 | s.source = { :git => 'https://github.com/tsucres/HNScraper.git', :tag => s.version.to_s } 14 | 15 | s.ios.deployment_target = '9.0' 16 | s.source_files = 'HNScraper/**/*.swift' 17 | 18 | end -------------------------------------------------------------------------------- /HNScraper.xcodeproj/project.pbxproj: -------------------------------------------------------------------------------- 1 | // !$*UTF8*$! 2 | { 3 | archiveVersion = 1; 4 | classes = { 5 | }; 6 | objectVersion = 48; 7 | objects = { 8 | 9 | /* Begin PBXBuildFile section */ 10 | 97DA10621F81466400ADF5D8 /* HNScraper.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 97DA10591F81466300ADF5D8 /* HNScraper.framework */; }; 11 | 97DA10701F81468C00ADF5D8 /* RessourceFetcher.swift in Sources */ = {isa = PBXBuildFile; fileRef = 97D0D8671F78EFCE007DE08A /* RessourceFetcher.swift */; }; 12 | 97DA10711F81468C00ADF5D8 /* Scanner+ScanBetweenString.swift in Sources */ = {isa = PBXBuildFile; fileRef = 97D0D8821F7E440F007DE08A /* Scanner+ScanBetweenString.swift */; }; 13 | 97DA10731F81468C00ADF5D8 /* HNComment.swift in Sources */ = {isa = PBXBuildFile; fileRef = 97D0D85C1F78EF83007DE08A /* HNComment.swift */; }; 14 | 97DA10741F81468C00ADF5D8 /* HNLogin.swift in Sources */ = {isa = PBXBuildFile; fileRef = 97D0D85E1F78EF83007DE08A /* HNLogin.swift */; }; 15 | 97DA10751F81468C00ADF5D8 /* HNParseConfig.swift in Sources */ = {isa = PBXBuildFile; fileRef = 97D0D85D1F78EF83007DE08A /* HNParseConfig.swift */; }; 16 | 97DA10761F81468C00ADF5D8 /* HNPost.swift in Sources */ = {isa = PBXBuildFile; fileRef = 97D0D85F1F78EF83007DE08A /* HNPost.swift */; }; 17 | 97DA10771F81468C00ADF5D8 /* HNScraper.swift in Sources */ = {isa = PBXBuildFile; fileRef = 97D0D8601F78EF83007DE08A /* HNScraper.swift */; }; 18 | 97DA10781F81468C00ADF5D8 /* HNUser.swift in Sources */ = {isa = PBXBuildFile; fileRef = 97D0D85B1F78EF83007DE08A /* HNUser.swift */; }; 19 | 97DA10791F8147C100ADF5D8 /* HNLoginTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 97D0D8691F793FD6007DE08A /* HNLoginTest.swift */; }; 20 | 97DA107A1F8147C100ADF5D8 /* HNScraperTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 97D0D87A1F795291007DE08A /* HNScraperTest.swift */; }; 21 | 97DA107B1F8147C100ADF5D8 /* HNPostTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 97D0D88B1F7E613D007DE08A /* HNPostTest.swift */; }; 22 | 97DA107C1F8147C100ADF5D8 /* RessourceFetcherTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 97D0D8881F7E464D007DE08A /* RessourceFetcherTest.swift */; }; 23 | /* End PBXBuildFile section */ 24 | 25 | /* Begin PBXContainerItemProxy section */ 26 | 97DA10631F81466400ADF5D8 /* PBXContainerItemProxy */ = { 27 | isa = PBXContainerItemProxy; 28 | containerPortal = 97D0D83E1F78EF42007DE08A /* Project object */; 29 | proxyType = 1; 30 | remoteGlobalIDString = 97DA10581F81466300ADF5D8; 31 | remoteInfo = HNScraper; 32 | }; 33 | /* End PBXContainerItemProxy section */ 34 | 35 | /* Begin PBXFileReference section */ 36 | 97D0D8551F78EF42007DE08A /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 37 | 97D0D85B1F78EF83007DE08A /* HNUser.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HNUser.swift; sourceTree = ""; }; 38 | 97D0D85C1F78EF83007DE08A /* HNComment.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HNComment.swift; sourceTree = ""; }; 39 | 97D0D85D1F78EF83007DE08A /* HNParseConfig.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HNParseConfig.swift; sourceTree = ""; }; 40 | 97D0D85E1F78EF83007DE08A /* HNLogin.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HNLogin.swift; sourceTree = ""; }; 41 | 97D0D85F1F78EF83007DE08A /* HNPost.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HNPost.swift; sourceTree = ""; }; 42 | 97D0D8601F78EF83007DE08A /* HNScraper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HNScraper.swift; sourceTree = ""; }; 43 | 97D0D8671F78EFCE007DE08A /* RessourceFetcher.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RessourceFetcher.swift; sourceTree = ""; }; 44 | 97D0D8691F793FD6007DE08A /* HNLoginTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HNLoginTest.swift; sourceTree = ""; }; 45 | 97D0D8731F79400C007DE08A /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 46 | 97D0D87A1F795291007DE08A /* HNScraperTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HNScraperTest.swift; sourceTree = ""; }; 47 | 97D0D8821F7E440F007DE08A /* Scanner+ScanBetweenString.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Scanner+ScanBetweenString.swift"; sourceTree = ""; }; 48 | 97D0D8881F7E464D007DE08A /* RessourceFetcherTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RessourceFetcherTest.swift; sourceTree = ""; }; 49 | 97D0D88B1F7E613D007DE08A /* HNPostTest.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HNPostTest.swift; sourceTree = ""; }; 50 | 97DA10591F81466300ADF5D8 /* HNScraper.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = HNScraper.framework; sourceTree = BUILT_PRODUCTS_DIR; }; 51 | 97DA10611F81466300ADF5D8 /* HNScraperTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = HNScraperTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; 52 | /* End PBXFileReference section */ 53 | 54 | /* Begin PBXFrameworksBuildPhase section */ 55 | 97DA10551F81466300ADF5D8 /* Frameworks */ = { 56 | isa = PBXFrameworksBuildPhase; 57 | buildActionMask = 2147483647; 58 | files = ( 59 | ); 60 | runOnlyForDeploymentPostprocessing = 0; 61 | }; 62 | 97DA105E1F81466300ADF5D8 /* Frameworks */ = { 63 | isa = PBXFrameworksBuildPhase; 64 | buildActionMask = 2147483647; 65 | files = ( 66 | 97DA10621F81466400ADF5D8 /* HNScraper.framework in Frameworks */, 67 | ); 68 | runOnlyForDeploymentPostprocessing = 0; 69 | }; 70 | /* End PBXFrameworksBuildPhase section */ 71 | 72 | /* Begin PBXGroup section */ 73 | 97D0D83D1F78EF42007DE08A = { 74 | isa = PBXGroup; 75 | children = ( 76 | 97D0D8481F78EF42007DE08A /* HNScraper */, 77 | 97D0D8701F79400C007DE08A /* HNScraperTests */, 78 | 97D0D8471F78EF42007DE08A /* Products */, 79 | ); 80 | sourceTree = ""; 81 | }; 82 | 97D0D8471F78EF42007DE08A /* Products */ = { 83 | isa = PBXGroup; 84 | children = ( 85 | 97DA10591F81466300ADF5D8 /* HNScraper.framework */, 86 | 97DA10611F81466300ADF5D8 /* HNScraperTests.xctest */, 87 | ); 88 | name = Products; 89 | sourceTree = ""; 90 | }; 91 | 97D0D8481F78EF42007DE08A /* HNScraper */ = { 92 | isa = PBXGroup; 93 | children = ( 94 | 97D0D88A1F7E5F4D007DE08A /* Helpers */, 95 | 97D0D85C1F78EF83007DE08A /* HNComment.swift */, 96 | 97D0D85E1F78EF83007DE08A /* HNLogin.swift */, 97 | 97D0D85D1F78EF83007DE08A /* HNParseConfig.swift */, 98 | 97D0D85F1F78EF83007DE08A /* HNPost.swift */, 99 | 97D0D8601F78EF83007DE08A /* HNScraper.swift */, 100 | 97D0D85B1F78EF83007DE08A /* HNUser.swift */, 101 | 97D0D8551F78EF42007DE08A /* Info.plist */, 102 | ); 103 | path = HNScraper; 104 | sourceTree = ""; 105 | }; 106 | 97D0D8701F79400C007DE08A /* HNScraperTests */ = { 107 | isa = PBXGroup; 108 | children = ( 109 | 97D0D8691F793FD6007DE08A /* HNLoginTest.swift */, 110 | 97D0D87A1F795291007DE08A /* HNScraperTest.swift */, 111 | 97D0D88B1F7E613D007DE08A /* HNPostTest.swift */, 112 | 97D0D8881F7E464D007DE08A /* RessourceFetcherTest.swift */, 113 | 97D0D8731F79400C007DE08A /* Info.plist */, 114 | ); 115 | path = HNScraperTests; 116 | sourceTree = ""; 117 | }; 118 | 97D0D88A1F7E5F4D007DE08A /* Helpers */ = { 119 | isa = PBXGroup; 120 | children = ( 121 | 97D0D8671F78EFCE007DE08A /* RessourceFetcher.swift */, 122 | 97D0D8821F7E440F007DE08A /* Scanner+ScanBetweenString.swift */, 123 | ); 124 | path = Helpers; 125 | sourceTree = ""; 126 | }; 127 | /* End PBXGroup section */ 128 | 129 | /* Begin PBXHeadersBuildPhase section */ 130 | 97DA10561F81466300ADF5D8 /* Headers */ = { 131 | isa = PBXHeadersBuildPhase; 132 | buildActionMask = 2147483647; 133 | files = ( 134 | ); 135 | runOnlyForDeploymentPostprocessing = 0; 136 | }; 137 | /* End PBXHeadersBuildPhase section */ 138 | 139 | /* Begin PBXNativeTarget section */ 140 | 97DA10581F81466300ADF5D8 /* HNScraper */ = { 141 | isa = PBXNativeTarget; 142 | buildConfigurationList = 97DA106A1F81466400ADF5D8 /* Build configuration list for PBXNativeTarget "HNScraper" */; 143 | buildPhases = ( 144 | 97DA10541F81466300ADF5D8 /* Sources */, 145 | 97DA10551F81466300ADF5D8 /* Frameworks */, 146 | 97DA10561F81466300ADF5D8 /* Headers */, 147 | 97DA10571F81466300ADF5D8 /* Resources */, 148 | ); 149 | buildRules = ( 150 | ); 151 | dependencies = ( 152 | ); 153 | name = HNScraper; 154 | productName = HNScraper; 155 | productReference = 97DA10591F81466300ADF5D8 /* HNScraper.framework */; 156 | productType = "com.apple.product-type.framework"; 157 | }; 158 | 97DA10601F81466300ADF5D8 /* HNScraperTests */ = { 159 | isa = PBXNativeTarget; 160 | buildConfigurationList = 97DA106D1F81466400ADF5D8 /* Build configuration list for PBXNativeTarget "HNScraperTests" */; 161 | buildPhases = ( 162 | 97DA105D1F81466300ADF5D8 /* Sources */, 163 | 97DA105E1F81466300ADF5D8 /* Frameworks */, 164 | 97DA105F1F81466300ADF5D8 /* Resources */, 165 | ); 166 | buildRules = ( 167 | ); 168 | dependencies = ( 169 | 97DA10641F81466400ADF5D8 /* PBXTargetDependency */, 170 | ); 171 | name = HNScraperTests; 172 | productName = HNScraperTests; 173 | productReference = 97DA10611F81466300ADF5D8 /* HNScraperTests.xctest */; 174 | productType = "com.apple.product-type.bundle.unit-test"; 175 | }; 176 | /* End PBXNativeTarget section */ 177 | 178 | /* Begin PBXProject section */ 179 | 97D0D83E1F78EF42007DE08A /* Project object */ = { 180 | isa = PBXProject; 181 | attributes = { 182 | LastSwiftUpdateCheck = 0900; 183 | LastUpgradeCheck = 0900; 184 | ORGANIZATIONNAME = "Stéphane Sercu"; 185 | TargetAttributes = { 186 | 97DA10581F81466300ADF5D8 = { 187 | CreatedOnToolsVersion = 9.0; 188 | ProvisioningStyle = Automatic; 189 | }; 190 | 97DA10601F81466300ADF5D8 = { 191 | CreatedOnToolsVersion = 9.0; 192 | ProvisioningStyle = Automatic; 193 | }; 194 | }; 195 | }; 196 | buildConfigurationList = 97D0D8411F78EF42007DE08A /* Build configuration list for PBXProject "HNScraper" */; 197 | compatibilityVersion = "Xcode 8.0"; 198 | developmentRegion = en; 199 | hasScannedForEncodings = 0; 200 | knownRegions = ( 201 | en, 202 | Base, 203 | ); 204 | mainGroup = 97D0D83D1F78EF42007DE08A; 205 | productRefGroup = 97D0D8471F78EF42007DE08A /* Products */; 206 | projectDirPath = ""; 207 | projectRoot = ""; 208 | targets = ( 209 | 97DA10581F81466300ADF5D8 /* HNScraper */, 210 | 97DA10601F81466300ADF5D8 /* HNScraperTests */, 211 | ); 212 | }; 213 | /* End PBXProject section */ 214 | 215 | /* Begin PBXResourcesBuildPhase section */ 216 | 97DA10571F81466300ADF5D8 /* Resources */ = { 217 | isa = PBXResourcesBuildPhase; 218 | buildActionMask = 2147483647; 219 | files = ( 220 | ); 221 | runOnlyForDeploymentPostprocessing = 0; 222 | }; 223 | 97DA105F1F81466300ADF5D8 /* Resources */ = { 224 | isa = PBXResourcesBuildPhase; 225 | buildActionMask = 2147483647; 226 | files = ( 227 | ); 228 | runOnlyForDeploymentPostprocessing = 0; 229 | }; 230 | /* End PBXResourcesBuildPhase section */ 231 | 232 | /* Begin PBXSourcesBuildPhase section */ 233 | 97DA10541F81466300ADF5D8 /* Sources */ = { 234 | isa = PBXSourcesBuildPhase; 235 | buildActionMask = 2147483647; 236 | files = ( 237 | 97DA10771F81468C00ADF5D8 /* HNScraper.swift in Sources */, 238 | 97DA10741F81468C00ADF5D8 /* HNLogin.swift in Sources */, 239 | 97DA10701F81468C00ADF5D8 /* RessourceFetcher.swift in Sources */, 240 | 97DA10751F81468C00ADF5D8 /* HNParseConfig.swift in Sources */, 241 | 97DA10781F81468C00ADF5D8 /* HNUser.swift in Sources */, 242 | 97DA10761F81468C00ADF5D8 /* HNPost.swift in Sources */, 243 | 97DA10711F81468C00ADF5D8 /* Scanner+ScanBetweenString.swift in Sources */, 244 | 97DA10731F81468C00ADF5D8 /* HNComment.swift in Sources */, 245 | ); 246 | runOnlyForDeploymentPostprocessing = 0; 247 | }; 248 | 97DA105D1F81466300ADF5D8 /* Sources */ = { 249 | isa = PBXSourcesBuildPhase; 250 | buildActionMask = 2147483647; 251 | files = ( 252 | 97DA107B1F8147C100ADF5D8 /* HNPostTest.swift in Sources */, 253 | 97DA107C1F8147C100ADF5D8 /* RessourceFetcherTest.swift in Sources */, 254 | 97DA107A1F8147C100ADF5D8 /* HNScraperTest.swift in Sources */, 255 | 97DA10791F8147C100ADF5D8 /* HNLoginTest.swift in Sources */, 256 | ); 257 | runOnlyForDeploymentPostprocessing = 0; 258 | }; 259 | /* End PBXSourcesBuildPhase section */ 260 | 261 | /* Begin PBXTargetDependency section */ 262 | 97DA10641F81466400ADF5D8 /* PBXTargetDependency */ = { 263 | isa = PBXTargetDependency; 264 | target = 97DA10581F81466300ADF5D8 /* HNScraper */; 265 | targetProxy = 97DA10631F81466400ADF5D8 /* PBXContainerItemProxy */; 266 | }; 267 | /* End PBXTargetDependency section */ 268 | 269 | /* Begin XCBuildConfiguration section */ 270 | 97D0D8561F78EF42007DE08A /* Debug */ = { 271 | isa = XCBuildConfiguration; 272 | buildSettings = { 273 | ALWAYS_SEARCH_USER_PATHS = NO; 274 | CLANG_ANALYZER_NONNULL = YES; 275 | CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; 276 | CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; 277 | CLANG_CXX_LIBRARY = "libc++"; 278 | CLANG_ENABLE_MODULES = YES; 279 | CLANG_ENABLE_OBJC_ARC = YES; 280 | CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; 281 | CLANG_WARN_BOOL_CONVERSION = YES; 282 | CLANG_WARN_COMMA = YES; 283 | CLANG_WARN_CONSTANT_CONVERSION = YES; 284 | CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; 285 | CLANG_WARN_DOCUMENTATION_COMMENTS = YES; 286 | CLANG_WARN_EMPTY_BODY = YES; 287 | CLANG_WARN_ENUM_CONVERSION = YES; 288 | CLANG_WARN_INFINITE_RECURSION = YES; 289 | CLANG_WARN_INT_CONVERSION = YES; 290 | CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; 291 | CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; 292 | CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; 293 | CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; 294 | CLANG_WARN_STRICT_PROTOTYPES = YES; 295 | CLANG_WARN_SUSPICIOUS_MOVE = YES; 296 | CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; 297 | CLANG_WARN_UNREACHABLE_CODE = YES; 298 | CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; 299 | CODE_SIGN_IDENTITY = "iPhone Developer"; 300 | COPY_PHASE_STRIP = NO; 301 | DEBUG_INFORMATION_FORMAT = dwarf; 302 | ENABLE_STRICT_OBJC_MSGSEND = YES; 303 | ENABLE_TESTABILITY = YES; 304 | GCC_C_LANGUAGE_STANDARD = gnu11; 305 | GCC_DYNAMIC_NO_PIC = NO; 306 | GCC_NO_COMMON_BLOCKS = YES; 307 | GCC_OPTIMIZATION_LEVEL = 0; 308 | GCC_PREPROCESSOR_DEFINITIONS = ( 309 | "DEBUG=1", 310 | "$(inherited)", 311 | ); 312 | GCC_WARN_64_TO_32_BIT_CONVERSION = YES; 313 | GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; 314 | GCC_WARN_UNDECLARED_SELECTOR = YES; 315 | GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; 316 | GCC_WARN_UNUSED_FUNCTION = YES; 317 | GCC_WARN_UNUSED_VARIABLE = YES; 318 | IPHONEOS_DEPLOYMENT_TARGET = 9.1; 319 | MTL_ENABLE_DEBUG_INFO = YES; 320 | ONLY_ACTIVE_ARCH = YES; 321 | SDKROOT = iphoneos; 322 | SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; 323 | SWIFT_OPTIMIZATION_LEVEL = "-Onone"; 324 | }; 325 | name = Debug; 326 | }; 327 | 97D0D8571F78EF42007DE08A /* Release */ = { 328 | isa = XCBuildConfiguration; 329 | buildSettings = { 330 | ALWAYS_SEARCH_USER_PATHS = NO; 331 | CLANG_ANALYZER_NONNULL = YES; 332 | CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; 333 | CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; 334 | CLANG_CXX_LIBRARY = "libc++"; 335 | CLANG_ENABLE_MODULES = YES; 336 | CLANG_ENABLE_OBJC_ARC = YES; 337 | CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; 338 | CLANG_WARN_BOOL_CONVERSION = YES; 339 | CLANG_WARN_COMMA = YES; 340 | CLANG_WARN_CONSTANT_CONVERSION = YES; 341 | CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; 342 | CLANG_WARN_DOCUMENTATION_COMMENTS = YES; 343 | CLANG_WARN_EMPTY_BODY = YES; 344 | CLANG_WARN_ENUM_CONVERSION = YES; 345 | CLANG_WARN_INFINITE_RECURSION = YES; 346 | CLANG_WARN_INT_CONVERSION = YES; 347 | CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; 348 | CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; 349 | CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; 350 | CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; 351 | CLANG_WARN_STRICT_PROTOTYPES = YES; 352 | CLANG_WARN_SUSPICIOUS_MOVE = YES; 353 | CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; 354 | CLANG_WARN_UNREACHABLE_CODE = YES; 355 | CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; 356 | CODE_SIGN_IDENTITY = "iPhone Developer"; 357 | COPY_PHASE_STRIP = NO; 358 | DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; 359 | ENABLE_NS_ASSERTIONS = NO; 360 | ENABLE_STRICT_OBJC_MSGSEND = YES; 361 | GCC_C_LANGUAGE_STANDARD = gnu11; 362 | GCC_NO_COMMON_BLOCKS = YES; 363 | GCC_WARN_64_TO_32_BIT_CONVERSION = YES; 364 | GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; 365 | GCC_WARN_UNDECLARED_SELECTOR = YES; 366 | GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; 367 | GCC_WARN_UNUSED_FUNCTION = YES; 368 | GCC_WARN_UNUSED_VARIABLE = YES; 369 | IPHONEOS_DEPLOYMENT_TARGET = 9.1; 370 | MTL_ENABLE_DEBUG_INFO = NO; 371 | SDKROOT = iphoneos; 372 | SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule"; 373 | VALIDATE_PRODUCT = YES; 374 | }; 375 | name = Release; 376 | }; 377 | 97DA106B1F81466400ADF5D8 /* Debug */ = { 378 | isa = XCBuildConfiguration; 379 | buildSettings = { 380 | CODE_SIGN_IDENTITY = ""; 381 | CODE_SIGN_STYLE = Automatic; 382 | CURRENT_PROJECT_VERSION = 1; 383 | DEFINES_MODULE = YES; 384 | DEVELOPMENT_TEAM = 5P2WT92MAV; 385 | DYLIB_COMPATIBILITY_VERSION = 1; 386 | DYLIB_CURRENT_VERSION = 1; 387 | DYLIB_INSTALL_NAME_BASE = "@rpath"; 388 | INFOPLIST_FILE = HNScraper/Info.plist; 389 | INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; 390 | IPHONEOS_DEPLOYMENT_TARGET = 9.1; 391 | LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; 392 | PRODUCT_BUNDLE_IDENTIFIER = StephSercu.HNScraper; 393 | PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)"; 394 | SKIP_INSTALL = NO; 395 | SWIFT_VERSION = 4.0; 396 | TARGETED_DEVICE_FAMILY = "1,2"; 397 | VERSIONING_SYSTEM = "apple-generic"; 398 | VERSION_INFO_PREFIX = ""; 399 | }; 400 | name = Debug; 401 | }; 402 | 97DA106C1F81466400ADF5D8 /* Release */ = { 403 | isa = XCBuildConfiguration; 404 | buildSettings = { 405 | CODE_SIGN_IDENTITY = ""; 406 | CODE_SIGN_STYLE = Automatic; 407 | CURRENT_PROJECT_VERSION = 1; 408 | DEFINES_MODULE = YES; 409 | DEVELOPMENT_TEAM = 5P2WT92MAV; 410 | DYLIB_COMPATIBILITY_VERSION = 1; 411 | DYLIB_CURRENT_VERSION = 1; 412 | DYLIB_INSTALL_NAME_BASE = "@rpath"; 413 | INFOPLIST_FILE = HNScraper/Info.plist; 414 | INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; 415 | IPHONEOS_DEPLOYMENT_TARGET = 9.1; 416 | LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; 417 | PRODUCT_BUNDLE_IDENTIFIER = StephSercu.HNScraper; 418 | PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)"; 419 | SKIP_INSTALL = NO; 420 | SWIFT_VERSION = 4.0; 421 | TARGETED_DEVICE_FAMILY = "1,2"; 422 | VERSIONING_SYSTEM = "apple-generic"; 423 | VERSION_INFO_PREFIX = ""; 424 | }; 425 | name = Release; 426 | }; 427 | 97DA106E1F81466400ADF5D8 /* Debug */ = { 428 | isa = XCBuildConfiguration; 429 | buildSettings = { 430 | ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; 431 | CODE_SIGN_STYLE = Automatic; 432 | DEVELOPMENT_TEAM = 5P2WT92MAV; 433 | INFOPLIST_FILE = HNScraperTests/Info.plist; 434 | LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; 435 | PRODUCT_BUNDLE_IDENTIFIER = StephSercu.HNScraperTests; 436 | PRODUCT_NAME = "$(TARGET_NAME)"; 437 | SWIFT_VERSION = 4.0; 438 | TARGETED_DEVICE_FAMILY = "1,2"; 439 | }; 440 | name = Debug; 441 | }; 442 | 97DA106F1F81466400ADF5D8 /* Release */ = { 443 | isa = XCBuildConfiguration; 444 | buildSettings = { 445 | ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES; 446 | CODE_SIGN_STYLE = Automatic; 447 | DEVELOPMENT_TEAM = 5P2WT92MAV; 448 | INFOPLIST_FILE = HNScraperTests/Info.plist; 449 | LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; 450 | PRODUCT_BUNDLE_IDENTIFIER = StephSercu.HNScraperTests; 451 | PRODUCT_NAME = "$(TARGET_NAME)"; 452 | SWIFT_VERSION = 4.0; 453 | TARGETED_DEVICE_FAMILY = "1,2"; 454 | }; 455 | name = Release; 456 | }; 457 | /* End XCBuildConfiguration section */ 458 | 459 | /* Begin XCConfigurationList section */ 460 | 97D0D8411F78EF42007DE08A /* Build configuration list for PBXProject "HNScraper" */ = { 461 | isa = XCConfigurationList; 462 | buildConfigurations = ( 463 | 97D0D8561F78EF42007DE08A /* Debug */, 464 | 97D0D8571F78EF42007DE08A /* Release */, 465 | ); 466 | defaultConfigurationIsVisible = 0; 467 | defaultConfigurationName = Release; 468 | }; 469 | 97DA106A1F81466400ADF5D8 /* Build configuration list for PBXNativeTarget "HNScraper" */ = { 470 | isa = XCConfigurationList; 471 | buildConfigurations = ( 472 | 97DA106B1F81466400ADF5D8 /* Debug */, 473 | 97DA106C1F81466400ADF5D8 /* Release */, 474 | ); 475 | defaultConfigurationIsVisible = 0; 476 | defaultConfigurationName = Release; 477 | }; 478 | 97DA106D1F81466400ADF5D8 /* Build configuration list for PBXNativeTarget "HNScraperTests" */ = { 479 | isa = XCConfigurationList; 480 | buildConfigurations = ( 481 | 97DA106E1F81466400ADF5D8 /* Debug */, 482 | 97DA106F1F81466400ADF5D8 /* Release */, 483 | ); 484 | defaultConfigurationIsVisible = 0; 485 | defaultConfigurationName = Release; 486 | }; 487 | /* End XCConfigurationList section */ 488 | }; 489 | rootObject = 97D0D83E1F78EF42007DE08A /* Project object */; 490 | } 491 | -------------------------------------------------------------------------------- /HNScraper.xcodeproj/project.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /HNScraper.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | IDEDidComputeMac32BitWarning 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /HNScraper/HNComment.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HNComment.swift 3 | // HackerNews2 4 | // 5 | // Created by Stéphane Sercu on 8/09/17. 6 | // Copyright © 2017 Stéphane Sercu. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | 11 | open class BaseComment { 12 | public var replies: [BaseComment]! = [] 13 | public var level: Int! = 0 14 | public weak var replyTo: BaseComment? 15 | 16 | public convenience init() { 17 | self.init(level: 0, replyTo: nil) 18 | } 19 | public init(level: Int, replyTo: BaseComment?) { 20 | self.level = level 21 | self.replyTo = replyTo 22 | } 23 | public func addReply(_ reply: BaseComment) { 24 | self.replies.append(reply) 25 | } 26 | 27 | } 28 | 29 | open class HNComment: BaseComment { 30 | public convenience init() { 31 | self.init(level: 0, replyTo: nil) 32 | } 33 | public override init(level: Int, replyTo: BaseComment?) { 34 | super.init(level: level, replyTo: replyTo) 35 | } 36 | public enum HNCommentType { 37 | case defaultType 38 | case askHN 39 | case jobs 40 | } 41 | 42 | 43 | public var type: HNCommentType! = .defaultType 44 | public var text: String! = "" 45 | public var id: String! = "" 46 | public var username: String! = "anonymous" 47 | public var isOPNoob: Bool! = false 48 | public var parentId: String? 49 | public var created: String! = "" 50 | public var replyUrl: String! = "" 51 | public var links: [String]! = [] 52 | public var upvoteUrl: String! = "" 53 | public var downvoteUrl: String! = "" 54 | public var upvoted: Bool = false 55 | 56 | 57 | public convenience init?(fromHtml html: String, withParsingConfig parseConfig: [String : Any], levelOffset: Int = 0) { 58 | self.init() 59 | var commentDict: [String : Any]? = parseConfig["Comment"] != nil ? parseConfig["Comment"] as? [String: Any] : nil 60 | if commentDict == nil { 61 | return nil 62 | } 63 | 64 | let scanner = Scanner(string: html) 65 | var upvoteString: NSString? 66 | let downvoteString: NSString? = "" 67 | var level: NSString? = "" 68 | //var parentPostId: NSString? = "" 69 | var cDict: [String : Any] = [:] 70 | 71 | 72 | // Get Comment Level 73 | scanner.scanBetweenString(stringA: (commentDict!["Level"] as! [String: String])["S"]!, stringB: (commentDict!["Level"] as! [String: String])["E"]!, into: &level) 74 | if (level != nil) { 75 | self.level = Int(level!.intValue) / 40 + levelOffset // TODO: add this constant in the parseConfig 76 | } else { 77 | self.level = levelOffset 78 | } 79 | 80 | // Parent isn't in the html anymore... it's parsable in the upvote/reply url... 81 | // Get parentPostId (only if the comment comes from the list of comment ssubmited by a user). 82 | /*scanner.scanBetweenString(stringA: (commentDict!["ParentPostId"] as! [String: String])["S"]!, stringB: (commentDict!["ParentPostId"] as! [String: String])["E"]!, into: &parentPostId) 83 | if (parentPostId != nil) { 84 | self.parentId = (parentPostId as String?) ?? "" 85 | }*/ 86 | 87 | 88 | 89 | 90 | 91 | // If Logged In - Grab Voting Strings 92 | if (html.contains((commentDict!["Upvote"] as! [String: String])["R"]!)) { 93 | // Scan Upvote String 94 | scanner.scanBetweenString(stringA: (commentDict!["Upvote"] as! [String: String])["S"]!, stringB: (commentDict!["Upvote"] as! [String: String])["E"]!, into:&upvoteString) 95 | if (upvoteString != nil) { 96 | self.upvoteUrl = upvoteString!.replacingOccurrences(of: "&", with: "&") 97 | } 98 | // Check for downvote String 99 | if (html.contains((commentDict!["Downvote"] as! [String: String])["R"]!)) { 100 | scanner.scanBetweenString(stringA: (commentDict!["Downvote"] as! [String: String])["S"]!, stringB: (commentDict!["Downvote"] as! [String: String])["E"]!, into:&upvoteString) 101 | if (downvoteString != nil) { 102 | self.downvoteUrl = downvoteString!.replacingOccurrences(of: "&", with: "&") 103 | } 104 | } 105 | } 106 | scanner.scanLocation = 0 107 | 108 | let regs = commentDict!["REG"] as! [[String : Any]] 109 | for dict in regs { 110 | var new: NSString? = "" 111 | let isTrash = dict["I"] as! String == "TRASH" 112 | 113 | scanner.scanBetweenString(stringA: dict["S"] as! String, stringB: dict["E"] as! String, into: &new) 114 | if (!isTrash && (new?.length)! > 0) { 115 | cDict[dict["I"] as! String] = new 116 | } 117 | } 118 | 119 | self.id = cDict["CommentId"] as? String ?? "" 120 | self.text = cDict["Text"] as? String ?? "" 121 | self.username = cDict["Username"] as? String ?? "" 122 | self.isOPNoob = HNUser.cleanNoobUsername(username: &(self.username!)) 123 | self.created = cDict["Time"] as? String ?? "" 124 | self.replyUrl = cDict["ReplyUrl"] as? String ?? "" 125 | 126 | if self.id != "" && html.contains(" HNComment? { 134 | var cDict: [String : Any] = [:] 135 | var commentDict: [String : Any]? = parseConfig["Comment"] != nil ? parseConfig["Comment"] as? [String: Any] : nil 136 | if commentDict == nil { 137 | return nil 138 | } 139 | 140 | let scanner = Scanner(string: html) 141 | var upvoteUrl: NSString? = "" 142 | 143 | 144 | if html.contains((commentDict!["Upvote"] as! [String: String])["R"]!) { 145 | scanner.scanBetweenString(stringA: (commentDict!["Upvote"] as! [String: String])["S"]!, stringB: (commentDict!["Upvote"] as! [String: String])["E"]!, into: &upvoteUrl) 146 | if (upvoteUrl != nil) { 147 | upvoteUrl = upvoteUrl!.replacingOccurrences(of: "&", with: "&") as NSString 148 | } 149 | } 150 | let asks = commentDict!["ASK"] as! [[String : Any]] 151 | for dict in asks { 152 | var new: NSString? = "" 153 | let isTrash = dict["I"] as! String == "TRASH" 154 | scanner.scanBetweenString(stringA: dict["S"] as! String, stringB: dict["E"] as! String, into: &new) 155 | if (!isTrash && (new?.length)! > 0) { 156 | cDict[dict["I"] as! String] = new 157 | } 158 | } 159 | 160 | let newComment = HNComment() 161 | newComment.level = 0 162 | newComment.username = cDict["Username"] as? String ?? "" 163 | newComment.isOPNoob = HNUser.cleanNoobUsername(username: &(newComment.username!)) 164 | newComment.created = cDict["Time"] as? String ?? "" 165 | newComment.text = cDict["Text"] as? String ?? "" 166 | //newComment.links = ... 167 | newComment.type = .askHN 168 | if upvoteUrl != nil { 169 | newComment.upvoteUrl = String(describing: upvoteUrl!) as String //(upvoteUrl?.length)! > 0 ? upvoteUrl : ""; 170 | } 171 | newComment.id = cDict["CommentId"] as? String ?? "" 172 | return newComment 173 | } 174 | public static func parseJobComment(html: String, withParsingConfig parseConfig: [String : Any]) -> HNComment? { 175 | var commentDict: [String : Any]? = parseConfig["Comment"] != nil ? parseConfig["Comment"] as? [String: Any] : nil 176 | if commentDict == nil { 177 | return nil 178 | } 179 | 180 | let scanner = Scanner(string: html) 181 | var cDict: [String : Any] = [:] 182 | 183 | let jobs = commentDict!["JOBS"] as! [[String : Any]] 184 | for dict in jobs { 185 | var new: NSString? = "" 186 | let isTrash = dict["I"] as! String == "TRASH" 187 | scanner.scanBetweenString(stringA: dict["S"] as! String, stringB: dict["E"] as! String, into: &new) 188 | if (!isTrash && (new?.length)! > 0) { 189 | cDict[dict["I"] as! String] = new 190 | } 191 | } 192 | 193 | let newComment = HNComment() 194 | newComment.level = 0 195 | newComment.text = cDict["Text"] as? String ?? "" 196 | //newComment.links = ... 197 | newComment.type = .jobs 198 | 199 | return newComment 200 | } 201 | } 202 | -------------------------------------------------------------------------------- /HNScraper/HNLogin.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HNLogin.swift 3 | // HackerNews2 4 | // 5 | // Created by Stéphane Sercu on 8/09/17. 6 | // Copyright © 2017 Stéphane Sercu. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | 11 | 12 | // abdurhtl 13 | // !Bullshit?Psw$ 14 | 15 | public protocol HNLoginDelegate { 16 | func didLogin(user: HNUser, cookie: HTTPCookie) 17 | } 18 | 19 | public class HNLogin { 20 | private var observers: [HNLoginDelegate] = [] 21 | 22 | public func addObserver(_ observer: HNLoginDelegate) { 23 | self.observers.append(observer) 24 | } 25 | private init() { 26 | if let cookie = self.retrieveSessionCookie() { 27 | self._sessionCookie = cookie 28 | self.getUsernameFromCookie(cookie, completion: {(user, cookie, error) -> Void in 29 | if cookie != nil { 30 | self._user = user 31 | self._sessionCookie = cookie 32 | if self.isLoggedIn() { 33 | for observer in self.observers { 34 | observer.didLogin(user: user!, cookie: cookie!) 35 | } 36 | } 37 | } 38 | }) 39 | } 40 | } 41 | 42 | public enum HNLoginError { 43 | case badCredentials 44 | case serverUnreachable 45 | case noInternet 46 | case unknown 47 | 48 | init?(_ error: RessourceFetcher.RessourceFetchingError?) { 49 | self.init(HNScraper.HNScraperError(error)) 50 | } 51 | init?(_ error: HNScraper.HNScraperError?) { 52 | if error == nil { 53 | return nil 54 | } 55 | if error == .noInternet { 56 | self = .noInternet 57 | } else if error == .serverUnreachable || error == .noData { 58 | self = .serverUnreachable 59 | } else { 60 | self = .unknown 61 | } 62 | } 63 | } 64 | 65 | public static let shared = HNLogin() 66 | 67 | private var _sessionCookie: HTTPCookie? 68 | private var _user: HNUser? 69 | 70 | public var sessionCookie: HTTPCookie? { 71 | get { 72 | return _sessionCookie 73 | } 74 | } 75 | public var user: HNUser? { 76 | get { 77 | return _user 78 | } 79 | } 80 | 81 | 82 | /** 83 | * Log a user in useing the specified credentials. In case of success, 84 | * a HNUser instance is built with the information of the conected 85 | * user and passed as paramater to the completion handler along with a 86 | * cookie containing the session data. 87 | */ 88 | public func login(username: String, psw: String, completion: @escaping ((HNUser?, HTTPCookie?, HNLoginError?) -> Void)) { 89 | let url = HNScraper.baseUrl + "login" 90 | let encodedPass = psw.addingPercentEncoding(withAllowedCharacters: CharacterSet(charactersIn: "!*'();:@&=+$,/?%#[]").inverted) 91 | let bodyString = "acct=\(username)&pw=\(encodedPass!)&whence=news" 92 | guard let bodyData = bodyString.data(using: .utf8) else { 93 | completion(nil, nil, .badCredentials) 94 | return 95 | } 96 | 97 | RessourceFetcher.shared.post(urlString: url, data: bodyData, completion: {data, reponse, error -> Void in 98 | if data == nil { 99 | completion(nil, nil, HNLoginError(error) ?? .unknown) 100 | return 101 | } 102 | if let html = String(data: data!, encoding: .utf8) { 103 | if (!html.contains("Bad login.") && !html.contains("Unknown or expired link.")) { 104 | let scanner = Scanner(string: html) 105 | var trash: NSString? = "" 106 | var karma: NSString? = "" 107 | 108 | scanner.scanUpTo("/a> (", into: &trash) // TODO: use config file 109 | scanner.scanString("/a> (", into: &trash) 110 | scanner.scanUpTo(")", into: &karma) 111 | self._user = HNUser(username: username, karma: karma as String!, age: "", aboutInfo: "") 112 | 113 | self.getLoggedInUser(user: self._user!, completion: {(user, cookie, error) -> Void in 114 | 115 | if self.isLoggedIn() { 116 | for observer in self.observers { 117 | observer.didLogin(user: user!, cookie: cookie!) 118 | } 119 | } 120 | completion(user, cookie, error) 121 | }) 122 | 123 | 124 | } else { 125 | print("Probably wrong password") // TODO: logging 126 | completion(nil, nil, .badCredentials) 127 | } 128 | 129 | 130 | } else { 131 | print("Post request failed") 132 | completion(nil, nil, HNLoginError(error) ?? .unknown) 133 | } 134 | 135 | }) 136 | 137 | } 138 | 139 | public func logout() { 140 | if self._sessionCookie != nil { 141 | HTTPCookieStorage.shared.deleteCookie(self._sessionCookie!) 142 | self._sessionCookie = nil 143 | 144 | } 145 | self._user = nil 146 | } 147 | 148 | // TODO: clean this up, use the HNScraper's getUser method 149 | /** 150 | * Fetch the informations about a user. 151 | * - parameters: 152 | * - user: a HNUser object with at least the username of the user you want the info about. 153 | All the other properties are just copied in the result object or replaced with the newly fetched informations. 154 | * - completion: 155 | */ 156 | private func getLoggedInUser(user: HNUser, completion: @escaping ((HNUser?, HTTPCookie?, HNLoginError?) -> Void)) { 157 | let url = "https://news.ycombinator.com/user?id=\(user.username!)" 158 | 159 | RessourceFetcher.shared.fetchData(urlString: url, completion: {(data, error) -> Void in 160 | 161 | if let html = String(data: data!, encoding: .utf8) { 162 | var newUser: HNUser? 163 | // Getting user info 164 | if !(html.contains("We've limited requests for this url.")) { 165 | HNParseConfig.shared.getDictionnary(completion: {(parsingConfig, configFileError) -> Void in 166 | if parsingConfig != nil { 167 | newUser = HNUser(fromHtml: html, withParsingConfig: parsingConfig!) 168 | if newUser == nil { 169 | newUser = user 170 | } 171 | } else { 172 | print("couldn't fetch the configFile") 173 | } 174 | // Getting cookie 175 | self._sessionCookie = self.retrieveSessionCookie() 176 | self._user = newUser 177 | completion(newUser, self._sessionCookie, HNLoginError(error)) 178 | 179 | }) 180 | } else { 181 | print("Couldn't fetch user informations") 182 | completion(nil, nil, HNLoginError(error) ?? .serverUnreachable) 183 | } 184 | } else { 185 | completion(nil, nil, HNLoginError(error) ?? .unknown) 186 | } 187 | 188 | }) 189 | } 190 | 191 | // TODO: better error gesture & logging 192 | private func getUsernameFromCookie(_ cookie: HTTPCookie, completion: @escaping ((HNUser?, HTTPCookie?, HNLoginError?) -> Void)) { 193 | let url = "https://news.ycombinator.com/user?id=pg" // any valid url would do 194 | 195 | RessourceFetcher.shared.fetchData(urlString: url, completion: {(data, error) -> Void in 196 | if data != nil { 197 | if let html = String(data: data!, encoding: .utf8) { 198 | if (!html.contains("", into: &userString) 207 | scanner.scanUpTo(" (", into: &trash) 208 | scanner.scanString(" (", into: &trash) 209 | scanner.scanUpTo(")", into: &karma) 210 | 211 | let user = HNUser(username: userString as String!, karma: karma as String!, age: "", aboutInfo: "") 212 | 213 | self.getLoggedInUser(user: user, completion: completion) 214 | 215 | 216 | } else { 217 | print("getUsernameFromCookie: bad cookie?") // TODO: Logging 218 | completion(nil, nil, HNLoginError(error)) 219 | } 220 | 221 | 222 | } else { 223 | print("getUsernameFromCookie: Get request failed: not html?") 224 | completion(nil, nil, HNLoginError(error)) 225 | } 226 | } else { 227 | print("getUsernameFromCookie: Get request failed: no data") 228 | completion(nil, nil, HNLoginError(error)) 229 | } 230 | 231 | 232 | }) 233 | 234 | 235 | 236 | } 237 | 238 | private func retrieveSessionCookie() -> HTTPCookie? { 239 | if let cookieArray = HTTPCookieStorage.shared.cookies(for: URL(string: HNScraper.baseUrl)!) { 240 | if cookieArray.count > 0 { 241 | for cookie in cookieArray { 242 | if cookie.name == "user" { 243 | return cookie 244 | } 245 | } 246 | 247 | } 248 | } 249 | 250 | return nil 251 | } 252 | 253 | public func isLoggedIn() -> Bool { 254 | return self.sessionCookie != nil && self._user != nil 255 | } 256 | 257 | 258 | } 259 | -------------------------------------------------------------------------------- /HNScraper/HNParseConfig.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HNParseConfig.swift 3 | // HackerNews2 4 | // 5 | // Created by Stéphane Sercu on 8/09/17. 6 | // Copyright © 2017 Stéphane Sercu. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | 11 | 12 | /** 13 | Manage to download, store and cache the json file 14 | used to parse the pages of the website. 15 | */ 16 | public class HNParseConfig { 17 | private let savingKey = "HNParseConfig" 18 | private let url = "https://raw.githubusercontent.com/tsucres/HNScraper/v0.2.2/hn.json" 19 | private var _config: [String: Any]? = nil 20 | private init() {} 21 | public static let shared = HNParseConfig() 22 | 23 | /// Returns the data if in cache (if not it returns 24 | /// nil, you need to call getDictionnary to fetch the file) 25 | public var data:[String: Any]? { 26 | get { 27 | if (_config == nil) { 28 | _config = cacheData 29 | } 30 | return _config 31 | } 32 | } 33 | /// Looks for the data in local storage 34 | private var cacheData: [String: Any]? { 35 | get { 36 | let defaults = UserDefaults.standard 37 | if defaults.object(forKey: self.savingKey) != nil { 38 | return (defaults.object(forKey: self.savingKey) as! [String: Any]) 39 | } else { 40 | return nil 41 | } 42 | } 43 | } 44 | /** 45 | The completion handler is called with the configration data as 46 | parameter when the json file has been fetched. It firstly 47 | checks in the clocal storage if it has already been fetched. 48 | */ 49 | internal func getDictionnary(completion: @escaping (([String: Any]?, RessourceFetcher.RessourceFetchingError?) -> Void)) { 50 | if self.data != nil { 51 | completion(self.data, nil) 52 | } else { 53 | self.downloadConfigFile(completion: completion) 54 | } 55 | } 56 | 57 | 58 | /// Downloads the configFile and store it locally. If a configFile is already saved, it's replaced. 59 | internal func downloadConfigFile(completion: @escaping (([String: Any]?, RessourceFetcher.RessourceFetchingError?) -> Void)) { 60 | RessourceFetcher.shared.getJson(url: self.url, completion: { (json, error) -> Void in 61 | if (json != nil) { 62 | let defaults = UserDefaults.standard 63 | defaults.set(json, forKey: self.savingKey) 64 | self._config = json 65 | completion(json, error) 66 | } else { 67 | completion(nil, error ?? .noData) 68 | } 69 | }) 70 | } 71 | 72 | /// Downloads the configFile and store it locally. If a configFile is already saved, it's replaced. 73 | public func forceRedownload(completion: @escaping ((HNScraper.HNScraperError?) -> Void)) { 74 | self.downloadConfigFile { (_, ressourceFetcherError) in 75 | completion(HNScraper.HNScraperError.init(ressourceFetcherError)) 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /HNScraper/HNPost.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HNPost.swift 3 | // HackerNews2 4 | // 5 | // Created by Stéphane Sercu on 8/09/17. 6 | // Copyright © 2017 Stéphane Sercu. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | 11 | 12 | /// Model used by the HN Scraper to store avery data about a post. 13 | open class HNPost { 14 | public enum PostType { 15 | case defaultType 16 | case askHN 17 | case jobs 18 | 19 | public init?(index: Int) { 20 | switch index { 21 | case 0: self = .defaultType 22 | case 1: self = .askHN 23 | case 2: self = .jobs 24 | default: return nil 25 | } 26 | } 27 | } 28 | 29 | public var type: PostType = .defaultType 30 | public var username: String = "" 31 | public var isOPNoob: Bool = false 32 | public var url: URL?// = URL(string: "")! 33 | public var urlDomain: String { 34 | get { 35 | if url == nil { 36 | return "" 37 | } 38 | var dom: String? = self.url!.host 39 | if dom != nil && dom!.hasPrefix("www.") { 40 | dom = String(dom!.dropFirst(4)) 41 | } 42 | return dom ?? "" 43 | } 44 | } 45 | public var title: String = "" 46 | public var points: Int = 0 47 | public var commentCount: Int = 0 48 | public var id: String = "" 49 | public var time: String = "" 50 | 51 | public var upvoted: Bool = false 52 | public var upvoteAdditionURL: String? 53 | 54 | public var favorited: Bool = false // TODO: there's no way to know from a "list page", but it could be filled from the discussion thread. 55 | 56 | public var replyAction: String? 57 | public var replyParent: String? 58 | public var replyGoto: String? 59 | public var replyHmac: String? 60 | public var replyText: String? 61 | 62 | 63 | public init() {} 64 | 65 | /** 66 | * Build the model by parsing the html of a post item on the HN website. 67 | * - parameters: 68 | * - html: the html code to parse 69 | * - parseConfig: the parameters from the json file containing all the needed parsing informations. 70 | */ 71 | public convenience init?(fromHtml html: String, withParsingConfig parseConfig: [String : Any]) { 72 | self.init() 73 | 74 | var postsConfig: [String : Any]? = (parseConfig["Post"] != nil) ? parseConfig["Post"] as? [String : Any] : nil 75 | if postsConfig == nil { 76 | return nil 77 | } 78 | 79 | if html.contains(" [dead] 0) { 104 | postDict[part["I"] as! String] = new 105 | } 106 | } 107 | 108 | 109 | // Set Values 110 | self.url = postDict["UrlString"] != nil ? URL(string: postDict["UrlString"] as! String) : nil 111 | self.title = postDict["Title"] as? String ?? "" 112 | self.points = Int(((postDict["Points"] as? String ?? "").components(separatedBy: " ")[0])) ?? 0 113 | self.username = postDict["Username"] as? String ?? "" 114 | self.isOPNoob = HNUser.cleanNoobUsername(username: &self.username) 115 | self.id = postDict["PostId"] as? String ?? "" 116 | self.time = postDict["Time"] as? String ?? "" 117 | if self.id != "" && html.contains(" Void)) { 713 | if !HNLogin.shared.isLoggedIn() { 714 | completion(.notLoggedIn) 715 | return 716 | } 717 | if post.upvoteAdditionURL != nil { 718 | let url = HNScraper.baseUrl + post.upvoteAdditionURL!.replacingOccurrences(of: "&", with: "&").replacingOccurrences(of: "how=up", with: "un=t").replacingOccurrences(of: "vote?id=", with: "fave?id=") 719 | 720 | downloadHtmlPage(urlString: url, cookie: HNLogin.shared.sessionCookie, completion: { html, error -> Void in 721 | // The id of the unfavorited post must be absent from the html 722 | if html == nil { 723 | completion(error ?? .noData) 724 | } else { 725 | if !html!.contains("id='" + post.id + "'") { 726 | completion(nil) 727 | } else { 728 | completion(error ?? .unknown) 729 | } 730 | } 731 | }) 732 | } else { 733 | completion(.invalidURL) 734 | } 735 | 736 | } 737 | 738 | public func getUserFrom(Username username: String, completion: ((HNUser?, HNScraperError?) -> Void)?) { 739 | getHtmlAndParsingConfig(url: HNScraper.baseUrl + "user?id=" + username, completion: { html, error -> Void in 740 | if html == nil { 741 | completion?(nil, error ?? .noData) 742 | return 743 | } 744 | if HNParseConfig.shared.data == nil { 745 | completion?(nil, .missingOrCorruptedConfigFile) 746 | return 747 | } 748 | if html!.contains("No such user.") { 749 | completion?(nil, .noSuchUser) 750 | return 751 | } 752 | completion?(HNUser(fromHtml: html!, withParsingConfig: HNParseConfig.shared.data!), error) 753 | }) 754 | } 755 | 756 | 757 | } 758 | -------------------------------------------------------------------------------- /HNScraper/HNUser.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HNUser.swift 3 | // HackerNews2 4 | // 5 | // Created by Stéphane Sercu on 8/09/17. 6 | // Copyright © 2017 Stéphane Sercu. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | 11 | open class HNUser { 12 | public var username: String! 13 | public var karma: Int! 14 | public var age: Date! 15 | public var aboutInfo: String? 16 | public var isNoob: Bool! = false // TODO 17 | 18 | 19 | public init(username: String, karma: Int, age: Date, aboutInfo: String? = nil, isNoob: Bool = false) { 20 | self.username = username 21 | self.age = age 22 | self.karma = karma 23 | self.aboutInfo = aboutInfo 24 | self.isNoob = isNoob 25 | } 26 | /** 27 | * - parameters: 28 | * - age: the number of days in the past relatively to current date. 29 | * 30 | */ 31 | public convenience init(username: String, karma: String, age: String, aboutInfo: String?, isNoob: Bool = false) { 32 | self.init(username: username, karma: Int(karma.replacingOccurrences(of: " ", with: "")) ?? 0, age: HNUser.dateFromFormat(date: age), aboutInfo: aboutInfo) 33 | } 34 | 35 | public convenience init?(fromHtml html: String, withParsingConfig parseConfig: [String : Any]) { 36 | var userDict: [String : Any]? = parseConfig["User"] != nil ? parseConfig["User"] as? [String: Any] : nil 37 | if (userDict == nil || userDict!["Parts"] == nil) { 38 | return nil 39 | } 40 | 41 | let scanner = Scanner(string: html) 42 | var uDict: [String: Any] = [:] 43 | var isNoob = false 44 | let parts = userDict!["Parts"] as! [[String : Any]] 45 | for dict in parts { 46 | var new: NSString? = "" 47 | let isTrash = (dict["I"] as! String) == "TRASH" 48 | let start = dict["S"] as! String 49 | let end = dict["E"] as! String 50 | if scanner.string.contains(start) && scanner.string.contains(end) { 51 | scanner.scanBetweenString(stringA: start, stringB: end, into: &new) 52 | if (!isTrash && (new?.length)! > 0) { 53 | if dict["I"] as! String == "user" { 54 | var newStr: String = String(describing: new!) 55 | isNoob = HNUser.cleanNoobUsername(username: &(newStr)) 56 | new = newStr as NSString 57 | /*if new!.contains("") { 58 | new = new!.replacingOccurrences(of: "", with: "") as NSString 59 | new = new!.replacingOccurrences(of: "", with: "") as NSString 60 | isNoob = true 61 | }*/ 62 | } 63 | uDict[dict["I"] as! String] = new 64 | } 65 | } 66 | } 67 | 68 | if uDict["user"] == nil { 69 | return nil 70 | } 71 | self.init(username: uDict["user"] as! String, karma: uDict["karma"] as? String ?? "", age: uDict["created"] as? String ?? "", aboutInfo: uDict["about"] as? String, isNoob: isNoob) 72 | 73 | } 74 | 75 | public static func cleanNoobUsername(username: inout String) -> Bool { 76 | if username.contains("") { 77 | username = username.replacingOccurrences(of: "", with: "") 78 | username = username.replacingOccurrences(of: "", with: "") 79 | return true 80 | } 81 | return false 82 | } 83 | 84 | /// Converts the number of days from current date to a Date instance. 85 | private static func dateFromNumberOfDays(_ numberOfDays: Int) -> Date { 86 | return Calendar.current.date(byAdding: .day, value: -numberOfDays, to: Date())! 87 | } 88 | 89 | private static func dateFromFormat(date: String, dateFormat: String = "yyyy-MM-dd") -> Date { 90 | let dateFormatter = DateFormatter() 91 | dateFormatter.dateFormat = dateFormat 92 | 93 | let date = dateFormatter.date(from: date) 94 | return date ?? Date() 95 | } 96 | 97 | } 98 | -------------------------------------------------------------------------------- /HNScraper/Helpers/RessourceFetcher.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HNHelpers.swift 3 | // HackerNews2 4 | // 5 | // Created by Stéphane Sercu on 8/09/17. 6 | // Copyright © 2017 Stéphane Sercu. All rights reserved. 7 | // 8 | 9 | import UIKit 10 | 11 | 12 | 13 | /// Singleton implementation of a GET/POST request sender 14 | class RessourceFetcher { 15 | public typealias FetchCompletion = (_ data: Data?, _ error: RessourceFetchingError?) -> Void 16 | public typealias JsonFetchCompletion = (_ json: Dictionary?, _ error: RessourceFetchingError?) -> Void 17 | 18 | public enum RessourceFetchingError: Error { 19 | case invalidURL 20 | case badHTTPRequest400Range 21 | case serverError500Range 22 | case serverUnreachable 23 | case noIternet 24 | case securityIssue 25 | case parsingError 26 | case noData 27 | case unknown 28 | } 29 | 30 | 31 | /// Turns a URLError to a simpler RessourceFetchingError 32 | private func errorParsing(_ error: Error) -> RessourceFetchingError { 33 | if let urlError = error as? URLError { 34 | if [URLError.unsupportedURL , URLError.badURL].contains(urlError.code) { 35 | return .invalidURL 36 | } else if [URLError.notConnectedToInternet , URLError.networkConnectionLost, URLError.internationalRoamingOff].contains(urlError.code) { 37 | return .noIternet 38 | } else if [URLError.secureConnectionFailed , URLError.appTransportSecurityRequiresSecureConnection].contains(urlError.code) { 39 | return .securityIssue 40 | } else if [URLError.cannotConnectToHost, URLError.timedOut, URLError.cannotFindHost].contains(urlError.code) { 41 | return .serverUnreachable 42 | } else { 43 | return .unknown 44 | } 45 | 46 | } else { 47 | return .unknown 48 | } 49 | } 50 | 51 | private init() { 52 | 53 | } 54 | 55 | static let shared = RessourceFetcher() 56 | 57 | /// Fetches the ressource pointed by the specified url and passes it to the completionHandler 58 | func fetchData(urlString: String, completion: @escaping FetchCompletion, timeout: TimeInterval = 20) { 59 | if let url = URL(string: urlString) { 60 | let req = URLRequest(url: url, cachePolicy: .reloadIgnoringCacheData, timeoutInterval: timeout) 61 | self.execRequest(req, completion: {(data, response, error) -> Void in 62 | completion(data, error) 63 | }) 64 | } else { 65 | completion(nil, .invalidURL) 66 | } 67 | } 68 | 69 | /// Creates a dataTask with the specified request and passes the response to the completion closure. Basically just helps handling URLErrors. 70 | private func execRequest(_ request: URLRequest, completion: @escaping ((Data?, URLResponse?, RessourceFetchingError?) -> Void)) { 71 | let task = URLSession.shared.dataTask(with: request, completionHandler: {(data, response, error) -> Void in 72 | var ressourceFetchingError: RessourceFetchingError? 73 | if data == nil { 74 | ressourceFetchingError = .noData 75 | } 76 | if let httpResponse = response as? HTTPURLResponse { 77 | if httpResponse.statusCode >= 500 { 78 | ressourceFetchingError = .serverError500Range 79 | } else if httpResponse.statusCode >= 400 { 80 | ressourceFetchingError = .badHTTPRequest400Range 81 | } 82 | } 83 | if error != nil { 84 | ressourceFetchingError = self.errorParsing(error!) 85 | } 86 | DispatchQueue.main.async(execute: { ()->() in 87 | completion(data, response, ressourceFetchingError) 88 | }) 89 | 90 | }) 91 | task.resume() 92 | } 93 | 94 | /// Fetch the ressource pointed by the specified url and try to parse it to a json dictionnary. The result is passed to the completion closure. 95 | func getJson(url: String, completion: @escaping JsonFetchCompletion) { 96 | self.fetchData(urlString: url, completion: {(data, error) -> Void in 97 | if error != nil { 98 | completion(nil, error) 99 | } else if data == nil { 100 | completion(nil, .noData) 101 | } else { 102 | guard let json = try? JSONSerialization.jsonObject(with: data!, options: .mutableContainers) as! [String: Any] 103 | else { 104 | completion(nil, .parsingError) 105 | return 106 | } 107 | completion(json, nil) 108 | 109 | 110 | } 111 | }) 112 | } 113 | 114 | /// Send a POST request to the specified URL with the specified data and cookie 115 | func post(urlString: String, data: Data, cookies: [HTTPCookie] = [], completion: @escaping ((Data?, URLResponse?, RessourceFetchingError?) -> Void)) { 116 | guard let url = URL(string: urlString) else { 117 | completion(nil, nil, .invalidURL) 118 | return 119 | } 120 | 121 | var request = URLRequest(url: url, cachePolicy: .reloadIgnoringCacheData, timeoutInterval: 30) 122 | request.httpShouldHandleCookies = true 123 | request.allHTTPHeaderFields = [:] 124 | request.httpMethod = "POST" 125 | request.httpBody = data 126 | request.allHTTPHeaderFields = HTTPCookie.requestHeaderFields(with: cookies) 127 | 128 | self.execRequest(request, completion: {(data, response, error) -> Void in 129 | completion(data, response, error) 130 | }) 131 | } 132 | 133 | func get(urlString: String, cookies: [HTTPCookie] = [], completion: @escaping ((Data?, URLResponse?, RessourceFetchingError?) -> Void)) { 134 | guard let url = URL(string: urlString) else { 135 | completion(nil, nil, .invalidURL) 136 | return 137 | } 138 | 139 | var request = URLRequest(url: url, cachePolicy: .reloadIgnoringCacheData, timeoutInterval: 30) 140 | request.httpShouldHandleCookies = true 141 | request.allHTTPHeaderFields = [:] 142 | request.httpMethod = "GET" 143 | request.allHTTPHeaderFields = HTTPCookie.requestHeaderFields(with: cookies) 144 | 145 | 146 | self.execRequest(request, completion: {(data, response, error) -> Void in 147 | completion(data, response, error) 148 | }) 149 | } 150 | } 151 | 152 | 153 | 154 | 155 | -------------------------------------------------------------------------------- /HNScraper/Helpers/Scanner+ScanBetweenString.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Scanner+ScanBetweenString.swift 3 | // HNScraper 4 | // 5 | // Created by Stéphane Sercu on 29/09/17. 6 | // Copyright © 2017 Stéphane Sercu. All rights reserved. 7 | // 8 | 9 | import Foundation 10 | extension Scanner { 11 | func scanBetweenString(stringA: String, stringB: String, into: AutoreleasingUnsafeMutablePointer?) { 12 | var trash: NSString? = "" 13 | self.scanUpTo(stringA, into: &trash) 14 | self.scanString(stringA, into: &trash) 15 | self.scanUpTo(stringB, into: into) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /HNScraper/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleDevelopmentRegion 6 | $(DEVELOPMENT_LANGUAGE) 7 | CFBundleExecutable 8 | $(EXECUTABLE_NAME) 9 | CFBundleIdentifier 10 | $(PRODUCT_BUNDLE_IDENTIFIER) 11 | CFBundleInfoDictionaryVersion 12 | 6.0 13 | CFBundleName 14 | $(PRODUCT_NAME) 15 | CFBundlePackageType 16 | FMWK 17 | CFBundleShortVersionString 18 | 0.2.2 19 | CFBundleVersion 20 | $(CURRENT_PROJECT_VERSION) 21 | NSPrincipalClass 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /HNScraperTests/HNLoginTest.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HNLoginTest.swift 3 | // HNScraper 4 | // 5 | // Created by Stéphane Sercu on 25/09/17. 6 | // Copyright © 2017 Stéphane Sercu. All rights reserved. 7 | // 8 | 9 | import XCTest 10 | @testable import HNScraper 11 | 12 | // 13 | // 14 | class HNLoginTest: XCTestCase { 15 | 16 | override func setUp() { 17 | super.setUp() 18 | // Put setup code here. This method is called before the invocation of each test method in the class. 19 | cleanCookies() 20 | } 21 | 22 | override func tearDown() { 23 | // Put teardown code here. This method is called after the invocation of each test method in the class. 24 | super.tearDown() 25 | cleanCookies() 26 | 27 | } 28 | func cleanCookies() { 29 | let cookieStore = HTTPCookieStorage.shared 30 | for cookie in cookieStore.cookies ?? [] { 31 | cookieStore.deleteCookie(cookie) 32 | } 33 | HNLogin.shared.logout() 34 | } 35 | 36 | func testGoodLogin() { 37 | // This is an example of a functional test case. 38 | // Use XCTAssert and related functions to verify your tests produce the correct results. 39 | let exp = expectation(description: "Correct login with abdurhtl") 40 | HNLogin.shared.login(username: "abdurhtl", psw: "!Bullshit?Psw$", completion: {(user, cookie, error) -> Void in 41 | XCTAssertNotNil(cookie) 42 | XCTAssertEqual(user?.username, "abdurhtl") 43 | exp.fulfill() 44 | }) 45 | let exp2 = expectation(description: "Correct login with testHNScrapper") 46 | HNLogin.shared.login(username: "testHNScrapper", psw: "&$!?èé%`ç\"'-some_thing", completion: {(user, cookie, error) -> Void in 47 | XCTAssertNotNil(cookie) 48 | XCTAssertEqual(user?.username, "testHNScrapper") 49 | exp2.fulfill() 50 | }) 51 | 52 | wait(for: [exp, exp2], timeout: HNScraperTest.defaultTimeOut) 53 | } 54 | 55 | func testBadPasswordLogin() { 56 | let exp = expectation(description: "not logged in") 57 | HNLogin.shared.login(username: "who?", psw: "random", completion: {(user, cookie, error) -> Void in 58 | XCTAssertNil(user) 59 | XCTAssertNil(cookie) 60 | XCTAssertEqual(error, HNLogin.HNLoginError.badCredentials) 61 | exp.fulfill() 62 | }) 63 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 64 | } 65 | 66 | 67 | func testIsLoggedIn() { 68 | let exp = expectation(description: "isLoggedIn() returns true") 69 | XCTAssertFalse(HNLogin.shared.isLoggedIn()) 70 | HNLogin.shared.login(username: "abdurhtl", psw: "!Bullshit?Psw$", completion: {(user, cookie, error) -> Void in 71 | XCTAssertTrue(HNLogin.shared.isLoggedIn()) 72 | exp.fulfill() 73 | 74 | }) 75 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 76 | 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /HNScraperTests/HNPostTest.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HNPostTest.swift 3 | // HNScraperTests 4 | // 5 | // Created by Stéphane Sercu on 29/09/17. 6 | // Copyright © 2017 Stéphane Sercu. All rights reserved. 7 | // 8 | 9 | import XCTest 10 | @testable import HNScraper 11 | 12 | class HNPostTest: XCTestCase { 13 | 14 | 15 | func testDefaultPostParsing() { 16 | // TODO: This tests nothing about upvoteURL, points, 17 | let exp = expectation(description: "All important field of HNPost are correctly parsed") 18 | HNScraper.shared.getPost(ById: "15364896", buildHierarchy: false) { (post, comments, error) in 19 | XCTAssertNil(error) 20 | XCTAssertNotNil(post) 21 | XCTAssertEqual(post?.title, "Cloudflare Workers: Run JavaScript Service Workers at the Edge") 22 | XCTAssertEqual(post?.username, "thomseddon") 23 | XCTAssertEqual(post?.urlDomain, "blog.cloudflare.com") 24 | XCTAssertGreaterThan(post!.points, 0) 25 | XCTAssertGreaterThan(post!.commentCount, 0) 26 | XCTAssertEqual(post?.commentCount, comments.count) 27 | XCTAssertEqual(post?.type, .defaultType) 28 | XCTAssertEqual(post?.url, URL(string:"https://blog.cloudflare.com/introducing-cloudflare-workers/")) 29 | exp.fulfill() 30 | } 31 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 32 | } 33 | 34 | func testAskPostParsing() { 35 | let exp = expectation(description: "All important field of HNPost are correctly parsed") 36 | HNScraper.shared.getPost(ById: "15361048", buildHierarchy: false) { (post, comments, error) in 37 | XCTAssertNil(error) 38 | XCTAssertNotNil(post) 39 | XCTAssertEqual(post?.title, "Ask HN: Library recommendations for a Client / Server project (all Java)") 40 | XCTAssertEqual(post?.username, "HockeyPlayer") 41 | XCTAssertEqual(post?.urlDomain, "news.ycombinator.com") 42 | XCTAssertGreaterThan(post!.points, 0) 43 | XCTAssertGreaterThan(post!.commentCount, 0) 44 | XCTAssertEqual(post?.commentCount, comments.count - 1) // The first comment is the Ask 45 | XCTAssertEqual(post?.type, .askHN) 46 | XCTAssertEqual(post?.url, URL(string:"https://news.ycombinator.com/item?id=15361048")) 47 | exp.fulfill() 48 | } 49 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 50 | } 51 | 52 | // TODO: test noob users... but noob users change every time 53 | 54 | func testJobPostParsing() { 55 | 56 | } 57 | 58 | func testShowPostParsing() { 59 | 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /HNScraperTests/HNScraperTest.swift: -------------------------------------------------------------------------------- 1 | // 2 | // HNScraperTest.swift 3 | // HNScraperTests 4 | // 5 | // Created by Stéphane Sercu on 25/09/17. 6 | // Copyright © 2017 Stéphane Sercu. All rights reserved. 7 | // 8 | 9 | import XCTest 10 | @testable import HNScraper 11 | 12 | class HNScraperLoginNeededTest: XCTestCase { 13 | override func setUp() { 14 | super.setUp() 15 | let exp = expectation(description: "Successfull login") 16 | login(completion: {(success) -> Void in 17 | XCTAssertTrue(success, "HNLogin is probably broken") 18 | exp.fulfill() 19 | }) 20 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 21 | } 22 | 23 | 24 | func login(completion: @escaping ((Bool) -> Void)) { 25 | if !HNLogin.shared.isLoggedIn() { 26 | let username = HNScraperTest.validCredential["username"]! 27 | let password = HNScraperTest.validCredential["password"]! 28 | HNLogin.shared.login(username: username, psw: password, completion: {(user, cookie, error) -> Void in 29 | completion(error == nil) 30 | }) 31 | } else { 32 | completion(true) 33 | } 34 | } 35 | 36 | func getFirstPost(completion: @escaping ((HNPost?) -> Void)) { 37 | HNScraper.shared.getPostsList(page: .news, completion: {(posts, linkForMore, error) -> Void in 38 | XCTAssertGreaterThan(posts.count, 0, "The getPostLists method is probably broken. Or hackernews is down...") 39 | completion(posts[0]) 40 | }) 41 | } 42 | func getPost(id: String, completion: @escaping ((HNPost?) -> Void)) { 43 | HNScraper.shared.getPost(ById: id) { (post, comments, error) in 44 | XCTAssertNotNil(post, "The getPostbyId method is probably broken. Or hackernews is down...") 45 | completion(post) 46 | } 47 | } 48 | 49 | // Try to upvote the first post of the home page 50 | func testUpvotePost() { 51 | let exp = expectation(description: "get no error") 52 | getFirstPost() { (post) in 53 | HNScraper.shared.upvote(Post: post!, completion: {(error) -> Void in 54 | XCTAssertNil(error) 55 | exp.fulfill() 56 | }) 57 | } 58 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 59 | } 60 | // Try to upvote the first post of the home page after it upvotes it. 61 | func testUpvoteUpvotedPost() { 62 | let exp = expectation(description: "get no error") 63 | getFirstPost(completion: {(post) -> Void in 64 | HNScraper.shared.upvote(Post: post!, completion: {(error) -> Void in 65 | XCTAssertNil(error, "If this fails, it probably means that the upvotePost method is broken.") 66 | HNScraper.shared.upvote(Post: post!, completion: {(error) -> Void in 67 | XCTAssertNil(error) 68 | exp.fulfill() 69 | }) 70 | 71 | }) 72 | }) 73 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 74 | } 75 | func testUpvoteBadPostId() { 76 | let post = HNPost() 77 | post.id = "where?" 78 | post.upvoteAdditionURL = "somewhereFarFarAway" 79 | let exp = expectation(description: "get a invalidUrl error") 80 | HNScraper.shared.upvote(Post: post, completion: {(error) -> Void in 81 | XCTAssertEqual(error, .invalidURL) 82 | exp.fulfill() 83 | }) 84 | 85 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 86 | } 87 | func testFavoritePost() { 88 | let exp = expectation(description: "get no error") 89 | getFirstPost(completion: {(post) -> Void in 90 | HNScraper.shared.favorite(Post: post!, completion: {(error) -> Void in 91 | XCTAssertNil(error) 92 | exp.fulfill() 93 | }) 94 | }) 95 | wait(for: [exp], timeout: 2*HNScraperTest.defaultTimeOut) 96 | } 97 | func testUnFavoriteFavoritedPost() { 98 | let exp = expectation(description: "get no error") 99 | getFirstPost(completion: {(post) -> Void in 100 | HNScraper.shared.favorite(Post: post!, completion: {(error) -> Void in 101 | XCTAssertNil(error, "If this fails, it probably means that the favoritePost method is broken.") 102 | HNScraper.shared.unfavorite(Post: post!, completion: {(error) -> Void in 103 | XCTAssertNil(error) 104 | exp.fulfill() 105 | }) 106 | 107 | }) 108 | }) 109 | wait(for: [exp], timeout: 3*HNScraperTest.defaultTimeOut) 110 | } 111 | func testUnFavoriteNonFavoritedPost() { 112 | let exp = expectation(description: "get no error") 113 | getPost(id: "15364646") { (post) in 114 | HNScraper.shared.unfavorite(Post: post!, completion: {(error) -> Void in 115 | XCTAssertNil(error) 116 | exp.fulfill() 117 | }) 118 | } 119 | wait(for: [exp], timeout: 2*HNScraperTest.defaultTimeOut) 120 | } 121 | func testUnVoteVotedPost() { 122 | let exp = expectation(description: "get no error") 123 | getPost(id: "15350139") { (post) in 124 | HNScraper.shared.upvote(Post: post!, completion: {(error) -> Void in 125 | XCTAssertNil(error, "If this fails, it probably means that the upvotePost method is broken.") 126 | HNScraper.shared.unvote(Post: post!, completion: {(error) -> Void in 127 | XCTAssertNil(error) 128 | exp.fulfill() 129 | }) 130 | 131 | }) 132 | } 133 | wait(for: [exp], timeout: 200*HNScraperTest.defaultTimeOut) 134 | } 135 | func testUnvoteNonVotedPost() { 136 | let exp = expectation(description: "get no error") 137 | getPost(id: "15350139") { (post) in 138 | HNScraper.shared.unvote(Post: post!, completion: {(error) -> Void in 139 | XCTAssertNil(error) 140 | exp.fulfill() 141 | }) 142 | } 143 | wait(for: [exp], timeout: 2*HNScraperTest.defaultTimeOut) 144 | } 145 | 146 | func testUpvoteComment() { 147 | let exp = expectation(description: "Get no error") 148 | getFirstPost() { (post) in // Will fail if the top post has no comments... 149 | HNScraper.shared.getComments(ForPost: post!) { (post, comments, error) in 150 | XCTAssertNil(error, "getComments methdod probably broken") 151 | XCTAssertGreaterThan(comments.count, 0) 152 | HNScraper.shared.upvote(Comment: comments[0], completion: { (error) in 153 | XCTAssertNil(error) 154 | HNScraper.shared.getComments(ForPost: post) { (post, comments, error) in 155 | XCTAssertNil(error) 156 | XCTAssertGreaterThan(comments.count, 0) 157 | XCTAssertTrue(comments[0].upvoted) 158 | exp.fulfill() 159 | } 160 | 161 | 162 | }) 163 | } 164 | } 165 | 166 | wait(for: [exp], timeout: 2*HNScraperTest.defaultTimeOut) 167 | } 168 | 169 | // TODO 170 | /// tests that the favorited attribute is correctly filled when parsing a post from the home page. 171 | /*func testFavoritedAttribute() { 172 | let exp = expectation(description: "the retrieved post has favorited=true") 173 | getFirstPost(completion: {(post) -> Void in 174 | HNScraper.shared.favorite(Post: post!, completion: {(error) -> Void in 175 | XCTAssertNil(error) 176 | self.getFirstPost(completion: { (post) in 177 | XCTAssertNotNil(post?.favorited) 178 | XCTAssertTrue((post?.favorited)!) 179 | exp.fulfill() 180 | }) 181 | 182 | }) 183 | }) 184 | wait(for: [exp], timeout: 2*HNScraperTest.defaultTimeOut) 185 | }*/ 186 | 187 | func testHasLoggedInUserVotedOnPost() { 188 | 189 | } 190 | 191 | // Same test as in HNScraperTest, but at some point, there was a parsing error when the user was logged in, so I added this test here. 192 | func testGetUser() { 193 | let exp = expectation(description: "get a entirely filled HNUser instance") 194 | HNScraper.shared.getUserFrom(Username: HNScraperTest.validFilledUsername, completion: { (user, error) in 195 | XCTAssertEqual(user?.username, HNScraperTest.validFilledUsername) 196 | XCTAssertEqual(String(describing: user!.age!).prefix(7), "2010-08") 197 | XCTAssertNotEqual(user?.karma, 0) 198 | XCTAssertNotNil(user?.aboutInfo) 199 | XCTAssertNotEqual(user!.aboutInfo!, "") 200 | exp.fulfill() 201 | }) 202 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 203 | } 204 | } 205 | class HNScraperTest: XCTestCase { 206 | static let defaultTimeOut: TimeInterval = 10 207 | static let validFilledUsername = "kposehn" // Chose him randomly 208 | static let invalidUsername = "ToBeOrNotToBeSureThatNoOneHasThatUsername" // *Resisting to the urge to create a new account with that username just to mess with these tests.* 209 | static let validCredential = ["username": "abdurhtl", "password": "!Bullshit?Psw$"] 210 | 211 | static let validPostId = "15331016" 212 | override func setUp() { 213 | super.setUp() 214 | // Put setup code here. This method is called before the invocation of each test method in the class. 215 | } 216 | 217 | override func tearDown() { 218 | // Put teardown code here. This method is called after the invocation of each test method in the class. 219 | super.tearDown() 220 | } 221 | 222 | 223 | 224 | func testGetUserByWrongUsername() { 225 | let exp = expectation(description: "Returns noSuchUser error") 226 | HNScraper.shared.getUserFrom(Username: HNScraperTest.invalidUsername, completion: {(user, error) -> Void in 227 | 228 | XCTAssertNil(user) 229 | XCTAssertEqual(error, HNScraper.HNScraperError.noSuchUser) 230 | exp.fulfill() 231 | 232 | }) 233 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 234 | } 235 | 236 | func testGetHomePage() { 237 | let exp = expectation(description: "get 30 items") 238 | HNScraper.shared.getPostsList(page: .news, completion: {(posts, linkForMore, error) -> Void in 239 | XCTAssertEqual(posts.count, 30) 240 | XCTAssertNotNil(linkForMore) 241 | XCTAssertNil(error) 242 | exp.fulfill() 243 | }) 244 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 245 | } 246 | 247 | func testGetAskHN() { // TODO: test askHN comment parsing 248 | let exp = expectation(description: "get 30 items") 249 | HNScraper.shared.getComments(ByPostId: "15465252") { (post, comments, error) in 250 | XCTAssertNil(error) 251 | XCTAssertEqual(comments.count, 1) 252 | XCTAssertGreaterThan(comments[0].text.count, 0) 253 | XCTAssertGreaterThan(comments[0].username.count, 0) 254 | XCTAssertGreaterThan((comments[0].replies[0] as! HNComment).text.count, 0) 255 | XCTAssertGreaterThan((comments[0].replies[0] as! HNComment).username.count, 0) 256 | exp.fulfill() 257 | } 258 | 259 | wait(for: [exp], timeout: 100*HNScraperTest.defaultTimeOut) 260 | } 261 | 262 | func testGet90ItemsFromHomePage() { 263 | let exp = expectation(description: "get 90 items") 264 | HNScraper.shared.getPostsList(page: .news) { (posts, linkForMore, error) in 265 | XCTAssertEqual(posts.count, 30, "the getPostsList method is probably broken") 266 | XCTAssertNotNil(linkForMore, "the getPostsList method is probably broken") 267 | XCTAssertNil(error, "the getPostsList method is probably broken") 268 | HNScraper.shared.getMoreItems(linkForMore: linkForMore!, completionHandler: { (posts, linkForMore, error) in 269 | XCTAssertEqual(posts.count, 30) 270 | XCTAssertNotNil(linkForMore) 271 | XCTAssertNil(error) 272 | HNScraper.shared.getMoreItems(linkForMore: linkForMore!, completionHandler: { (posts, linkForMore, error) in 273 | XCTAssertEqual(posts.count, 30) 274 | XCTAssertNotNil(linkForMore) 275 | XCTAssertNil(error) 276 | exp.fulfill() 277 | }) 278 | }) 279 | } 280 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 281 | } 282 | 283 | func testGetSubmissionOfNonExistingUser() { 284 | let exp = expectation(description: "get noSuchUser error") 285 | HNScraper.shared.getSubmissions(ForUserWithUsername: HNScraperTest.invalidUsername, completion: {(posts, linkForMore, error) -> Void in 286 | XCTAssertEqual(posts.count, 0) 287 | XCTAssertNil(linkForMore) 288 | XCTAssertEqual(error, HNScraper.HNScraperError.noSuchUser) 289 | exp.fulfill() 290 | }) 291 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 292 | } 293 | 294 | func testGetSubmissions() { 295 | let exp = expectation(description: "get some items") 296 | HNScraper.shared.getSubmissions(ForUserWithUsername: HNScraperTest.validFilledUsername, completion: {(posts, linkForMore, error) -> Void in 297 | XCTAssertEqual(posts.count, 30) 298 | XCTAssertNotNil(linkForMore) 299 | XCTAssertNil(error) 300 | exp.fulfill() 301 | }) 302 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 303 | } 304 | func testGetEmptySubmissionList() { 305 | 306 | let exp = expectation(description: "get 0 items") 307 | HNScraper.shared.getSubmissions(ForUserWithUsername: HNScraperTest.validCredential["username"]!, completion: {(posts, linkForMore, error) -> Void in 308 | XCTAssertEqual(posts.count, 0) 309 | XCTAssertNil(linkForMore) 310 | XCTAssertNil(error) 311 | exp.fulfill() 312 | }) 313 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 314 | } 315 | 316 | func testGetUser() { 317 | let exp = expectation(description: "get a entirely filled HNUser instance") 318 | HNLogin.shared.logout() 319 | HNScraper.shared.getUserFrom(Username: HNScraperTest.validFilledUsername, completion: { (user, error) in 320 | XCTAssertEqual(user?.username, HNScraperTest.validFilledUsername) 321 | XCTAssertEqual(String(String(describing: user!.age!).prefix(7)), "2010-08") 322 | XCTAssertNotEqual(user?.karma, 0) 323 | XCTAssertNotNil(user?.aboutInfo) 324 | XCTAssertNotEqual(user!.aboutInfo!, "") 325 | exp.fulfill() 326 | }) 327 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 328 | } 329 | 330 | func testUpvoteWithoutLogin() { 331 | let exp = expectation(description: "get notLoggedIn error") 332 | HNScraper.shared.getPostsList(page: .news, completion: {(posts, linkForMore, error) -> Void in 333 | if posts.count == 0 { 334 | XCTFail("The getPostLists method is probably broken. Or the hackernews is down...") 335 | exp.fulfill() 336 | } 337 | let postToUpvote = posts[0] // first post of the home page 338 | HNScraper.shared.upvote(Post: postToUpvote, completion: {(error) -> Void in 339 | XCTAssertEqual(error, .notLoggedIn) 340 | exp.fulfill() 341 | }) 342 | }) 343 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 344 | } 345 | 346 | 347 | func testGetCommentForValidPostId() { 348 | let exp = expectation(description: "Get some comments") 349 | HNScraper.shared.getComments(ByPostId: HNScraperTest.validPostId) { (post, comments, error) in 350 | XCTAssertNil(error) 351 | XCTAssertGreaterThan(comments.count, 0) 352 | XCTAssertGreaterThan(comments[0].text.count, 0) 353 | XCTAssertGreaterThan(comments[0].username.count, 0) 354 | exp.fulfill() 355 | } 356 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 357 | } 358 | 359 | func testGetCommentsForBadPostId() { 360 | let exp = expectation(description: "Get noSuchPost error") 361 | HNScraper.shared.getComments(ByPostId: "whatpostId") { (post, comments, error) in 362 | XCTAssertEqual(error, .noSuchPost) 363 | exp.fulfill() 364 | } 365 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 366 | } 367 | 368 | func testGetCommentsForUser() { 369 | let exp = expectation(description: "Get some comments with parentId filled") 370 | HNScraper.shared.getComments(ForUserWithUsername: "yoda_sl") { (comments, linkForMore, error) in 371 | XCTAssertNil(error) 372 | XCTAssertGreaterThan(comments.count, 0) 373 | XCTAssertNotNil(linkForMore) 374 | XCTAssertNotEqual(comments[0].parentId, "") 375 | XCTAssertGreaterThan(comments[0].text.count, 0) 376 | XCTAssertGreaterThan(comments[0].username.count, 0) 377 | exp.fulfill() 378 | } 379 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 380 | } 381 | 382 | func testGetMoreComments() { 383 | let exp = expectation(description: "Get some comments with parentId filled") 384 | HNScraper.shared.getComments(ForUserWithUsername: "yoda_sl") { (comments, linkForMore, error) in 385 | XCTAssertNil(error) 386 | XCTAssertGreaterThan(comments.count, 0) 387 | XCTAssertNotNil(linkForMore) 388 | HNScraper.shared.getMoreComments(linkForMore: linkForMore!, completionHandler: { (comments, linkForMore, error) in 389 | XCTAssertNil(error) 390 | XCTAssertGreaterThan(comments.count, 0) 391 | XCTAssertNotNil(linkForMore) 392 | exp.fulfill() 393 | }) 394 | 395 | } 396 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 397 | } 398 | } 399 | -------------------------------------------------------------------------------- /HNScraperTests/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleDevelopmentRegion 6 | $(DEVELOPMENT_LANGUAGE) 7 | CFBundleExecutable 8 | $(EXECUTABLE_NAME) 9 | CFBundleIdentifier 10 | $(PRODUCT_BUNDLE_IDENTIFIER) 11 | CFBundleInfoDictionaryVersion 12 | 6.0 13 | CFBundleName 14 | $(PRODUCT_NAME) 15 | CFBundlePackageType 16 | BNDL 17 | CFBundleShortVersionString 18 | 1.0 19 | CFBundleVersion 20 | 1 21 | 22 | 23 | -------------------------------------------------------------------------------- /HNScraperTests/RessourceFetcherTest.swift: -------------------------------------------------------------------------------- 1 | // 2 | // RessourceFetcherTest.swift 3 | // HNScraperTests 4 | // 5 | // Created by Stéphane Sercu on 29/09/17. 6 | // Copyright © 2017 Stéphane Sercu. All rights reserved. 7 | // 8 | 9 | import XCTest 10 | @testable import HNScraper 11 | class RessourceFetcherTest: XCTestCase { 12 | 13 | override func setUp() { 14 | super.setUp() 15 | // Put setup code here. This method is called before the invocation of each test method in the class. 16 | } 17 | 18 | override func tearDown() { 19 | // Put teardown code here. This method is called after the invocation of each test method in the class. 20 | super.tearDown() 21 | } 22 | 23 | func testGetJson() { 24 | let exp = expectation(description: "No error & valid parsed data") 25 | RessourceFetcher.shared.getJson(url: "https://httpbin.org/headers") { (json, error) in 26 | XCTAssertNil(error) 27 | XCTAssertNotNil(json) 28 | XCTAssertNotNil(json!["headers"] as? [String: String]) 29 | exp.fulfill() 30 | } 31 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 32 | } 33 | func testPostRequest() { 34 | let exp = expectation(description: "Get the post data back as response in json format") 35 | let bodyData = "attr1=val1&attr2=val2".data(using: .utf8) 36 | let cookie = HTTPCookie(properties: [.value:"value", .name:"name", .domain:"httpbin.org", .path:"."]) 37 | 38 | RessourceFetcher.shared.post(urlString: "https://httpbin.org/post", data: bodyData!, cookies: [cookie!]) { (data, response, error) in 39 | XCTAssertNil(error) 40 | XCTAssertNotNil(response) 41 | XCTAssertNotNil(data) 42 | let json:[String: Any]? = try? JSONSerialization.jsonObject(with: data!, options: .mutableContainers) as! [String: Any] 43 | XCTAssertNotNil(json ?? nil) 44 | XCTAssertEqual((json!["form"] as! [String: String])["attr1"], "val1") 45 | XCTAssertEqual((json!["form"] as! [String: String])["attr2"], "val2") 46 | XCTAssertEqual(((json!["headers"] as! [String:String])["Cookie"]), "name=value") 47 | exp.fulfill() 48 | } 49 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 50 | 51 | } 52 | 53 | func testGetRequest() { 54 | let expWithoutCookie = expectation(description: "Get the post data back as response in json format") 55 | let expWithCookie = expectation(description: "Get the post data back as response in json format (containing a cookies field)") 56 | let cookie = HTTPCookie(properties: [.value:"value", .name:"name", .domain:"httpbin.org", .path:"."]) 57 | RessourceFetcher.shared.get(urlString: "https://httpbin.org/cookies", cookies: [cookie!]) { (data, response, error) in 58 | XCTAssertNil(error) 59 | XCTAssertNotNil(response) 60 | XCTAssertNotNil(data) 61 | let json:[String: Any]? = try? JSONSerialization.jsonObject(with: data!, options: .mutableContainers) as! [String: Any] 62 | XCTAssertNotNil(json ?? nil) 63 | XCTAssertEqual(json!["cookies"] as! [String:String], ["name":"value"]) 64 | expWithCookie.fulfill() 65 | } 66 | RessourceFetcher.shared.get(urlString: "https://httpbin.org/cookies") { (data, response, error) in 67 | XCTAssertNil(error) 68 | XCTAssertNotNil(response) 69 | XCTAssertNotNil(data) 70 | let json:[String: Any]? = try? JSONSerialization.jsonObject(with: data!, options: .mutableContainers) as! [String: Any] 71 | XCTAssertNotNil(json ?? nil) 72 | expWithoutCookie.fulfill() 73 | } 74 | wait(for: [expWithoutCookie, expWithCookie], timeout: HNScraperTest.defaultTimeOut) 75 | 76 | } 77 | 78 | func testBadGetRequest() { 79 | let exp = expectation(description: "Get a invalidURL error") 80 | RessourceFetcher.shared.get(urlString: "where?") { (data, response, error) in 81 | XCTAssertEqual(error, .invalidURL) 82 | exp.fulfill() 83 | } 84 | wait(for: [exp], timeout: HNScraperTest.defaultTimeOut) 85 | 86 | } 87 | 88 | 89 | 90 | } 91 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2017-2018 by Stéphane Sercu 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HNScraper 2 | 3 |

4 | 5 | 6 | 7 | 8 | 9 | License 10 | 11 |

12 | 13 | 14 | HNScraper is a scraping library for hackernews, written in swift. It allows you to fetch all the stories, comments and user informations directly from the website. It also supports logging in and vote on/favorite posts and comments. 15 | 16 | This library is similar to the [LibHN](https://github.com/bennyguitar/libHN) library. Actually, some parts of the project, such as the post, comment and user models and the parsing rules are basically a swift translation of the LibHN library. 17 | 18 | 19 | Before going further, note that there is an [official API](https://github.com/HackerNews/API) that implements the fundamentals features such as grabbing the new, top and best stories, user informations and comments. However, it doesn't support login and any related functionalities and suffers from a lot of restrictions. If it fits your needs though, I would recommend you use the official API over this scraper. 20 | 21 | 22 | ## Table of content 23 | 24 | * [Installation](#installation) 25 | * [Documentation](#documentation) 26 | * [Completion handlers & error gesture](#completion-handlers-&-error-gesture) 27 | * [Fetch list of posts](#fetch-list-of-posts) 28 | * [Fetch comments about a post](#fetch-comments-about-a-post) 29 | * [Fetch informations about a user](#fetch-informations-about-a-user) 30 | * [Fetch the submissions, comments and favorites of a user](#fetch-the-submissions,-comments-and-favorites-of-a-user) 31 | * [Login / Logout](#login-/-logout) 32 | * [Vote on posts/comments](#vote-on-posts/comments) 33 | * [Favorite a post](#favorite-a-post) 34 | * [Structure of the project](#structure-of-the-project) 35 | * [Contribution](#contribution) 36 | * [TODO](#todo) 37 | * [License](#license) 38 | 39 | 40 | ## Installation 41 | 42 | #### Pod 43 | 44 | Ensure you have at least the following code in your `Podfile`: 45 | 46 | ``` 47 | use_frameworks! 48 | 49 | target 'YourAppName' do 50 | pod 'HNScraper', '~> 0.2.1' 51 | end 52 | ``` 53 | 54 | Run `pod install` in your project's folder. 55 | 56 | Then just add `import HNScraper` wherever you need the scraper. 57 | 58 | 59 | #### Manually 60 | 61 | Just add all the `.swift` files from the `HNScraper` folder in your project. 62 | 63 | ## Documentation 64 | 65 | ### Completion handlers & error gesture 66 | All the following actions are performed in the same way: you call a method on the right instance with the required parameters and a completion handler which will give you back the results of your request and, eventually, the produced error. 67 | 68 | ```swift 69 | func getSomething(dependingOn: , completion: ((results?, error?) -> Void)) 70 | ``` 71 | 72 | The possible errors are defined by `HNScrapperError ` and `HNLoginError `. Those structures simplifies the error handling by classifying the most recurrent errors into distinct, self-explanatory error types. This abstraction allows you to handle common problems, such as no Internet connection, bad credentials, non-existing post-id, etc, without worrying about "low level" errors such as URLErrors and JSON errors. 73 | 74 | 75 | ### Fetch list of posts 76 | A list of posts is any of the HN pages defined in `HNScraper.PostListPageName` 77 | 78 | [`news`](https://news.ycombinator.com/news) 79 | [`front`](https://news.ycombinator.com/front) 80 | [`new`](https://news.ycombinator.com/newest) 81 | [`jobs`](https://news.ycombinator.com/jobs) 82 | [`asks`](https://news.ycombinator.com/ask) 83 | [`shows`](https://news.ycombinator.com/show) 84 | [`newshows`](https://news.ycombinator.com/shownew) 85 | [`active`](https://news.ycombinator.com/active) 86 | [`best`](https://news.ycombinator.com/best) 87 | [`noob`](https://news.ycombinator.com/noobstories) 88 | 89 | To scrap the 30 first posts of one of them, you have to use `getPostsList` (from `HNScraper`) which will give you an array of `HNPost` objects and the link to the next page that you can use to fetch the 30 following items. 90 | 91 | ```swift 92 | typealias PostListDownloadCompletionHandler = (([HNPost], String?, HNScrapperError?) -> Void) 93 | 94 | func getPostsList(page: PostListPageName, completion: PostListDownloadCompletionHandler) 95 | ``` 96 | 97 | For example: 98 | 99 | ```swift 100 | HNScraper.shared.getPostsList(page: .news) { (posts, linkForMore, error) in 101 | // Don't forget to handle the eventual error 102 | for post in posts { 103 | print(post.title) 104 | } 105 | // You also may want to save the linkForMore somewhere. 106 | } 107 | ``` 108 | 109 | For the 30+ items, you have to use `getMoreItems` at which you pass the "link for more" you got with the 30 first items. This will also give you a list of `HNPost` instances and a link for the next page: 110 | 111 | ```swift 112 | func getMoreItems(linkForMore: String, completionHandler: PostListDownloadCompletionHandler) 113 | ``` 114 | 115 | For example: 116 | 117 | ```swift 118 | HNScraper.shared.getMoreItems(linkForMore: "s") { (posts, linkForMore, error) in 119 | // do whatever you want with the stories 120 | } 121 | 122 | ``` 123 | 124 | 125 | 126 | 127 | 128 | ### Fetch comments about a post 129 | The comments are parsed from a discussion thread (at `news.ycombinator.com/item?id=`*``*). You can fetch those in 2 ways: either with 130 | 131 | ```swift 132 | func getComments(ByPostId postId: String, buildHierarchy: Bool = true, completion: @escaping ((HNPost?, [HNComment], HNScraperError?) -> Void)) 133 | ``` 134 | 135 | or with 136 | 137 | ```swift 138 | func getComments(ForPost post: HNPost, buildHierarchy: Bool = true, completion: @escaping ((HNPost, [HNComment], HNScraperError?) -> Void)) 139 | ``` 140 | 141 | 142 | The parameter `buildHierarchy` indicates if the comments have to be returned in nested (meaning that only the root comments are in the resulting array and they are pointing to their replies) or in linear (flat) format. 143 | 144 | With the `ByPostId` method, the data about the post itself will be parsed to build a `HNPost` object that is passed to the completion closure. 145 | 146 | With the `ForPost`method, the `HNPost` instance given to the completion closure is the same (unmodified) post you passed to the `getComments`method. 147 | 148 | 149 | For a `askHN` type of posts, the first comment is the OP's ask itself. 150 | 151 | As for `job` type of posts, there should be no comments. 152 | 153 | ### Fetch informations about a user 154 | You can get the karma, description and age of any user by giving its username to the `getUser`method: 155 | 156 | ```swift 157 | func getUserFrom(Username username: String, completion: ((HNUser?, HNScraperError?) -> Void)?) 158 | ``` 159 | 160 | ### Fetch the submissions, comments and favorites of a user 161 | 162 | Use the following methods (from `HNScraper`) according to which list you want to grab: 163 | 164 | ```swift 165 | func getFavorites(ForUserWithUsername username: String, completion: @escaping PostListDownloadCompletionHandler) 166 | 167 | func getSubmissions(ForUserWithUsername username: String, completion: PostListDownloadCompletionHandler) 168 | ``` 169 | 170 | 171 | In the same way as for the list of post described earlier, the completion closure will give you a "link for more" that you can use to fetch more items (in the case there are more than 30 items to fetch of course). You can use the `getMoreItems(linkForMore: completionHandler:)` method as earlier. 172 | 173 | 174 | 175 | ### Login / Logout 176 | 177 | Those actions are handled by the singleton class `HNLogin`. 178 | 179 | 180 | You can login with 181 | 182 | ```swift 183 | func login(username: String, psw: String, completion: @escaping ((HNUser?, HTTPCookie?, HNLoginError?) -> Void)) 184 | ``` 185 | 186 | 187 | Once a user has logged in, the `HNLogin`class takes care to store the session cookie and make it available to the other classes. In addition, with that cookie saved, the html retrieved from the website by any of the requests made by the scrapper will be as if the user was logged in. Which means that it will contain all the upvote (and eventual downvote, if the user has more than 500 points) links, favorite links, comment links, etc. 188 | 189 | 190 | The `HNLogin`class comes with its own error enum, `HNLoginError` , which contains the `badCredentials` case. It's returned as an error in the case of, ... well, wrong credentials. 191 | 192 | Example: 193 | 194 | ```swift 195 | HNLogin.shared.login(username: "username", psw: "pass") { (user, cookie, error) in 196 | if let connected_user = user { 197 | print("logged in user: " + connected_user.username) 198 | } else { 199 | // Handle error 200 | if error == .badCredentials { 201 | print("wrong creds") 202 | } else { 203 | // Check other types of error 204 | } 205 | } 206 | } 207 | ``` 208 | 209 | You can logout a user by calling 210 | 211 | ```swift 212 | func logout() 213 | ``` 214 | 215 | All this method does is delete the stored session cookie, which make the retrieved html from the HN website looking as it would to an unsigned visitor. 216 | 217 | ### Up/Down/Un vote a post/comment 218 | 219 | Simply use one of the following methods: 220 | 221 | ```swift 222 | func upvote(Comment comment: HNComment, completion: ((HNScraperError?) -> Void)) 223 | func upvote(Post post: HNPost, completion: ((HNScraperError?) -> Void)) 224 | func unvote(Post post: HNPost, completion: ((HNScraperError?) -> Void)) 225 | func unvote(Comment comment: HNComment, completion: ((HNScraperError?) -> Void)) 226 | ``` 227 | 228 | The user obviously needs to be logged in to do that. Otherwise, an error of type `.notLoggedIn` is passed to the completion closure. 229 | 230 | If the `error` parameter of the completion closure is `nil`, then the action was succesfull. Otherwise there was a problem. 231 | 232 | **Note**: an action is considered *succesfull* when the final state of the item is the one itended by the request. So if you try to unvote a post that hasn't been upvoted, there will be no error. Same if you try to upvote an upvoted item. 233 | 234 | 235 | 236 | ### (Un)Favorite a post 237 | 238 | Use: 239 | 240 | ```swift 241 | func favorite(Post post: HNPost, completion: ((HNScraperError?) -> Void)) 242 | func unfavorite(Post post: HNPost, completion: ((HNScraperError?) -> Void)) 243 | 244 | ``` 245 | 246 | 247 | Those methods works in the same ways as the ones for voting on items. 248 | 249 | 250 | Again, the user needs to be logged in, otherwise an error of type `.notLoggedIn` is passed to the completion closure. 251 | 252 | ## Structure of the project 253 | 254 | ### Models 255 | 256 | The scraper uses 3 models: 257 | 258 | * `HNPost` 259 | * `HNComment` 260 | * `HNUser` 261 | 262 | ### Endpoints 263 | 264 | There are basically 2 singleton classes that you'll use to make requests: 265 | 266 | - **`HNScraper`** 267 | - **`HNLogin`** 268 | 269 | ### Parsing configuration file 270 | 271 | `hn.json` contains most of the informations needed to parse every HN pages. It was introduced in LibHN to 272 | 273 | Although some things changed/have been added in it, its structure is the same as in LibHN. The following section is an update of the original version of the LibHN documentation. 274 | 275 | 276 | This file is downloaded and stored by the singleton class `HNParseConfig`. 277 | 278 | 279 | ### Tests 280 | 281 | There are tests for most of the methods in the `HNScraperTests` folder. 282 | 283 | 284 | # Contribution 285 | 286 | Contribution of any kind is welcome. 287 | 288 | If you spot an error, you think of an amelioration, you have a suggestion or you g just open an issue or post directly a PR. 289 | 290 | Also, I'm not a native English speaker, so don't hesitate to correct some of my sentences :) 291 | 292 | # TODO 293 | 294 | - Complete the hn.json config file with the rest of hardcoded strings needed for parsing 295 | - Submit story 296 | - Post comments 297 | - Downvote 298 | - Edit account (about, options, mail, ...) 299 | - Search 300 | - Test for mac os apps 301 | 302 | 303 | # License 304 | 305 | HNScraper is licensed under the standard MIT License. 306 | 307 | **Copyright (C) 2017-2018 by Stéphane Sercu** 308 | 309 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 310 | 311 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 312 | 313 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 314 | -------------------------------------------------------------------------------- /hn.json: -------------------------------------------------------------------------------- 1 | { 2 | "Post": { 3 | "CS": "", 4 | "Vote": { 5 | "R": "votearrow", 6 | "S": "href='", 7 | "E": "'" 8 | }, 9 | "Parts": [{ 10 | "S": "", 15 | "E": "<", 16 | "I": "Title" 17 | }, { 18 | "S": "score_", 19 | "E": "\"", 20 | "I": "TRASH" 21 | }, { 22 | "S": ">", 23 | "E": "", 24 | "I": "Points" 25 | }, { 26 | "S": "", 35 | "E": "", 39 | "E": "item?id", 40 | "I": "TRASH" 41 | }, { 42 | "S": "=", 43 | "E": "\"", 44 | "I": "TRASH" 45 | }, { 46 | "S": ">", 47 | "E": "<", 48 | "I": "Comments" 49 | }], 50 | "LinkForMore": { 51 | "S": "" 84 | }, 85 | "Upvote": { 86 | "R": "vote(event, this, \"up\")", 87 | "S": "href='", 88 | "E": "'" 89 | }, 90 | "Downvote": { 91 | "R": "dir=down", 92 | "S": "href=\"", 93 | "E": "\">" 94 | }, 95 | "Level": { 96 | "S": "height=\"1\" width=\"", 97 | "E": "\">", 98 | "I": "Level" 99 | }, 100 | "ParentPostId": { 101 | "S": "on: ", 103 | "I": "ParentPostId" 104 | }, 105 | "ASK": [{ 106 | "S": "class=\"storylink\">", 107 | "E": "", 108 | "I": "Title" 109 | }, { 110 | "S": "class=\"hnuser\">", 111 | "E": "", 112 | "I": "Username" 113 | }, { 114 | "S": "", 115 | "E": "", 119 | "E": "", 120 | "I": "Time" 121 | }, { 122 | "S": "href=\"item?id=", 123 | "E": "\"", 124 | "I": "CommentId" 125 | }, { 126 | "S": "\n ", 127 | "E": "\n \n ", 132 | "E": "", 133 | "I": "Text" 134 | }], 135 | "REG": [{ 136 | "S": "class=\"hnuser\">", 137 | "E": "", 138 | "I": "Username" 139 | }, { 140 | "S": "item?id=", 141 | "E": "\"", 142 | "I": "CommentId" 143 | }, { 144 | "S": ">", 145 | "E": "<", 146 | "I": "Time" 147 | }, { 148 | "S": "class=\"comment\">", 149 | "E": "class=\"c", 150 | "I": "TRASH" 151 | }, { 152 | "S": "\">", 153 | "E": "", 154 | "I": "Text" 155 | }, { 156 | "S": "href=\"", 157 | "E": "\"", 158 | "I": "ReplyUrl" 159 | }], 160 | "LinkForMore": { 161 | "S": "", 170 | "I": "parent" 171 | }, { 172 | "S": "name=\"hmac\" value=\"", 173 | "E": "\">", 174 | "I": "hmac" 175 | }] 176 | }, 177 | "Submit": { 178 | "Action": "r", 179 | "Parts": [{ 180 | "S": "name=\"fnid\" value=\"", 181 | "E": "\">", 182 | "I": "fnid" 183 | }], 184 | "Url": "url", 185 | "Title": "title", 186 | "Text": "text" 187 | }, 188 | "User": { 189 | "Parts": [{ 190 | "S": "class=\"hnuser\">", 191 | "E": "", 192 | "I": "user" 193 | }, { 194 | "S": "href=\"front?day=", 195 | "E": "&birth", 196 | "I": "created" 197 | }, { 198 | "S": "karma:", 199 | "E": "", 200 | "I": "karma" 201 | }, { 202 | "S": "name=\"about\">", 203 | "E": "", 204 | "I": "about" 205 | }, { 206 | "S": "about:\n", 207 | "E": "", 208 | "I": "about" 209 | }, { 210 | "S": "created:", 211 | "E": " ", 212 | "I": "age" 213 | }] 214 | }, 215 | "ParsingKeys": { 216 | "UserCreationDateFormat": "yyyy-MM-dd" 217 | } 218 | } --------------------------------------------------------------------------------