├── LICENSE
├── README.md
├── davereader.js
├── examples
    ├── feedFiler
    │   ├── README.md
    │   ├── filer.js
    │   └── package.json
    └── river5
    │   ├── config.json
    │   ├── lists
    │       └── myopmlfeeds.opml
    │   ├── package.json
    │   └── river5.js
└── package.json


/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2017 Dave Winer
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | ## davereader package
  2 | 
  3 | It's the core of River5, without the top level, released as an NPM package. Can be used to build all kinds of feed-based apps.
  4 | 
  5 | ### How to install
  6 | 
  7 | `npm install davereader`
  8 | 
  9 | ### Story
 10 | 
 11 | Here's the story.
 12 | 
 13 | I've always wanted to have a package that made it possible to write quick apps that do stuff with RSS.
 14 | 
 15 | I write them all the time, but I always have to start over from the beginning, create a parser, then catch the new items as they come in, and do whatever it is I have to do, usually move the bits to another service like Twitter, or Slack or whatever. There are so many possible applications. 
 16 | 
 17 | When I was doing this, I realized I was solving a problem that was already solved, in my River software, but it wasn't configured correctly to make this easy. It was faster just to crib the code and start from scratch.
 18 | 
 19 | Finally, I have it set up so that this works. So the beauty in this is in the apps, not the engine. It's a solved problem that can now be used to solve new problems. 
 20 | 
 21 | ### Hello world
 22 | 
 23 | The Hello World app is <a href="https://github.com/scripting/reader/tree/master/examples/feedFiler">feedFiler</a>. It simply writes every new story, in JSON, to a calendar-structured folder. 
 24 | 
 25 | Not very useful on its own, but Hello World apps aren't supposed to be useful. ;-)
 26 | 
 27 | ### River5 is a demo app
 28 | 
 29 | River5 is even smaller than feedFiler!
 30 | 
 31 | It's included as <a href="https://github.com/scripting/reader/tree/master/examples/river5">an example</a>. 
 32 | 
 33 | ### Updates
 34 | 
 35 | #### v0.6.7 -- 4/8/18 by DW
 36 | 
 37 | Fixed a bug where if you unsubbed from a feed, River5 could crash when rebuilding a river that contains items from that feed. 
 38 | 
 39 | #### v0.6.6 -- 4/8/18 by DW
 40 | 
 41 | Feed reading is now charset-aware. If you were seeing garbled text in feeds in German, for example, this version should fix that.  
 42 | 
 43 | To update, quit the app, type <i>npm update</i> at the command line, then restart the app. 
 44 | 
 45 | #### v0.6.2 -- 1/17/18 by DW
 46 | 
 47 | It's been a while since there was an update so I bumped from 0.5.x to 0.6.x. 
 48 | 
 49 | A bug was <a href="https://github.com/scripting/river5/issues/14">reported</a>, that the first time new items appear in a river in a given River5 run, only one of the items actually appears in the river. This applies to both kinds of rivers, ones that correspond to lists, and ones that accumulate all the items in a given river (a feature that appeared first in v0.5.21). The other new items are lost. I was able to reproduce the problem, and spent a <a href="http://scripting.com/2018/01/16/180415.html">few</a> <a href="http://scripting.com/2018/01/15/234534.html">days</a> <a href="http://scripting.com/2018/01/14/155244.html">discussing</a> <a href="http://scripting.com/2018/01/13/183433.html">possible</a> solutions, and then arrived at a very simple approach, that appears to work.
 50 | 
 51 | We do two things differently:
 52 | 
 53 | 1. At startup, before reading any feeds, we load all the river files for all the rivers that correspond to lists. 
 54 | 
 55 | 2. When reading a feed, we no longer process items as they are returned by <a href="https://github.com/danmactough/node-feedparser">feedparser</a>, we accumulate all the items in an array, and process them all at the end. Before processing we make sure the feed's river is in the cache, and read it if it's not. That way all the new items make it into the river. 
 56 | 
 57 | #### v0.5.22 -- 8/18/17 by DW
 58 | 
 59 | There were a couple of places where we would read a feed even if no one was subscribed to it. This created problems in davecast when I wanted to unfollow Scripting News. I imagine it's an annoyance for some when they unsub from a feed only to have updates still show up in the river. It should stop when the rssCloud pings stop, though. The right thing to do (which we now do) is to check if there's at least one subscriber before reading the feed. 
 60 | 
 61 | #### v0.5.21 -- 7/10/17 by DW
 62 | 
 63 | The new single-feed viewer feature is now available for all River5 installations. To activate the feature, update your installation to v0.5.21. When you access the home page of your server through the web, when you click on the favicon of a feed, it will take you to the single-feed viewer with the items for that feed. 
 64 | 
 65 | Note that the only change in this release is that there's a new element in config, urlFeedViewerApp. It's picked up in the home page app in constructing the link for the favicon. 
 66 | 
 67 | In v0.5.19 we added a feature that keeps a river for each feed. That's where the major work was on the server for this feature. 
 68 | 
 69 | #### v0.5.16 -- 6/29/17 by DW
 70 | 
 71 | We now optionally maintain rivers for each feed. This could make a single-feed viewer possible in <a href="http://this.how/electricRiver">Electric River</a> and other environments. That is, all the stories from one feed in reverse-chronologic order.
 72 | 
 73 | #### v0.5.15 -- 6/25/17 by DW
 74 | 
 75 | HTTP requests made by davereader now accept zip compression. 
 76 | 
 77 | #### v0.5.12 -- 6/21/17 by DW
 78 | 
 79 | New callbacks for handling an HTTP request (the host application can override), callbacks that are called every second and every minute. 
 80 | 
 81 | Exposed the davereader function that sends a webSockets message to all registered listeners.
 82 | 
 83 | #### v0.5.5 -- 6/13/17 by DW
 84 | 
 85 | New support for <a href="https://github.com/scripting/Scripting-News/blob/master/rss-in-json/README.md">RSS-in-JSON</a> feeds. 
 86 | 
 87 | #### v0.5.4 -- 6/7/17 by DW
 88 | 
 89 | One small change, the default value of config.flDownloadPodcasts changed from true to false. 
 90 | 
 91 | Previously a new installation would download podcasts automatically, and this could cause problems because they are such large files. 
 92 | 
 93 | Now you have to set this true yourself in config.json.
 94 | 
 95 | #### v0.5.1 -- 5/17/17 by DW
 96 | 
 97 | The version number changed to use the format favored by NPM. There is no way around it if we want to use NPM, and we do. 
 98 | 
 99 | Instead of baking the utils package into the project, we require the new daveutils package. 
100 | 
101 | There's a new optional element in config, config.newItemCallback. It's a function that takes three params, the URL of the feed, the internal struct returned by feedparser and a digested version of that data produced by davereader/river5.
102 | 
103 | ### Questions, support
104 | 
105 | If you have questions or need help, post an <a href="https://github.com/scripting/river5/issues">Issue</a> on the River5 site (historically that's where this work is done) or on the River5 <a href="https://groups.google.com/forum/#!forum/river5">mail list</a>. 
106 | 
107 | 


--------------------------------------------------------------------------------
/davereader.js:
--------------------------------------------------------------------------------
   1 | var myProductName = "River5"; myVersion = "0.6.10"; 
   2 | 
   3 | /*  The MIT License (MIT)
   4 | 	Copyright (c) 2014-2018 Dave Winer
   5 | 	
   6 | 	Permission is hereby granted, free of charge, to any person obtaining a copy
   7 | 	of this software and associated documentation files (the "Software"), to deal
   8 | 	in the Software without restriction, including without limitation the rights
   9 | 	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10 | 	copies of the Software, and to permit persons to whom the Software is
  11 | 	furnished to do so, subject to the following conditions:
  12 | 	
  13 | 	The above copyright notice and this permission notice shall be included in all
  14 | 	copies or substantial portions of the Software.
  15 | 	
  16 | 	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 | 	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 | 	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  19 | 	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20 | 	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21 | 	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  22 | 	SOFTWARE.
  23 | 	*/
  24 | 
  25 | exports.init = init;
  26 | exports.httpRequest = handleHttpRequest; //3/24/17 by DW
  27 | exports.readAllFeedsNow = readAllFeedsNow; //4/18/17 by DW
  28 | exports.notifyWebSocketListeners = notifyWebSocketListeners; //6/20/17 by DW
  29 | 
  30 | const fs = require ("fs");
  31 | const request = require ("request");
  32 | const http = require ("http"); 
  33 | const urlpack = require ("url");
  34 | const md5 = require ("md5");
  35 | const websocket = require ("nodejs-websocket"); 
  36 | const qs = require ("querystring");
  37 | const OpmlParser = require ("opmlparser");
  38 | const FeedParser = require ("feedparser");
  39 | const utils = require ("daveutils");
  40 | const feedRead = require ("davefeedread"); //4/8/18 by DW
  41 | 
  42 | var config = {
  43 | 	enabled: true,
  44 | 	
  45 | 	httpPort: 1337,
  46 | 	flHttpEnabled: true,
  47 | 	webSocketPort: 1338,
  48 | 	flWebSocketEnabled: true,
  49 | 	
  50 | 	dataFolder: "data/",
  51 | 	listsFolder: "lists/",
  52 | 	riversFolder: "rivers/",
  53 | 	podcastsFolder: "podcasts/", //4/17/17 by DW
  54 | 	
  55 | 	localStoragePath: "localStorage.json",
  56 | 	statsFilePath: "serverStats.json",
  57 | 	templatePath: "misc/template.html",
  58 | 	addToRiverCallbacksFolder: "callbacks/addToRiver/",
  59 | 	buildRiverCallbacksFolder: "callbacks/buildRiver/", //4/23/17 by DW
  60 | 	
  61 | 	riverDataFileName: "riverData.json",
  62 | 	listInfoFileName: "listInfo.json",
  63 | 	
  64 | 	flAddItemsFromNewSubs: true,
  65 | 	maxRiverItems: 250,
  66 | 	maxBodyLength: 280,
  67 | 	flSkipDuplicateTitles: true,
  68 | 	flRequestCloudNotify: true, 
  69 | 	flMaintainCalendarStructure: false,
  70 | 	flWriteItemsToFiles: false,
  71 | 	ctMinutesBetwBuilds: 15,
  72 | 	maxConcurrentFileWrites: 100,
  73 | 	remotePassword: "",
  74 | 	
  75 | 	flWatchAppDateChange: false,
  76 | 	fnameApp: "lib/feedtools.js",
  77 | 	
  78 | 	
  79 | 	urlServerHomePageSource: "https://s3.amazonaws.com/scripting.com/code/river5/serverhomepage.html", 
  80 | 	urlDashboardSource: "https://s3.amazonaws.com/scripting.com/code/river5/dashboard.html",
  81 | 	urlFeedViewerApp: "https://cdn.jsdelivr.net/npm/river5/includes/feedviewer/index.html", //7/10/17 by DW
  82 | 	urlFavicon: "https://cdn.jsdelivr.net/npm/river5/includes/misc/favicon.ico",
  83 | 	flIntegratedFeedViewer: true, //4/14/18 by DW
  84 | 	
  85 | 	notifyListenersCallback: undefined, //3/25/17 by DW
  86 | 	statsChangedCallback: undefined, //3/25/17 by DW
  87 | 	consoleLogCallback: undefined, //3/28/17 by DW
  88 | 	newItemCallback: undefined, //5/17/17 by DW
  89 | 	handleHttpRequestCallback: undefined, //6/20/17 by DW
  90 | 	everyMinuteCallback: undefined, //6/20/17 by DW
  91 | 	everySecondCallback: undefined, //6/20/17 by DW
  92 | 	
  93 | 	flBuildEveryFiveSeconds: false, //3/29/17 by DW
  94 | 	
  95 | 	flDownloadPodcasts: false, //6/7/17 by DW -- changed to false
  96 | 	maxFileNameLength: 32, //4/17/17 by DW
  97 | 	maxConcurrentPodcastDownloads: 10, //4/17/17 by DW
  98 | 	
  99 | 	flSaveFeedRivers: true, //6/29/17 by DW
 100 | 	
 101 | 	httpUserAgent: myProductName + " v" + myVersion //4/29/18 by DW
 102 | 	};
 103 | var serverStats = {
 104 | 	aggregator: myProductName + " v" + myVersion,
 105 | 	
 106 | 	ctStarts: 0,
 107 | 	ctFeedReads: 0, 
 108 | 	ctFeedReadsThisRun: 0,
 109 | 	ctFeedReadsToday: 0,
 110 | 	ctFeedReadsLastHour: 0,
 111 | 	ctRiverSaves: 0, 
 112 | 	ctStoriesAdded: 0,
 113 | 	ctStoriesAddedThisRun: 0,
 114 | 	ctStoriesAddedToday: 0,
 115 | 	ctHits: 0, 
 116 | 	ctHitsToday: 0, 
 117 | 	ctHitsThisRun: 0,
 118 | 	ctListFolderReads: 0,
 119 | 	ctRssCloudUpdates: 0,
 120 | 	ctLocalStorageWrites: 0,
 121 | 	ctStatsSaves: 0,
 122 | 	ctFeedStatsSaves: 0,
 123 | 	ctRiverJsonSaves: 0,
 124 | 	ctRssCloudRenews: 0,
 125 | 	
 126 | 	ctSecsSinceLastStart: 0,
 127 | 	ctSecsSinceLastFeedReed: 0,
 128 | 	
 129 | 	whenFirstStart: new Date (),
 130 | 	whenLastStart: new Date (0),
 131 | 	whenLastFeedRead: new Date (0),
 132 | 	whenLastRiverSave: new Date (0), 
 133 | 	whenLastStoryAdded: new Date (0),
 134 | 	whenLastListFolderRead: new Date (0),
 135 | 	whenLastRssCloudUpdate: new Date (0),
 136 | 	whenLastLocalStorageWrite: new Date (0),
 137 | 	whenLastStatsSave: new Date (0),
 138 | 	whenLastFeedStatsSave: new Date (0),
 139 | 	whenLastRiverJsonSave: new Date (0),
 140 | 	whenLastRssCloudRenew: new Date (0),
 141 | 	
 142 | 	lastFeedRead: "",
 143 | 	serialnum: 0, //each new story gets an ID
 144 | 	urlFeedLastCloudUpdate: "", //the last feed we got pinged about
 145 | 	listNames: new Array (),
 146 | 	listsThatChanged: new Array (),
 147 | 	
 148 | 	lastStoryAdded: new Object ()
 149 | 	};
 150 | var flStatsChanged = false;
 151 | var flEveryMinuteScheduled = false;
 152 | var lastEveryMinuteHour = -1;
 153 | var whenServerStart = new Date ();
 154 | var origAppModDate = new Date (0);
 155 | 
 156 | function getRequestOptions (urlToRequest) {
 157 | 	var options = {
 158 | 		url: urlToRequest,
 159 | 		jar: true,
 160 | 		gzip: true, //6/25/17 by DW
 161 | 		maxRedirects: 5,
 162 | 		headers: {
 163 | 			"User-Agent": myProductName + " v" + myVersion
 164 | 			}
 165 | 		};
 166 | 	return (options);
 167 | 	}
 168 | function myRequestCall (url) { //2/11/17 by DW
 169 | 	return (request (getRequestOptions (url)));
 170 | 	}
 171 | function myConsoleLog (s) { //3/28/17 by DW
 172 | 	if (config.consoleLogCallback !== undefined) {
 173 | 		config.consoleLogCallback (s);
 174 | 		}
 175 | 	console.log (s);
 176 | 	}
 177 | 
 178 | //files
 179 | 	function readFile (relpath, callback) {
 180 | 		var f = config.dataFolder + relpath;
 181 | 		fsSureFilePath (f, function () {
 182 | 			fs.exists (f, function (flExists) {
 183 | 				if (flExists) {
 184 | 					fs.readFile (f, function (err, data) {
 185 | 						if (err) {
 186 | 							console.log ("readFile: error reading file " + f + " == " + err.message)
 187 | 							callback (undefined);
 188 | 							}
 189 | 						else {
 190 | 							callback (data);
 191 | 							}
 192 | 						});
 193 | 					}
 194 | 				else {
 195 | 					callback (undefined);
 196 | 					}
 197 | 				});
 198 | 			});
 199 | 		}
 200 | 	function writeFile (relpath, data, callback) {
 201 | 		var f = config.dataFolder + relpath;
 202 | 		fsSureFilePath (f, function () {
 203 | 			fs.writeFile (f, data, function (err) {
 204 | 				if (err) {
 205 | 					myConsoleLog ("writeFile: relpath == " + relpath + ", error == " + err.message);
 206 | 					}
 207 | 				if (callback !== undefined) {
 208 | 					callback ();
 209 | 					}
 210 | 				});
 211 | 			});
 212 | 		}
 213 | 	function readStats (relpath, stats, callback) {
 214 | 		readFile (relpath, function (data) {
 215 | 			if (data !== undefined) {
 216 | 				try {
 217 | 					var savedStats = JSON.parse (data.toString ());
 218 | 					for (var x in savedStats) {
 219 | 						stats [x] = savedStats [x];
 220 | 						}
 221 | 					}
 222 | 				catch (err) {
 223 | 					writeStats (relpath, stats); //write initial values
 224 | 					}
 225 | 				}
 226 | 			else {
 227 | 				writeStats (relpath, stats); //write initial values
 228 | 				}
 229 | 			if (callback !== undefined) {
 230 | 				callback ();
 231 | 				}
 232 | 			});
 233 | 		}
 234 | 	function writeStats (relpath, stats, callback) {
 235 | 		writeFile (relpath, utils.jsonStringify (stats), function () {
 236 | 			if (callback !== undefined) {
 237 | 				callback ();
 238 | 				}
 239 | 			});
 240 | 		}
 241 | 	function listFiles (folder, callback) {
 242 | 		fsSureFilePath (folder + "xxx", function () {
 243 | 			fs.readdir (folder, function (err, list) {
 244 | 				if (!endsWithChar (folder, "/")) {
 245 | 					folder += "/";
 246 | 					}
 247 | 				if (list !== undefined) { //6/4/15 by DW
 248 | 					for (var i = 0; i < list.length; i++) {
 249 | 						callback (folder + list [i]);
 250 | 						}
 251 | 					}
 252 | 				callback (undefined);
 253 | 				});
 254 | 			});
 255 | 		}
 256 | 	
 257 | //file write queue
 258 | 	var fileWriteQueue = new Array (), flFileWriteQueueChanged = false;
 259 | 	function pushFileWriteQueue (theFile, theData) {
 260 | 		fileWriteQueue [fileWriteQueue.length] = {
 261 | 			f: theFile,
 262 | 			jsontext: utils.jsonStringify (theData)
 263 | 			};
 264 | 		flFileWriteQueueChanged = true;
 265 | 		}
 266 | 	function checkFileWriteQueue () {
 267 | 		var ct = 0;
 268 | 		while (fileWriteQueue.length > 0) {
 269 | 			var item = fileWriteQueue [0];
 270 | 			fileWriteQueue.shift (); //remove first element
 271 | 			writeFile (item.f, item.jsontext);
 272 | 			if (++ct > config.maxConcurrentFileWrites) {
 273 | 				break;
 274 | 				}
 275 | 			}
 276 | 		}
 277 | //feeds array
 278 | 	var feedsArray = [], flFeedsArrayChanged = false, fnameFeedsStats = "feedsStats.json";
 279 | 	
 280 | 	function initFeedsArrayItem (feedstats) {
 281 | 		if (feedstats.description === undefined) { //5/28/14 by DW
 282 | 			feedstats.description = "";
 283 | 			}
 284 | 		
 285 | 		if (feedstats.ctReads === undefined) {
 286 | 			feedstats.ctReads = 0;
 287 | 			}
 288 | 		if (feedstats.whenLastRead === undefined) {
 289 | 			feedstats.whenLastRead = new Date (0);
 290 | 			}
 291 | 		
 292 | 		if (feedstats.ctItems === undefined) {
 293 | 			feedstats.ctItems = 0;
 294 | 			}
 295 | 		if (feedstats.whenLastNewItem === undefined) {
 296 | 			feedstats.whenLastNewItem = new Date (0);
 297 | 			}
 298 | 		
 299 | 		if (feedstats.ctReadErrors === undefined) {
 300 | 			feedstats.ctReadErrors = 0;
 301 | 			}
 302 | 		if (feedstats.whenLastReadError === undefined) {
 303 | 			feedstats.whenLastReadError = new Date (0);
 304 | 			}
 305 | 		if (feedstats.ctConsecutiveReadErrors === undefined) {
 306 | 			feedstats.ctConsecutiveReadErrors = 0;
 307 | 			}
 308 | 		
 309 | 		if (feedstats.ctTimesChosen === undefined) {
 310 | 			feedstats.ctTimesChosen = 0;
 311 | 			}
 312 | 		if (feedstats.whenLastChosenToRead === undefined) {
 313 | 			feedstats.whenLastChosenToRead = new Date (0);
 314 | 			}
 315 | 		
 316 | 		if (feedstats.ctCloudRenew === undefined) {
 317 | 			feedstats.ctCloudRenew = 0;
 318 | 			}
 319 | 		if (feedstats.ctCloudRenewErrors === undefined) {
 320 | 			feedstats.ctCloudRenewErrors = 0;
 321 | 			}
 322 | 		if (feedstats.ctConsecutiveCloudRenewErrors === undefined) {
 323 | 			feedstats.ctConsecutiveCloudRenewErrors = 0;
 324 | 			}
 325 | 		if (feedstats.whenLastCloudRenew === undefined) {
 326 | 			feedstats.whenLastCloudRenew = new Date (0);
 327 | 			}
 328 | 		if (feedstats.whenLastCloudRenewError === undefined) {
 329 | 			feedstats.whenLastCloudRenewError = new Date (0);
 330 | 			}
 331 | 		}
 332 | 	function findInFeedsArray (urlfeed) {
 333 | 		var lowerfeed = urlfeed.toLowerCase (), flfound = false, ixfeed;
 334 | 		for (var i = 0; i < feedsArray.length; i++) {
 335 | 			if (feedsArray [i].url.toLowerCase () == lowerfeed) {
 336 | 				var feedstats = feedsArray [i];
 337 | 				initFeedsArrayItem (feedstats);
 338 | 				return (feedstats);
 339 | 				}
 340 | 			}
 341 | 		return (undefined);
 342 | 		}
 343 | 	function addFeedToFeedsArray (urlfeed, listname) {
 344 | 		var obj = {
 345 | 			url: urlfeed,
 346 | 			lists: []
 347 | 			}
 348 | 		if (listname !== undefined) {
 349 | 			obj.lists [obj.lists.length] = listname;
 350 | 			}
 351 | 		initFeedsArrayItem (obj);
 352 | 		feedsArray [feedsArray.length] = obj;
 353 | 		flFeedsArrayChanged = true;
 354 | 		return (obj);
 355 | 		}
 356 | 	function saveFeedsArray () {
 357 | 		serverStats.ctFeedStatsSaves++;
 358 | 		serverStats.whenLastFeedStatsSave = new Date ();
 359 | 		flStatsChanged = true;
 360 | 		writeStats (fnameFeedsStats, feedsArray);
 361 | 		}
 362 | 	function getFeedTitle (urlfeed) {
 363 | 		var feedStats = findInFeedsArray (urlfeed); 
 364 | 		return (feedStats.title);
 365 | 		}
 366 | 	
 367 | //feeds in lists object
 368 | 	var feedsInLists = new Object (), flFeedsInListsChanged = false, fnameFeedsInLists = "feedsInLists.json";
 369 | 	
 370 | 	function atLeastOneSubscriber (urlfeed) {
 371 | 		var ctsubscribers = feedsInLists [urlfeed];
 372 | 		if (ctsubscribers === undefined) {
 373 | 			return (false);
 374 | 			}
 375 | 		else {
 376 | 			return (Number (ctsubscribers) > 0);
 377 | 			}
 378 | 		}
 379 | 	function addToFeedsInLists (urlfeed) { //5/30/14 by DW
 380 | 		if (feedsInLists [urlfeed] === undefined) {
 381 | 			feedsInLists [urlfeed] = 1;
 382 | 			}
 383 | 		else {
 384 | 			feedsInLists [urlfeed]++;
 385 | 			}
 386 | 		flFeedsInListsChanged = true;
 387 | 		}
 388 | 	function saveFeedsInLists () {
 389 | 		writeStats (fnameFeedsInLists, feedsInLists);
 390 | 		}
 391 | //feeds
 392 | 	function getFolderForFeed (urlfeed) { //return path to the folder for the feed
 393 | 		var s = urlfeed;
 394 | 		if (utils.beginsWith (s, "http://")) {
 395 | 			s = utils.stringDelete (s, 1, 7);
 396 | 			}
 397 | 		else {
 398 | 			if (utils.beginsWith (s, "https://")) {
 399 | 				s = utils.stringDelete (s, 1, 8);
 400 | 				}
 401 | 			}
 402 | 		s = cleanFilenameForPlatform (s); 
 403 | 		s = "feeds/" + s + "/";
 404 | 		return (s);
 405 | 		}
 406 | 	function writeFeedInfoFile (feed, callback) {
 407 | 		var f = getFolderForFeed (feed.prefs.url) + "feedInfo.json";
 408 | 		
 409 | 		feed.stats.ctInfoWrites++;
 410 | 		feed.stats.whenLastInfoWrite = new Date ();
 411 | 		
 412 | 		writeFile (f, utils.jsonStringify (feed), function () {
 413 | 			if (callback !== undefined) {
 414 | 				callback ();
 415 | 				}
 416 | 			});
 417 | 		}
 418 | 	function initFeed (urlfeed, callback) {
 419 | 		var f = getFolderForFeed (urlfeed) + "feedInfo.json";
 420 | 		function initFeedStruct (obj) {
 421 | 			//prefs
 422 | 				if (obj.prefs === undefined) {
 423 | 					obj.prefs = new Object ();
 424 | 					}
 425 | 				if (obj.prefs.enabled === undefined) {
 426 | 					obj.prefs.enabled = true;
 427 | 					}
 428 | 				if (obj.prefs.url === undefined) {
 429 | 					obj.prefs.url = urlfeed;
 430 | 					}
 431 | 				if (obj.prefs.ctSecsBetwRenews === undefined) {
 432 | 					obj.prefs.ctSecsBetwRenews = 24 * 60 * 60; //24 hours
 433 | 					}
 434 | 				if (obj.prefs.flNonListSubscribe === undefined) {
 435 | 					obj.prefs.flNonListSubscribe = false;
 436 | 					}
 437 | 			//data
 438 | 				if (obj.data === undefined) {
 439 | 					obj.data = new Object ();
 440 | 					}
 441 | 				if (obj.data.feedhash === undefined) {
 442 | 					obj.data.feedhash = "";
 443 | 					}
 444 | 			//stats
 445 | 				if (obj.stats === undefined) {
 446 | 					obj.stats = new Object ();
 447 | 					}
 448 | 				if (obj.stats.ctReads === undefined) {
 449 | 					obj.stats.ctReads = 0;
 450 | 					}
 451 | 				if (obj.stats.ctReadErrors === undefined) {
 452 | 					obj.stats.ctReadErrors = 0;
 453 | 					}
 454 | 				if (obj.stats.ctConsecutiveReadErrors === undefined) {
 455 | 					obj.stats.ctConsecutiveReadErrors = 0;
 456 | 					}
 457 | 				if (obj.stats.ctItems === undefined) {
 458 | 					obj.stats.ctItems = 0;
 459 | 					}
 460 | 				if (obj.stats.ctEnclosures === undefined) {
 461 | 					obj.stats.ctEnclosures = 0;
 462 | 					}
 463 | 				if (obj.stats.ctFeedTextChanges === undefined) {
 464 | 					obj.stats.ctFeedTextChanges = 0;
 465 | 					}
 466 | 				if (obj.stats.ct304s === undefined) {
 467 | 					obj.stats.ct304s = 0;
 468 | 					}
 469 | 				if (obj.stats.ctItemsTooOld === undefined) {
 470 | 					obj.stats.ctItemsTooOld = 0;
 471 | 					}
 472 | 				if (obj.stats.ctReadsSkipped === undefined) {
 473 | 					obj.stats.ctReadsSkipped = 0;
 474 | 					}
 475 | 				if (obj.stats.ctInfoReads === undefined) {
 476 | 					obj.stats.ctInfoReads = 0;
 477 | 					}
 478 | 				if (obj.stats.ctInfoWrites === undefined) {
 479 | 					obj.stats.ctInfoWrites = 0;
 480 | 					}
 481 | 				if (obj.stats.whenSubscribed === undefined) {
 482 | 					obj.stats.whenSubscribed = new Date ();
 483 | 					}
 484 | 				if (obj.stats.whenLastRead === undefined) {
 485 | 					obj.stats.whenLastRead = new Date (0);
 486 | 					}
 487 | 				if (obj.stats.whenLastNewItem === undefined) {
 488 | 					obj.stats.whenLastNewItem = new Date (0);
 489 | 					}
 490 | 				if (obj.stats.mostRecentPubDate === undefined) {
 491 | 					obj.stats.mostRecentPubDate = new Date (0);
 492 | 					}
 493 | 				if (obj.stats.whenLastInfoWrite === undefined) {
 494 | 					obj.stats.whenLastInfoWrite = new Date (0);
 495 | 					}
 496 | 				if (obj.stats.whenLastReadError === undefined) {
 497 | 					obj.stats.whenLastReadError = new Date (0);
 498 | 					}
 499 | 				if (obj.stats.whenLastInfoRead === undefined) {
 500 | 					obj.stats.whenLastInfoRead = new Date (0);
 501 | 					}
 502 | 				if (obj.stats.lastReadError === undefined) {
 503 | 					obj.stats.lastReadError = "";
 504 | 					}
 505 | 				if (obj.stats.itemSerialnum === undefined) {
 506 | 					obj.stats.itemSerialnum = 0;
 507 | 					}
 508 | 				
 509 | 			//feedInfo
 510 | 				if (obj.feedInfo === undefined) {
 511 | 					obj.feedInfo = new Object ();
 512 | 					}
 513 | 				if (obj.feedInfo.title === undefined) {
 514 | 					obj.feedInfo.title = "";
 515 | 					}
 516 | 				if (obj.feedInfo.link === undefined) {
 517 | 					obj.feedInfo.link = "";
 518 | 					}
 519 | 				if (obj.feedInfo.description === undefined) {
 520 | 					obj.feedInfo.description = "";
 521 | 					}
 522 | 			//misc
 523 | 				if (obj.history === undefined) {
 524 | 					obj.history = new Array ();
 525 | 					}
 526 | 				if (obj.lists === undefined) {
 527 | 					obj.lists = new Array ();
 528 | 					}
 529 | 				if (obj.calendar === undefined) {
 530 | 					obj.calendar = new Object ();
 531 | 					}
 532 | 			}
 533 | 		
 534 | 		readFile (f, function (data) {
 535 | 			if (data === undefined) {
 536 | 				var jstruct = new Object ();
 537 | 				initFeedStruct (jstruct);
 538 | 				callback (jstruct);
 539 | 				}
 540 | 			else {
 541 | 				var jstruct;
 542 | 				try {
 543 | 					jstruct = JSON.parse (data.toString ());
 544 | 					}
 545 | 				catch (err) {
 546 | 					jstruct = new Object ();
 547 | 					}
 548 | 				initFeedStruct (jstruct);
 549 | 				
 550 | 				jstruct.stats.ctInfoReads++;
 551 | 				jstruct.stats.whenLastInfoRead = new Date ();
 552 | 				
 553 | 				callback (jstruct);
 554 | 				}
 555 | 			});
 556 | 		
 557 | 		
 558 | 		}
 559 | 	function readFeed (urlfeed, callback) {
 560 | 		var starttime = new Date ();
 561 | 		var itemsInFeed = new Object (); //6/3/15 by DW
 562 | 		function getItemGuid (item) {
 563 | 			function ok (val) {
 564 | 				if (val != undefined) {
 565 | 					if (val != "null") {
 566 | 						return (true);
 567 | 						}
 568 | 					}
 569 | 				return (false);
 570 | 				}
 571 | 			if (ok (item.guid)) {
 572 | 				return (item.guid);
 573 | 				}
 574 | 			var guid = "";
 575 | 			if (ok (item.pubDate)) {
 576 | 				guid += item.pubDate;
 577 | 				}
 578 | 			if (ok (item.link)) {
 579 | 				guid += item.link;
 580 | 				}
 581 | 			if (ok (item.title)) {
 582 | 				guid += item.title;
 583 | 				}
 584 | 			if (guid.length > 0) {
 585 | 				guid = md5 (guid);
 586 | 				}
 587 | 			return (guid);
 588 | 			}
 589 | 		initFeed (urlfeed, function (feed) {
 590 | 			function writeFeed () {
 591 | 				feed.stats.ctSecsLastRead = utils.secondsSince (starttime);
 592 | 				writeFeedInfoFile (feed);
 593 | 				}
 594 | 			function feedError (message) {
 595 | 				feed.stats.ctReadErrors++;
 596 | 				feed.stats.ctConsecutiveReadErrors++;
 597 | 				feed.stats.whenLastReadError = starttime;
 598 | 				if (message !== undefined) {
 599 | 					feed.stats.lastReadError = message;
 600 | 					}
 601 | 				writeFeed ();
 602 | 				
 603 | 				feedstats.ctReadErrors++;
 604 | 				feedstats.ctConsecutiveReadErrors++;
 605 | 				feedstats.whenLastReadError = starttime;
 606 | 				}
 607 | 			function processFeedItem (item) {
 608 | 				if (new Date (item.pubDate) > new Date (feed.stats.mostRecentPubDate)) {
 609 | 					feed.stats.mostRecentPubDate = item.pubDate;
 610 | 					feedstats.mostRecentPubDate = item.pubDate;
 611 | 					}
 612 | 				
 613 | 				//copy cloud info, if present -- 6/3/15 by DW
 614 | 					if (item.meta.cloud !== undefined) {
 615 | 						if (item.meta.cloud.domain !== undefined) {
 616 | 							feed.feedInfo.cloud = {
 617 | 								domain: item.meta.cloud.domain,
 618 | 								port: item.meta.cloud.port,
 619 | 								path: item.meta.cloud.path,
 620 | 								port: item.meta.cloud.port,
 621 | 								registerProcedure: item.meta.cloud.registerprocedure,
 622 | 								protocol: item.meta.cloud.protocol
 623 | 								};
 624 | 							feedstats.cloud = {
 625 | 								domain: item.meta.cloud.domain,
 626 | 								port: item.meta.cloud.port,
 627 | 								path: item.meta.cloud.path,
 628 | 								port: item.meta.cloud.port,
 629 | 								registerProcedure: item.meta.cloud.registerprocedure,
 630 | 								protocol: item.meta.cloud.protocol,
 631 | 								};
 632 | 							}
 633 | 						}
 634 | 				
 635 | 				//set flnew -- do the history thing
 636 | 					var theGuid = getItemGuid (item);
 637 | 					itemsInFeed [theGuid] = true; //6/3/15 by DW
 638 | 					flnew = true;
 639 | 					for (var i = 0; i < feed.history.length; i++) {
 640 | 						if (feed.history [i].guid == theGuid) { //we've already seen it
 641 | 							flnew = false;
 642 | 							break;
 643 | 							}
 644 | 						}
 645 | 				if (flnew) { //add to the history array
 646 | 					var obj = new Object (), flAddToRiver = true;
 647 | 					obj.title = item.title; 
 648 | 					obj.link = item.link; 
 649 | 					obj.description = getItemDescription (item);
 650 | 					obj.guid = theGuid;
 651 | 					obj.when = starttime;
 652 | 					feed.history [feed.history.length] = obj;
 653 | 					
 654 | 					//stats
 655 | 						feed.stats.ctItems++;
 656 | 						feed.stats.whenLastNewItem = starttime;
 657 | 						
 658 | 						feedstats.ctItems++;
 659 | 						feedstats.whenLastNewItem = starttime;
 660 | 						
 661 | 					
 662 | 					//copy feed info from item into the feed record -- 6/1/14 by DW
 663 | 						feed.feedInfo.title = item.meta.title;
 664 | 						feed.feedInfo.link = item.meta.link;
 665 | 						feed.feedInfo.description = item.meta.description;
 666 | 					//copy feeds info from item into feeds in-memory array element -- 6/1/14 by DW
 667 | 						feedstats.title = item.meta.title;
 668 | 						feedstats.text = item.meta.title;
 669 | 						feedstats.htmlurl = item.meta.link;
 670 | 						feedstats.description = item.meta.description;
 671 | 						flFeedsArrayChanged = true;
 672 | 					
 673 | 					//exclude items that newly appear in feed but have a too-old pubdate
 674 | 						if ((item.pubDate != null) && (new Date (item.pubDate) < utils.dateYesterday (feed.stats.mostRecentPubDate)) && (!flFirstRead)) { 
 675 | 							flAddToRiver = false;
 676 | 							feed.stats.ctItemsTooOld++;
 677 | 							feed.stats.whenLastTooOldItem = starttime;
 678 | 							}
 679 | 					
 680 | 					if (flFirstRead) {
 681 | 						if (config.flAddItemsFromNewSubs) {
 682 | 							flAddToRiver = true;
 683 | 							}
 684 | 						else {
 685 | 							flAddToRiver = false;
 686 | 							}
 687 | 						}
 688 | 					
 689 | 					if (flAddToRiver) {
 690 | 						addToRiver (urlfeed, item);
 691 | 						if (config.flWriteItemsToFiles) {
 692 | 							var relpath = getFolderForFeed (urlfeed) + "items/" + utils.padWithZeros (feed.stats.itemSerialnum++, 3) + ".json";
 693 | 							pushFileWriteQueue (relpath, utils.jsonStringify (item, true))
 694 | 							}
 695 | 						}
 696 | 					}
 697 | 				}
 698 | 			function finishFeedProcessing () {
 699 | 				//delete items in the history array that are no longer in the feed -- 6/3/15 by DW
 700 | 					var ctHistoryItemsDeleted = 0;
 701 | 					for (var i = feed.history.length - 1; i >= 0; i--) { //6/3/15 by DW
 702 | 						if (itemsInFeed [feed.history [i].guid] === undefined) { //it's no longer in the feed
 703 | 							feed.history.splice (i, 1);
 704 | 							ctHistoryItemsDeleted++;
 705 | 							}
 706 | 						}
 707 | 					
 708 | 				writeFeed ();
 709 | 				if (callback !== undefined) { //6/5/15 by DW
 710 | 					callback ();
 711 | 					}
 712 | 				}
 713 | 			function readJsonFeed (urlfeed, callback) {
 714 | 				request (getRequestOptions (urlfeed), function (err, response, body) {
 715 | 					if (err) {
 716 | 						feedError (err.message);
 717 | 						}
 718 | 					else {
 719 | 						if (response.statusCode !== 200) {
 720 | 							feedError ("readJsonFeed: response.statusCode == " + response.statusCode);
 721 | 							}
 722 | 						else {
 723 | 							try {
 724 | 								var jstruct = JSON.parse (body);
 725 | 								var items = jstruct.rss.channel.item;
 726 | 								for (var i = 0; i < items.length; i++) {
 727 | 									var item = items [i];
 728 | 									if (item.title === undefined) {
 729 | 										item.title = null;
 730 | 										}
 731 | 									item.meta = {
 732 | 										title: jstruct.rss.channel.title,
 733 | 										link: jstruct.rss.channel.link,
 734 | 										description: jstruct.rss.channel.description,
 735 | 										cloud: jstruct.rss.channel.cloud
 736 | 										};
 737 | 									
 738 | 									if (item ["source:outline"] !== undefined) { //it's already in the correct format for addToRiver, no conversion needed
 739 | 										item.outline = item ["source:outline"];
 740 | 										}
 741 | 									
 742 | 									processFeedItem (item);
 743 | 									}
 744 | 								finishFeedProcessing ();
 745 | 								}
 746 | 							catch (err) {
 747 | 								feedError ("readJsonFeed: err.message == " + err.message);
 748 | 								}
 749 | 							}
 750 | 						}
 751 | 					if (callback !== undefined) {
 752 | 						callback ();
 753 | 						}
 754 | 					});
 755 | 				}
 756 | 			function readXmlFeed (urlfeed) { //1/16/18 by DW
 757 | 				feedRead.parseUrl (urlfeed, undefined, function (err, theFeed, httpResponse) { 
 758 | 					if (err) {
 759 | 						feedError (err.message);
 760 | 						}
 761 | 					else {
 762 | 						getFeedRiver (urlfeed, function (jstruct) { //make sure the feed's river is loaded in cache
 763 | 							for (var i = 0; i < theFeed.items.length; i++) {
 764 | 								processFeedItem (theFeed.items [i]);
 765 | 								}
 766 | 							finishFeedProcessing ();
 767 | 							});
 768 | 						}
 769 | 					});
 770 | 				}
 771 | 			if (feed.prefs.enabled) {
 772 | 				var flFirstRead = feed.stats.ctReads == 0, feedstats;
 773 | 				feedstats = findInFeedsArray (urlfeed); //the in-memory feed stats, stuff the scanner uses to figure out which feed to read next
 774 | 				if (feedstats === undefined) {
 775 | 					feedstats = addFeedToFeedsArray (urlfeed);
 776 | 					}
 777 | 				//stats
 778 | 					serverStats.ctFeedReads++;
 779 | 					serverStats.ctFeedReadsLastHour++;
 780 | 					serverStats.ctFeedReadsThisRun++;
 781 | 					serverStats.ctFeedReadsToday++;
 782 | 					serverStats.lastFeedRead = urlfeed;
 783 | 					serverStats.whenLastFeedRead = starttime;
 784 | 					
 785 | 					feed.stats.ctReads++;
 786 | 					feed.stats.whenLastRead = starttime;
 787 | 					
 788 | 					feedstats.ctReads++;
 789 | 					feedstats.whenLastRead = starttime;
 790 | 					
 791 | 					flFeedsArrayChanged = true;
 792 | 				if (utils.beginsWith (urlfeed, "feed://")) { //8/13/15 by DW
 793 | 					urlfeed = "http://" + utils.stringDelete (urlfeed, 1, 7);
 794 | 					}
 795 | 				
 796 | 				if (utils.endsWith (urlfeed, ".json")) { //6/13/17 by DW
 797 | 					readJsonFeed (urlfeed);
 798 | 					}
 799 | 				else {
 800 | 					readXmlFeed (urlfeed); //1/16/18 by DW
 801 | 					}
 802 | 				}
 803 | 			});
 804 | 		}
 805 | 	function readFeedIfSubscribed (urlfeed, callback) { //8/18/17 by DW
 806 | 		if (atLeastOneSubscriber (urlfeed)) {
 807 | 			readFeed (urlfeed, callback);
 808 | 			}
 809 | 		}
 810 | 	function subscribeToFeed (urlfeed, listname) {
 811 | 		if ((urlfeed !== undefined) && (urlfeed.length > 0)) { 
 812 | 			var feedStats = findInFeedsArray (urlfeed);
 813 | 			if (feedStats === undefined) { //new subscription
 814 | 				addFeedToFeedsArray (urlfeed, listname);
 815 | 				}
 816 | 			else { //be sure this list is in its array of lists
 817 | 				var fladd = true;
 818 | 				for (var i = 0; i < feedStats.lists.length; i++) {
 819 | 					if (feedStats.lists [i] == listname) {
 820 | 						fladd = false;
 821 | 						break;
 822 | 						}
 823 | 					}
 824 | 				if (fladd) {
 825 | 					feedStats.lists [feedStats.lists.length] = listname;
 826 | 					flFeedsArrayChanged = true;
 827 | 					}
 828 | 				}
 829 | 			addToFeedsInLists (urlfeed); 
 830 | 			}
 831 | 		}
 832 | 	function findNextFeedToRead (callback) {
 833 | 		var now = new Date (), whenLeastRecent = now, itemLeastRecent = undefined;
 834 | 		for (var i = 0; i < feedsArray.length; i++) {
 835 | 			var item = feedsArray [i];
 836 | 			if (atLeastOneSubscriber (item.url)) {
 837 | 				var when = new Date (item.whenLastChosenToRead);
 838 | 				if (when < whenLeastRecent) {
 839 | 					itemLeastRecent = item;
 840 | 					whenLeastRecent = when;
 841 | 					}
 842 | 				}
 843 | 			}
 844 | 		if (itemLeastRecent !== undefined) { //at least one element in array
 845 | 			if (utils.secondsSince (itemLeastRecent.whenLastChosenToRead) >= (config.ctMinutesBetwBuilds * 60)) { //ready to read
 846 | 				itemLeastRecent.whenLastChosenToRead = now;
 847 | 				itemLeastRecent.ctTimesChosen++;
 848 | 				flFeedsArrayChanged = true;
 849 | 				if (callback !== undefined) {
 850 | 					callback (itemLeastRecent.url);
 851 | 					}
 852 | 				}
 853 | 			}
 854 | 		}
 855 | 	function getOneFeed (urlfeed, callback) { //11/26/14 by DW
 856 | 		initFeed (urlfeed, function (feed) {
 857 | 			callback (feed);
 858 | 			});
 859 | 		}
 860 | 	function readAllFeedsNow () { //4/18/17 by DW
 861 | 		function readNext (ix) {
 862 | 			if (ix < feedsArray.length) {
 863 | 				var item = feedsArray [ix];
 864 | 				if (atLeastOneSubscriber (item.url)) {
 865 | 					readFeed (item.url, function () {
 866 | 						readNext (ix + 1);
 867 | 						});
 868 | 					}
 869 | 				else {
 870 | 					readNext (ix + 1);
 871 | 					}
 872 | 				}
 873 | 			}
 874 | 		readNext (0);
 875 | 		}
 876 | //lists
 877 | 	function listChanged (listname) {
 878 | 		var flAdd = true;
 879 | 		for (var i = 0; i < serverStats.listsThatChanged.length; i++) {
 880 | 			if (serverStats.listsThatChanged [i] == listname) {
 881 | 				flAdd = false;
 882 | 				}
 883 | 			}
 884 | 		if (flAdd) {
 885 | 			serverStats.listsThatChanged [serverStats.listsThatChanged.length] = listname;
 886 | 			}
 887 | 		}
 888 | 	function getListFilename (listname) {
 889 | 		return ("lists/" + utils.stringPopExtension (listname) + "/" + config.listInfoFileName);
 890 | 		}
 891 | 	function initList (name, callback) {
 892 | 		var f = getListFilename (name);
 893 | 		function initListStruct (obj) {
 894 | 			//prefs
 895 | 				if (obj.prefs == undefined) {
 896 | 					obj.prefs = new Object ();
 897 | 					}
 898 | 				if (obj.prefs.enabled == undefined) {
 899 | 					obj.prefs.enabled = true;
 900 | 					}
 901 | 			//stats
 902 | 				if (obj.stats == undefined) {
 903 | 					obj.stats = new Object ();
 904 | 					}
 905 | 				if (obj.stats.ctReads == undefined) {
 906 | 					obj.stats.ctReads = 0;
 907 | 					}
 908 | 				if (obj.stats.whenLastRead == undefined) {
 909 | 					obj.stats.whenLastRead = new Date (0);
 910 | 					}
 911 | 				if (obj.stats.whenSubscribed == undefined) {
 912 | 					obj.stats.whenSubscribed = new Date ();
 913 | 					}
 914 | 				if (obj.stats.ctBlockedItems == undefined) {
 915 | 					obj.stats.ctBlockedItems = 0;
 916 | 					}
 917 | 			//listInfo
 918 | 				if (obj.listInfo == undefined) {
 919 | 					obj.listInfo = new Object ();
 920 | 					}
 921 | 				if (obj.listInfo.title == undefined) {
 922 | 					obj.listInfo.title = "";
 923 | 					}
 924 | 			//misc
 925 | 				if (obj.feeds == undefined) {
 926 | 					obj.feeds = new Array ();
 927 | 					}
 928 | 				if (obj.feedsBlocked == undefined) {
 929 | 					obj.feedsBlocked = new Array ();
 930 | 					}
 931 | 				if (obj.calendar == undefined) {
 932 | 					obj.calendar = new Object ();
 933 | 					}
 934 | 				if (obj.river == undefined) {
 935 | 					obj.river = new Object ();
 936 | 					}
 937 | 			}
 938 | 		readFile (f, function (data) {
 939 | 			if (data === undefined) {
 940 | 				var jstruct = new Object ();
 941 | 				initListStruct (jstruct);
 942 | 				callback (jstruct);
 943 | 				}
 944 | 			else {
 945 | 				try {
 946 | 					var jstruct = JSON.parse (data.toString ());
 947 | 					initListStruct (jstruct);
 948 | 					callback (jstruct);
 949 | 					}
 950 | 				catch (err) {
 951 | 					var jstruct = new Object ();
 952 | 					initListStruct (jstruct);
 953 | 					callback (jstruct);
 954 | 					}
 955 | 				}
 956 | 			});
 957 | 		
 958 | 		}
 959 | 	function writeListInfoFile (listname, listObj, callback) {
 960 | 		var f = getListFilename (listname);
 961 | 		writeFile (f, utils.jsonStringify (listObj), callback);
 962 | 		}
 963 | 	function readIncludedList (listname, urloutline) { //6/17/14 by DW
 964 | 		var req = myRequestCall (urloutline);
 965 | 		var opmlparser = new OpmlParser ();
 966 | 		req.on ("response", function (res) {
 967 | 			var stream = this;
 968 | 			if (res.statusCode == 200) {
 969 | 				stream.pipe (opmlparser);
 970 | 				}
 971 | 			});
 972 | 		req.on ("error", function (res) {
 973 | 			});
 974 | 		opmlparser.on ("error", function (error) {
 975 | 			myConsoleLog ("readIncludedList: opml parser error == " + error.message);
 976 | 			});
 977 | 		opmlparser.on ("readable", function () {
 978 | 			var outline;
 979 | 			while (outline = this.read ()) {
 980 | 				switch (outline ["#type"]) {
 981 | 					case "feed":
 982 | 						subscribeToFeed (outline.xmlurl, listname);
 983 | 						break;
 984 | 					}
 985 | 				}
 986 | 			});
 987 | 		opmlparser.on ("end", function () {
 988 | 			});
 989 | 		}
 990 | 	function readOneList (listname, f, callback) {
 991 | 		initList (listname, function (listObj) {
 992 | 			var opmlparser = new OpmlParser ();
 993 | 			opmlparser.on ("error", function (error) {
 994 | 				myConsoleLog ("readOneList: opml parser error == " + error.message);
 995 | 				});
 996 | 			opmlparser.on ("readable", function () {
 997 | 				var outline;
 998 | 				while (outline = this.read ()) {
 999 | 					switch (outline ["#type"]) {
1000 | 						case "feed":
1001 | 							subscribeToFeed (outline.xmlurl, listname);
1002 | 							break;
1003 | 						}
1004 | 					switch (outline.type) {
1005 | 						case "include":
1006 | 							readIncludedList (listname, outline.url);
1007 | 							break;
1008 | 						}
1009 | 					}
1010 | 				});
1011 | 			opmlparser.on ("end", function () {
1012 | 				writeListInfoFile (listname, listObj, function () {
1013 | 					if (callback !== undefined) { 
1014 | 						callback ();
1015 | 						}
1016 | 					});
1017 | 				});
1018 | 			
1019 | 			fs.readFile (f, function (err, data) {
1020 | 				if (err) {
1021 | 					myConsoleLog ("readOneList: error reading list file == " + f + ", err.message == " + err.message);
1022 | 					if (callback !== undefined) { 
1023 | 						callback ();
1024 | 						}
1025 | 					}
1026 | 				else {
1027 | 					opmlparser.end (data.toString ());
1028 | 					}
1029 | 				});
1030 | 			});
1031 | 		}
1032 | 	function readOneTxtList (listname, f, callback) {
1033 | 		initList (listname, function (listObj) {
1034 | 			fs.readFile (f, function (err, data) {
1035 | 				if (err) {
1036 | 					myConsoleLog ("readOneTxtList: error reading list file == " + f + ", err.message == " + err.message);
1037 | 					}
1038 | 				else {
1039 | 					var s = data.toString (), url = "";
1040 | 					for (var i = 0; i < s.length; i++) {
1041 | 						switch (s [i]) {
1042 | 							case "\n": case "\r": 
1043 | 								if (url.length > 0) {
1044 | 									subscribeToFeed (url, listname);
1045 | 									url = "";
1046 | 									}
1047 | 								break;
1048 | 							case "\t": //ignore tabs
1049 | 								break;
1050 | 							case " ": //spaces only significant if inside a url
1051 | 								if (url.length > 0) {
1052 | 									url += " ";
1053 | 									}
1054 | 								break;
1055 | 							default: 
1056 | 								url += s [i];
1057 | 								break;
1058 | 							}
1059 | 						}
1060 | 					if (url.length > 0) {
1061 | 						subscribeToFeed (url, listname);
1062 | 						}
1063 | 					}
1064 | 				if (callback !== undefined) { 
1065 | 					callback ();
1066 | 					}
1067 | 				});
1068 | 			});
1069 | 		}
1070 | 	function readOneJsonList (listname, f, callback) {
1071 | 		initList (listname, function (listObj) {
1072 | 			fs.readFile (f, function (err, data) {
1073 | 				if (err) {
1074 | 					myConsoleLog ("readOneJsonList: error reading list file == " + f + ", err.message == " + err.message);
1075 | 					}
1076 | 				else {
1077 | 					try {
1078 | 						var feedArray = JSON.parse (data.toString ());
1079 | 						for (var i = 0; i < feedArray.length; i++) { 
1080 | 							subscribeToFeed (feedArray [i], listname);
1081 | 							}
1082 | 						}
1083 | 					catch (err) {
1084 | 						myConsoleLog ("readOneJsonList: error parsing JSON list file == " + f + ", err.message == " + err.message);
1085 | 						}
1086 | 					}
1087 | 				if (callback !== undefined) { 
1088 | 					callback ();
1089 | 					}
1090 | 				});
1091 | 			});
1092 | 		}
1093 | 	function loadListsFromFolder (callback) {
1094 | 		var now = new Date ();
1095 | 		for (var i = 0; i < feedsArray.length; i++) { //6/7/14 by DW
1096 | 			feedsArray [i].lists = [];
1097 | 			}
1098 | 		serverStats.ctListFolderReads++;
1099 | 		serverStats.whenLastListFolderRead = now;
1100 | 		serverStats.listNames = new Array ();
1101 | 		feedsInLists = new Object ();
1102 | 		listFiles (config.listsFolder, function (f) {
1103 | 			if (f === undefined) { //no more files
1104 | 				flFirstListLoad = false;
1105 | 				if (callback !== undefined) {
1106 | 					callback ();
1107 | 					}
1108 | 				}
1109 | 			else {
1110 | 				function addListToStats (listname) {
1111 | 					serverStats.listNames [serverStats.listNames.length] = listname; 
1112 | 					flStatsChanged = true;
1113 | 					}
1114 | 				var listname = utils.stringLastField (f, "/"); //something like myList.opml
1115 | 				var ext = utils.stringLower (utils.stringLastField (listname, "."));
1116 | 				switch (ext) {
1117 | 					case "opml":
1118 | 						readOneList (listname, f);
1119 | 						addListToStats (listname);
1120 | 						break;
1121 | 					case "txt":
1122 | 						readOneTxtList (listname, f);
1123 | 						addListToStats (listname);
1124 | 						break;
1125 | 					case "json":
1126 | 						readOneJsonList (listname, f);
1127 | 						addListToStats (listname);
1128 | 						break;
1129 | 					}
1130 | 				}
1131 | 			});
1132 | 		}
1133 | 	function getAllLists (callback) {
1134 | 		var theLists = new Array ();
1135 | 		function getOneFile (ix) {
1136 | 			if (ix >= serverStats.listNames.length) {
1137 | 				callback (theLists);
1138 | 				}
1139 | 			else {
1140 | 				var fname = serverStats.listNames [ix], f = config.listsFolder + fname;
1141 | 				fs.readFile (f, function (err, data) {
1142 | 					if (err) {
1143 | 						myConsoleLog ("getAllLists: error reading list " + f + " err.message == " + err.message);
1144 | 						}
1145 | 					else {
1146 | 						theLists [theLists.length] = {
1147 | 							listname: fname,
1148 | 							opmltext: data.toString ()
1149 | 							};
1150 | 						}
1151 | 					getOneFile (ix + 1);
1152 | 					});
1153 | 				}
1154 | 			}
1155 | 		getOneFile (0);
1156 | 		}
1157 | 	function getOneList (fname, callback) { 
1158 | 		var f = config.listsFolder + fname;
1159 | 		fs.readFile (f, function (err, data) {
1160 | 			if (err) {
1161 | 				myConsoleLog ("getOneList: f == " + f + ", err.message == " + err.message);
1162 | 				callback (undefined);
1163 | 				}
1164 | 			else {
1165 | 				callback (data.toString ());
1166 | 				}
1167 | 			});
1168 | 		}
1169 | 	function saveSubscriptionList (listname, xmltext, callback) {
1170 | 		var f = config.listsFolder + listname, now = new Date ();
1171 | 		fsSureFilePath (f, function () {
1172 | 			fs.writeFile (f, xmltext, function (err) {
1173 | 				if (err) {
1174 | 					myConsoleLog ("saveSubscriptionList: f == " + f + ", err.message == " + err.message);
1175 | 					}
1176 | 				if (callback !== undefined) {
1177 | 					callback ();
1178 | 					}
1179 | 				});
1180 | 			});
1181 | 		}
1182 | //each list's river -- 2/2/16 by DW
1183 | 	var allTheRivers = new Object ();
1184 | 	
1185 | 	function getRiverDataFilename (listname) {
1186 | 		return ("lists/" + utils.stringPopExtension (listname) + "/" + config.riverDataFileName);
1187 | 		}
1188 | 	function initRiverData (theData) {
1189 | 		if (theData.ctRiverBuilds === undefined) {
1190 | 			theData.ctRiverBuilds = 0;
1191 | 			theData.flDirty = true;
1192 | 			}
1193 | 		if (theData.whenLastRiverBuild === undefined) {
1194 | 			theData.whenLastRiverBuild = new Date (0);
1195 | 			theData.flDirty = true;
1196 | 			}
1197 | 		}
1198 | 	function getRiverData (listname, callback) {
1199 | 		if (allTheRivers [listname] !== undefined) { //we already have it in memory
1200 | 			if (callback !== undefined) {
1201 | 				var jstruct = allTheRivers [listname];
1202 | 				initRiverData (jstruct);
1203 | 				callback (jstruct);
1204 | 				}
1205 | 			}
1206 | 		else { //read it from the file into allTheRivers struct
1207 | 			var f = getRiverDataFilename (listname);
1208 | 			readFile (f, function (data) {
1209 | 				var jstruct = {
1210 | 					ctItemsAdded: 0,
1211 | 					whenLastItemAdded: new Date (0),
1212 | 					ctSaves: 0,
1213 | 					whenLastSave: new Date (0),
1214 | 					flDirty: true,
1215 | 					ctRiverBuilds: 0,
1216 | 					whenLastRiverBuild: new Date (0),
1217 | 					items: new Array ()
1218 | 					};
1219 | 				if (data !== undefined) {
1220 | 					try {
1221 | 						jstruct = JSON.parse (data.toString ());
1222 | 						}
1223 | 					catch (err) {
1224 | 						}
1225 | 					}
1226 | 				initRiverData (jstruct);
1227 | 				allTheRivers [listname] = jstruct;
1228 | 				if (callback !== undefined) {
1229 | 					callback (allTheRivers [listname]);
1230 | 					}
1231 | 				});
1232 | 			}
1233 | 		}
1234 | 	function addRiverItemToList (listname, item, callback) {
1235 | 		getRiverData (listname, function (jstruct) {
1236 | 			jstruct.items [jstruct.items.length] = item; //3/14/16 by DW
1237 | 			if (jstruct.items.length > config.maxRiverItems) {
1238 | 				jstruct.items.shift ();
1239 | 				}
1240 | 			jstruct.ctItemsAdded++;
1241 | 			jstruct.whenLastItemAdded = new Date ();
1242 | 			jstruct.flDirty = true;
1243 | 			if (callback !== undefined) {
1244 | 				callback ();
1245 | 				}
1246 | 			});
1247 | 		}
1248 | 	function saveChangedRiverStructs () {
1249 | 		for (var x in allTheRivers) {
1250 | 			var item = allTheRivers [x];
1251 | 			if (item.flDirty) {
1252 | 				var f = getRiverDataFilename (x);
1253 | 				item.flDirty = false;
1254 | 				item.ctSaves++;
1255 | 				item.whenLastSave = new Date ();
1256 | 				writeFile (f, utils.jsonStringify (item));
1257 | 				}
1258 | 			}
1259 | 		}
1260 | 	function buildOneRiver (listname, callback) { 
1261 | 		var theRiver = new Object (), starttime = new Date (), ctitems = 0, titles = new Object (), ctDuplicatesSkipped = 0;
1262 | 		theRiver.updatedFeeds = new Object ();
1263 | 		theRiver.updatedFeeds.updatedFeed = new Array ();
1264 | 		getRiverData (listname, function (myRiverData) { //an array of all the items in the river
1265 | 			var lastfeedurl = undefined, theRiverFeed, flThisFeedInList = true;
1266 | 			function finishBuild () {
1267 | 				var jsontext;
1268 | 				
1269 | 				myRiverData.ctRiverBuilds++;
1270 | 				myRiverData.whenLastRiverBuild = starttime;
1271 | 				myRiverData.flDirty = true;
1272 | 				
1273 | 				theRiver.metadata = {
1274 | 					name: listname,
1275 | 					docs: "http://scripting.com/stories/2010/12/06/innovationRiverOfNewsInJso.html",
1276 | 					secs: utils.secondsSince (starttime),
1277 | 					ctBuilds: myRiverData.ctRiverBuilds,
1278 | 					ctDuplicatesSkipped: ctDuplicatesSkipped,
1279 | 					whenGMT: starttime.toUTCString (),
1280 | 					whenLocal: starttime.toLocaleString (),
1281 | 					aggregator: myProductName + " v" + myVersion
1282 | 					};
1283 | 				jsontext = utils.jsonStringify (theRiver, true);
1284 | 				jsontext = "onGetRiverStream (" + jsontext + ")";
1285 | 				var fname = utils.stringPopLastField (listname, ".") + ".js", f = config.riversFolder + fname;
1286 | 				fsSureFilePath (f, function () {
1287 | 					fs.writeFile (f, jsontext, function (err) {
1288 | 						if (err) {
1289 | 							myConsoleLog ("finishBuild: f == " + f + ", error == " + err.message);
1290 | 							}
1291 | 						else {
1292 | 							}
1293 | 						serverStats.ctRiverJsonSaves++;
1294 | 						serverStats.whenLastRiverJsonSave = starttime;
1295 | 						flStatsChanged = true;
1296 | 						notifyWebSocketListeners ("updated " + listname);
1297 | 						callBuildRiverCallbacks (fname, jsontext); //4/23/17 by DW
1298 | 						if (callback !== undefined) {
1299 | 							callback ();
1300 | 							}
1301 | 						});
1302 | 					});
1303 | 				}
1304 | 			for (var i = myRiverData.items.length - 1; i >= 0; i--) {
1305 | 				var story = myRiverData.items [i], flskip = false, reducedtitle;
1306 | 				if (config.flSkipDuplicateTitles) { //5/29/14 by DW
1307 | 					reducedtitle = utils.trimWhitespace (utils.stringLower (story.title));
1308 | 					if (reducedtitle.length > 0) { //6/6/14 by DW
1309 | 						if (titles [reducedtitle] != undefined) { //duplicate
1310 | 							ctDuplicatesSkipped++;
1311 | 							flskip = true;
1312 | 							}
1313 | 						}
1314 | 					}
1315 | 				if (!flskip) {
1316 | 					if (story.feedUrl != lastfeedurl) {
1317 | 						var feedstats = findInFeedsArray (story.feedUrl);
1318 | 						if (feedstats === undefined) { //we could have unsubbed the feed -- 4/8/18 by DW
1319 | 							flskip = true;
1320 | 							}
1321 | 						else {
1322 | 							var ix = theRiver.updatedFeeds.updatedFeed.length;
1323 | 							theRiver.updatedFeeds.updatedFeed [ix] = new Object ();
1324 | 							theRiverFeed = theRiver.updatedFeeds.updatedFeed [ix];
1325 | 							
1326 | 							theRiverFeed.feedTitle = feedstats.title;
1327 | 							theRiverFeed.feedUrl = story.feedUrl;
1328 | 							theRiverFeed.websiteUrl = feedstats.htmlurl;
1329 | 							//description
1330 | 								if (feedstats.description == undefined) {
1331 | 									theRiverFeed.feedDescription = "";
1332 | 									}
1333 | 								else {
1334 | 									theRiverFeed.feedDescription = feedstats.description;
1335 | 									}
1336 | 							//whenLastUpdate -- 6/7/15 by DW
1337 | 								if (story.when !== undefined) {
1338 | 									theRiverFeed.whenLastUpdate = new Date (story.when).toUTCString ();
1339 | 									}
1340 | 								else {
1341 | 									theRiverFeed.whenLastUpdate = new Date (feedstats.whenLastNewItem).toUTCString ();
1342 | 									}
1343 | 							theRiverFeed.item = new Array ();
1344 | 							
1345 | 							lastfeedurl = story.feedUrl;
1346 | 							}
1347 | 						}
1348 | 					if (!flskip) {
1349 | 						var thePubDate = story.pubdate; //2/10/16 by DW
1350 | 						if (thePubDate == null) {
1351 | 							thePubDate = starttime;
1352 | 							}
1353 | 						
1354 | 						var theItem = {
1355 | 							title: story.title,
1356 | 							link: story.link,
1357 | 							body: story.description,
1358 | 							pubDate: new Date (thePubDate).toUTCString (),
1359 | 							permaLink: story.permalink
1360 | 							};
1361 | 						if (story.outline != undefined) { //7/16/14 by DW
1362 | 							theItem.outline = story.outline;
1363 | 							}
1364 | 						if (story.comments.length > 0) { //6/7/14 by DW
1365 | 							theItem.comments = story.comments;
1366 | 							}
1367 | 						//enclosure -- 5/30/14 by DW
1368 | 							if (story.enclosure != undefined) {
1369 | 								var flgood = true;
1370 | 								
1371 | 								if ((story.enclosure.type == undefined) || (story.enclosure.length === undefined)) { //both are required
1372 | 									flgood = false; //sorry! :-(
1373 | 									}
1374 | 								else {
1375 | 									if (utils.stringCountFields (story.enclosure.type, "/") < 2) { //something like "image" -- not a valid type
1376 | 										flgood = false; //we read the spec, did you? :-)
1377 | 										}
1378 | 									}
1379 | 								
1380 | 								if (flgood) {
1381 | 									theItem.enclosure = [story.enclosure];
1382 | 									}
1383 | 								}
1384 | 						//id
1385 | 							if (story.id == undefined) {
1386 | 								theItem.id = "";
1387 | 								}
1388 | 							else {
1389 | 								theItem.id = utils.padWithZeros (story.id, 7);
1390 | 								}
1391 | 						
1392 | 						theRiverFeed.item [theRiverFeed.item.length] = theItem;
1393 | 						
1394 | 						if (config.flSkipDuplicateTitles) { //5/29/14 by DW -- add the title to the titles object
1395 | 							titles [reducedtitle] = true;
1396 | 							}
1397 | 						}
1398 | 					}
1399 | 				}
1400 | 			finishBuild ();
1401 | 			});
1402 | 		}
1403 | 	function loadListBasedRivers (callback) { //1/16/18 by DW -- load all the list-based rivers at startup
1404 | 		fs.readdir (config.listsFolder, function (err, list) {
1405 | 			if (list !== undefined) { //6/4/15 by DW
1406 | 				function doNextFolder (ix) {
1407 | 					if (ix < list.length) {
1408 | 						var listname = list [ix];
1409 | 						getRiverData (listname, function () {
1410 | 							doNextFolder (ix + 1);
1411 | 							});
1412 | 						}
1413 | 					else {
1414 | 						callback ();
1415 | 						}
1416 | 					}
1417 | 				doNextFolder (0);
1418 | 				}
1419 | 			});
1420 | 		}
1421 | //keep a river for each feed -- 6/29/17 by DW
1422 | 	var allTheFeedRivers = new Object (); 
1423 | 	
1424 | 	function readRiverData (f, callback) {
1425 | 		readFile (f, function (data) {
1426 | 			var jstruct = {
1427 | 				ctItemsAdded: 0,
1428 | 				whenLastItemAdded: new Date (0),
1429 | 				ctSaves: 0,
1430 | 				whenLastSave: new Date (0),
1431 | 				flDirty: true,
1432 | 				ctRiverBuilds: 0,
1433 | 				whenLastRiverBuild: new Date (0),
1434 | 				items: new Array ()
1435 | 				};
1436 | 			if (data !== undefined) {
1437 | 				try {
1438 | 					jstruct = JSON.parse (data.toString ());
1439 | 					}
1440 | 				catch (err) {
1441 | 					}
1442 | 				}
1443 | 			initRiverData (jstruct);
1444 | 			if (callback !== undefined) {
1445 | 				callback (jstruct);
1446 | 				}
1447 | 			});
1448 | 		}
1449 | 	function getFeedRiver (urlfeed, callback) {
1450 | 		if (allTheFeedRivers [urlfeed] !== undefined) {
1451 | 			var theRiver = allTheFeedRivers [urlfeed];
1452 | 			theRiver.ctAccesses++;
1453 | 			theRiver.whenLastAccess = new Date ();
1454 | 			callback (theRiver.jstruct);
1455 | 			}
1456 | 		else {
1457 | 			var f = getFolderForFeed (urlfeed) + "feedRiver.json";
1458 | 			readRiverData (f, function (jstruct) {
1459 | 				allTheFeedRivers [urlfeed] = {
1460 | 					f: f,
1461 | 					whenLastAccess: new Date (),
1462 | 					ctAccesses: 1,
1463 | 					flChanged: true,
1464 | 					jstruct: jstruct
1465 | 					};
1466 | 				callback (jstruct);
1467 | 				});
1468 | 			}
1469 | 		}
1470 | 	function addItemToFeedRiver (urlfeed, item, itemFromParser) {
1471 | 		if (config.flSaveFeedRivers) {
1472 | 			getFeedRiver (urlfeed, function (jstruct) {
1473 | 				var itemToPush = new Object ();
1474 | 				utils.copyScalars (item, itemToPush);
1475 | 				itemToPush.fullDescription = itemFromParser.description;
1476 | 				if (item.outline !== undefined) { //7/6/17 by DW
1477 | 					itemToPush.outline = item.outline;
1478 | 					}
1479 | 				jstruct.items.push (itemToPush);
1480 | 				
1481 | 				while (jstruct.items.length > config.maxRiverItems) {
1482 | 					jstruct.items.shift ();
1483 | 					}
1484 | 				
1485 | 				jstruct.ctItemsAdded++;
1486 | 				jstruct.whenLastItemAdded = new Date ();
1487 | 				jstruct.flDirty = true;
1488 | 				});
1489 | 			}
1490 | 		}
1491 | 	function saveChangedFeedRivers () {
1492 | 		for (var x in allTheFeedRivers) {
1493 | 			var theRiver = allTheFeedRivers [x], item = theRiver.jstruct;
1494 | 			if (item.flDirty) {
1495 | 				item.flDirty = false;
1496 | 				item.ctSaves++;
1497 | 				item.whenLastSave = new Date ();
1498 | 				writeFile (theRiver.f, utils.jsonStringify (item));
1499 | 				}
1500 | 			}
1501 | 		}
1502 | 	function tossOldFeedRivers () { //delete rivers that haven't been accessed in the last minute 
1503 | 		for (var x in allTheFeedRivers) {
1504 | 			if (utils.secondsSince (allTheFeedRivers [x].whenLastAccess) > 60) {
1505 | 				delete allTheFeedRivers [x];
1506 | 				}
1507 | 			}
1508 | 		}
1509 | 	function getFeedRiverForServer (urlfeed, callback) {
1510 | 		getFeedRiver (urlfeed, function (jstruct) {
1511 | 			var stats = findInFeedsArray (urlfeed);
1512 | 			if (stats === undefined) {
1513 | 				callback (returnstruct);
1514 | 				}
1515 | 			else {
1516 | 				var returnstruct = {
1517 | 					title: stats.title,
1518 | 					link: stats.htmlurl,
1519 | 					description: stats.description,
1520 | 					url: stats.url,
1521 | 					items: new Array (),
1522 | 					stats: {
1523 | 						ctReads: stats.ctReads,
1524 | 						ctItems: stats.ctItems,
1525 | 						ctReadErrors: stats.ctReadErrors,
1526 | 						ctConsecutiveReadErrors: stats.ctConsecutiveReadErrors,
1527 | 						whenLastNewItem: stats.whenLastNewItem,
1528 | 						whenLastReadError: stats.whenLastReadError,
1529 | 						whenLastRead: stats.whenLastRead,
1530 | 						mostRecentPubDate: stats.mostRecentPubDate,
1531 | 						cloud: {
1532 | 							ctCloudRenew: stats.ctCloudRenew,
1533 | 							ctCloudRenewErrors: stats.ctCloudRenewErrors,
1534 | 							ctConsecutiveCloudRenewErrors: stats.ctConsecutiveCloudRenewErrors,
1535 | 							whenLastCloudRenew: stats.whenLastCloudRenew,
1536 | 							whenLastCloudRenewError: stats.whenLastCloudRenewError
1537 | 							}
1538 | 						}
1539 | 					};
1540 | 				for (var i = jstruct.items.length - 1; i >= 0; i--) {
1541 | 					returnstruct.items.unshift (jstruct.items [i]);
1542 | 					}
1543 | 				callback (returnstruct);
1544 | 				}
1545 | 			});
1546 | 		}
1547 | //podcasts -- 4/17/17 by DW
1548 | 	var podcastQueue = new Array (), ctConcurrentPodcastDownloads = 0;
1549 | 	
1550 | 	function checkPodcastQueue () {
1551 | 		if (config.flDownloadPodcasts) {
1552 | 			if (podcastQueue.length > 0) {
1553 | 				if (ctConcurrentPodcastDownloads < config.maxConcurrentPodcastDownloads) {
1554 | 					var item = podcastQueue.shift (); //remove and return first element
1555 | 					ctConcurrentPodcastDownloads++;
1556 | 					downloadBigFile (item.url, item.f, item.pubdate, function () {
1557 | 						ctConcurrentPodcastDownloads--;
1558 | 						});
1559 | 					}
1560 | 				}
1561 | 			}
1562 | 		}
1563 | 	function pushPodcastDownloadQueue (url, f, pubdate) {
1564 | 		podcastQueue [podcastQueue.length] = {
1565 | 			url: url,
1566 | 			f: f,
1567 | 			pubdate: pubdate
1568 | 			};
1569 | 		flPodcastQueueChanged = true;
1570 | 		}
1571 | 	function downloadPodcast (itemFromRiver, urlfeed) { //4/17/17 by DW
1572 | 		function goodFilename (fname) { 
1573 | 			fname = cleanFilenameForPlatform (fname);
1574 | 			fname = utils.maxStringLength (fname, config.maxFileNameLength, false, false);
1575 | 			return (fname);
1576 | 			}
1577 | 		if (config.flDownloadPodcasts) {
1578 | 			if (itemFromRiver.enclosure !== undefined) {
1579 | 				if (utils.beginsWith (itemFromRiver.enclosure.type, "audio/")) {
1580 | 					var subfoldername = goodFilename (getFeedTitle (urlfeed));
1581 | 					var fname = utils.stringLastField (itemFromRiver.enclosure.url, "/");
1582 | 					fname = utils.stringNthField (fname, "?", 1);
1583 | 					var extension = utils.stringLastField (fname, ".");
1584 | 					var f = config.podcastsFolder + subfoldername + "/" + itemFromRiver.id + "." + extension;
1585 | 					pushPodcastDownloadQueue (itemFromRiver.enclosure.url, f, itemFromRiver.pubdate);
1586 | 					}
1587 | 				}
1588 | 			}
1589 | 		}
1590 | //rivers
1591 | 	var todaysRiver = [], flRiverChanged = false, dayRiverCovers = new Date ();
1592 | 	
1593 | 	function getItemDescription (item) {
1594 | 		var s = item.description;
1595 | 		if (s == null) {
1596 | 			s = "";
1597 | 			}
1598 | 		s = utils.stripMarkup (s);
1599 | 		s = utils.trimWhitespace (s);
1600 | 		if (s.length > config.maxBodyLength) {
1601 | 			s = utils.trimWhitespace (utils.maxStringLength (s, config.maxBodyLength));
1602 | 			}
1603 | 		return (s);
1604 | 		}
1605 | 	function addToRiver (urlfeed, itemFromParser, callback) {
1606 | 		var now = new Date (), item = new Object ();
1607 | 		//copy selected elements from the object from feedparser, into the item for the river
1608 | 			function convertOutline (jstruct) { //7/16/14 by DW
1609 | 				var theNewOutline = {}, atts, subs;
1610 | 				if (jstruct ["source:outline"] != undefined) {
1611 | 					if (jstruct ["@"] != undefined) {
1612 | 						atts = jstruct ["@"];
1613 | 						subs = jstruct ["source:outline"];
1614 | 						}
1615 | 					else {
1616 | 						atts = jstruct ["source:outline"] ["@"];
1617 | 						subs = jstruct ["source:outline"] ["source:outline"];
1618 | 						}
1619 | 					}
1620 | 				else {
1621 | 					atts = jstruct ["@"];
1622 | 					subs = undefined;
1623 | 					}
1624 | 				for (var x in atts) {
1625 | 					theNewOutline [x] = atts [x];
1626 | 					}
1627 | 				if (subs != undefined) {
1628 | 					theNewOutline.subs = [];
1629 | 					if (subs instanceof Array) {
1630 | 						for (var i = 0; i < subs.length; i++) {
1631 | 							theNewOutline.subs [i] = convertOutline (subs [i]);
1632 | 							}
1633 | 						}
1634 | 					else {
1635 | 						theNewOutline.subs = [];
1636 | 						theNewOutline.subs [0] = {};
1637 | 						for (var x in subs ["@"]) {
1638 | 							theNewOutline.subs [0] [x] = subs ["@"] [x];
1639 | 							}
1640 | 						}
1641 | 					}
1642 | 				return (theNewOutline);
1643 | 				}
1644 | 			function newConvertOutline (jstruct) { //10/16/14 by DW
1645 | 				var theNewOutline = {};
1646 | 				if (jstruct ["@"] != undefined) {
1647 | 					utils.copyScalars (jstruct ["@"], theNewOutline);
1648 | 					}
1649 | 				if (jstruct ["source:outline"] != undefined) {
1650 | 					if (jstruct ["source:outline"] instanceof Array) {
1651 | 						var theArray = jstruct ["source:outline"];
1652 | 						theNewOutline.subs = [];
1653 | 						for (var i = 0; i < theArray.length; i++) {
1654 | 							theNewOutline.subs [theNewOutline.subs.length] = newConvertOutline (theArray [i]);
1655 | 							}
1656 | 						}
1657 | 					else {
1658 | 						theNewOutline.subs = [
1659 | 							newConvertOutline (jstruct ["source:outline"])
1660 | 							];
1661 | 						}
1662 | 					}
1663 | 				return (theNewOutline);
1664 | 				}
1665 | 			function getString (s) {
1666 | 				if (s == null) {
1667 | 					s = "";
1668 | 					}
1669 | 				return (utils.stripMarkup (s));
1670 | 				}
1671 | 			function getDate (d) {
1672 | 				if (d == null) {
1673 | 					d = now;
1674 | 					}
1675 | 				return (new Date (d))
1676 | 				}
1677 | 			
1678 | 			item.title = getString (itemFromParser.title);
1679 | 			item.link = getString (itemFromParser.link);
1680 | 			item.description = getItemDescription (itemFromParser);
1681 | 			
1682 | 			//permalink -- updated 5/30/14 by DW
1683 | 				if (itemFromParser.permalink == undefined) {
1684 | 					item.permalink = "";
1685 | 					}
1686 | 				else {
1687 | 					item.permalink = itemFromParser.permalink;
1688 | 					}
1689 | 				
1690 | 			//enclosure -- 5/30/14 by DW
1691 | 				if (itemFromParser.enclosures != undefined) { //it's an array, we want the first one
1692 | 					item.enclosure = itemFromParser.enclosures [0];
1693 | 					}
1694 | 			//outline -- 6/14/17 by DW
1695 | 				if (itemFromParser.outline !== undefined) {
1696 | 					item.outline = itemFromParser.outline;
1697 | 					}
1698 | 				else {
1699 | 					if (itemFromParser ["source:outline"] != undefined) { 
1700 | 						item.outline = newConvertOutline (itemFromParser ["source:outline"]);
1701 | 						}
1702 | 					}
1703 | 			item.pubdate = getDate (itemFromParser.pubDate);
1704 | 			item.comments = getString (itemFromParser.comments);
1705 | 			item.feedUrl = urlfeed;
1706 | 			item.when = now; //6/7/15 by DW
1707 | 			item.aggregator = myProductName + " v" + myVersion;
1708 | 			item.id = serverStats.serialnum++; //5/28/14 by DW
1709 | 		if (config.flMaintainCalendarStructure) {
1710 | 			todaysRiver [todaysRiver.length] = item;
1711 | 			}
1712 | 		flRiverChanged = true;
1713 | 		//stats
1714 | 			serverStats.ctStoriesAdded++;
1715 | 			serverStats.ctStoriesAddedThisRun++;
1716 | 			serverStats.ctStoriesAddedToday++;
1717 | 			serverStats.whenLastStoryAdded = now;
1718 | 			serverStats.lastStoryAdded = item;
1719 | 		//show in console
1720 | 			var storyTitle = itemFromParser.title;
1721 | 			if (storyTitle == null) {
1722 | 				storyTitle = utils.maxStringLength (utils.stripMarkup (itemFromParser.description), 80);
1723 | 				}
1724 | 			myConsoleLog (getFeedTitle (urlfeed) + ": " + storyTitle);
1725 | 		//add the item to each of the lists it belongs to, and mark the river as changed
1726 | 			var feedstats = findInFeedsArray (urlfeed), listname;
1727 | 			if (feedstats !== undefined) {
1728 | 				for (var i = 0; i < feedstats.lists.length; i++) {
1729 | 					listname = feedstats.lists [i];
1730 | 					listChanged (listname);
1731 | 					addRiverItemToList (listname, item);
1732 | 					}
1733 | 				}
1734 | 		//add the item to the feed's river -- 6/29/17 by DW
1735 | 			addItemToFeedRiver (urlfeed, item, itemFromParser)
1736 | 		
1737 | 		downloadPodcast (item, urlfeed); //4/17/17 by DW
1738 | 		
1739 | 		if (config.newItemCallback !== undefined) { //5/17/17 by DW
1740 | 			config.newItemCallback (urlfeed, itemFromParser, item);
1741 | 			}
1742 | 		
1743 | 		callAddToRiverCallbacks (urlfeed, itemFromParser, item); //6/19/15 by DW
1744 | 		
1745 | 		notifyWebSocketListeners ("item " + utils.jsonStringify (item));
1746 | 		}
1747 | 	function getCalendarPath (d) {
1748 | 		if (d === undefined) {
1749 | 			d = dayRiverCovers;
1750 | 			}
1751 | 		return ("calendar/" + utils.getDatePath (d, false) + ".json");
1752 | 		}
1753 | 	function saveTodaysRiver (callback) {
1754 | 		if (config.flMaintainCalendarStructure) {
1755 | 			serverStats.ctRiverSaves++;
1756 | 			serverStats.whenLastRiverSave = new Date ();
1757 | 			flStatsChanged = true;
1758 | 			writeStats (getCalendarPath (), todaysRiver, callback);
1759 | 			}
1760 | 		}
1761 | 	function loadTodaysRiver (callback) {
1762 | 		if (config.flMaintainCalendarStructure) {
1763 | 			readStats (getCalendarPath (), todaysRiver, function () {
1764 | 				if (callback !== undefined) {
1765 | 					callback ();
1766 | 					}
1767 | 				});
1768 | 			}
1769 | 		else {
1770 | 			if (callback !== undefined) {
1771 | 				callback ();
1772 | 				}
1773 | 			}
1774 | 		}
1775 | 	function checkRiverRollover () { 
1776 | 		var now = new Date ();
1777 | 		function roll () {
1778 | 			if (config.flMaintainCalendarStructure) {
1779 | 				todaysRiver = new Array (); //clear it out
1780 | 				dayRiverCovers = now;
1781 | 				saveTodaysRiver (); 
1782 | 				}
1783 | 			serverStats.ctHitsToday = 0;
1784 | 			serverStats.ctFeedReadsToday = 0;
1785 | 			serverStats.ctStoriesAddedToday = 0;
1786 | 			flStatsChanged = true;
1787 | 			}
1788 | 		if (utils.secondsSince (serverStats.whenLastStoryAdded) >= 60) {
1789 | 			if (!utils.sameDay (now, dayRiverCovers)) { //rollover
1790 | 				if (flRiverChanged) {
1791 | 					saveTodaysRiver (roll);
1792 | 					}
1793 | 				else {
1794 | 					roll ();
1795 | 					}
1796 | 				}
1797 | 			}
1798 | 		}
1799 | 	function buildChangedRivers (callback) {
1800 | 		if (serverStats.listsThatChanged.length > 0) {
1801 | 			var listname = serverStats.listsThatChanged.shift (), whenstart = new Date ();
1802 | 			flStatsChanged = true;
1803 | 			buildOneRiver (listname, function () {
1804 | 				myConsoleLog ("buildChangedRivers: listname == " + listname + ", secs == " + utils.secondsSince (whenstart));
1805 | 				buildChangedRivers (callback);
1806 | 				});
1807 | 			}
1808 | 		else {
1809 | 			if (callback !== undefined) {
1810 | 				callback ();
1811 | 				}
1812 | 			}
1813 | 		}
1814 | 	function buildAllRivers () {
1815 | 		for (var i=0; i < serverStats.listNames.length; i++) {
1816 | 			listChanged (serverStats.listNames [i]);
1817 | 			}
1818 | 		buildChangedRivers ();
1819 | 		}
1820 | 	function getOneRiver (fname, callback) { 
1821 | 		var name = utils.stringPopLastField (fname, "."); //get rid of .opml extension if present
1822 | 		var f = config.riversFolder + name + ".js";
1823 | 		fs.readFile (f, function (err, data) {
1824 | 			if (err) {
1825 | 				myConsoleLog ("getOneRiver: f == " + f + ", err.message == " + err.message);
1826 | 				callback (undefined);
1827 | 				}
1828 | 			else {
1829 | 				callback (data.toString ());
1830 | 				}
1831 | 			});
1832 | 		}
1833 | //misc
1834 | 	function cleanFilenameForPlatform (s) {
1835 | 		switch (process.platform) {
1836 | 			case "win32":
1837 | 				s = utils.replaceAll (s, "/", "_");
1838 | 				s = utils.replaceAll (s, "?", "_");
1839 | 				s = utils.replaceAll (s, ":", "_");
1840 | 				s = utils.replaceAll (s, "<", "_");
1841 | 				s = utils.replaceAll (s, ">", "_");
1842 | 				s = utils.replaceAll (s, "\"", "_");
1843 | 				s = utils.replaceAll (s, "\\", "_");
1844 | 				s = utils.replaceAll (s, "|", "_");
1845 | 				s = utils.replaceAll (s, "*", "_");
1846 | 				break;
1847 | 			case "darwin":
1848 | 				s = utils.replaceAll (s, "/", ":");
1849 | 				break;
1850 | 			}
1851 | 		return (s);
1852 | 		}
1853 | 	function downloadBigFile (url, f, pubDate, callback) { //4/17/17 by DW
1854 | 		fsSureFilePath (f, function () {
1855 | 			var theStream = fs.createWriteStream (f);
1856 | 			theStream.on ("finish", function () {
1857 | 				console.log ("downloadBigFile: finished writing to f == " + f);
1858 | 				pubDate = new Date (pubDate);
1859 | 				fs.utimes (f, pubDate, pubDate, function () {
1860 | 					});
1861 | 				if (callback !== undefined) {
1862 | 					callback ();
1863 | 					}
1864 | 				});
1865 | 			request.get (url)
1866 | 				.on ('error', function (err) {
1867 | 					console.log (err);
1868 | 					})
1869 | 				.pipe (theStream);
1870 | 			});
1871 | 		}
1872 | 	function httpReadUrl (url, callback) { //11/16/16 by DW
1873 | 		request (url, function (error, response, body) {
1874 | 			if (!error && (response.statusCode == 200)) {
1875 | 				callback (body) 
1876 | 				}
1877 | 			});
1878 | 		}
1879 | 	function endsWithChar (s, chPossibleEndchar) {
1880 | 		if ((s === undefined) || (s.length == 0)) { 
1881 | 			return (false);
1882 | 			}
1883 | 		else {
1884 | 			return (s [s.length - 1] == chPossibleEndchar);
1885 | 			}
1886 | 		}
1887 | 	function fsSureFilePath (path, callback) { 
1888 | 		var splits = path.split ("/");
1889 | 		path = ""; //1/8/15 by DW
1890 | 		if (splits.length > 0) {
1891 | 			function doLevel (levelnum) {
1892 | 				if (levelnum < (splits.length - 1)) {
1893 | 					path += splits [levelnum] + "/";
1894 | 					fs.exists (path, function (flExists) {
1895 | 						if (flExists) {
1896 | 							doLevel (levelnum + 1);
1897 | 							}
1898 | 						else {
1899 | 							fs.mkdir (path, undefined, function () {
1900 | 								doLevel (levelnum + 1);
1901 | 								});
1902 | 							}
1903 | 						});
1904 | 					}
1905 | 				else {
1906 | 					if (callback != undefined) {
1907 | 						callback ();
1908 | 						}
1909 | 					}
1910 | 				}
1911 | 			doLevel (0);
1912 | 			}
1913 | 		else {
1914 | 			if (callback != undefined) {
1915 | 				callback ();
1916 | 				}
1917 | 			}
1918 | 		}
1919 | 	function saveStats () {
1920 | 		serverStats.ctStatsSaves++;
1921 | 		serverStats.whenLastStatsSave = new Date ();
1922 | 		writeStats (config.statsFilePath, serverStats);
1923 | 		}
1924 | 	function getFeedMetadata (url, callback) { //12/1/14 by DW
1925 | 		var req = myRequestCall (url), feedparser = new FeedParser ();
1926 | 		req.on ("response", function (res) {
1927 | 			var stream = this;
1928 | 			if (res.statusCode == 200) {
1929 | 				stream.pipe (feedparser);
1930 | 				}
1931 | 			else {
1932 | 				callback (undefined);
1933 | 				}
1934 | 			});
1935 | 		req.on ("error", function (res) {
1936 | 			callback (undefined);
1937 | 			});
1938 | 		feedparser.on ("readable", function () {
1939 | 			var item = this.read ();
1940 | 			callback (item.meta);    
1941 | 			});
1942 | 		feedparser.on ("end", function () {
1943 | 			callback (undefined);
1944 | 			});
1945 | 		feedparser.on ("error", function () {
1946 | 			callback (undefined);
1947 | 			});
1948 | 		}
1949 | //rsscloud
1950 | 	function pleaseNotify (urlServer, domain, port, path, urlFeed, feedstats, callback) { //6/4/15 by DW
1951 | 		var now = new Date ();
1952 | 		var theRequest = {
1953 | 			url: urlServer,
1954 | 			followRedirect: true, 
1955 | 			headers: {Accept: "application/json"},
1956 | 			method: "POST",
1957 | 			form: {
1958 | 				port: port,
1959 | 				path: path,
1960 | 				url1: urlFeed,
1961 | 				protocol: "http-post"
1962 | 				}
1963 | 			};
1964 | 		
1965 | 		myConsoleLog ("pleaseNotify: urlFeed == " + urlFeed);
1966 | 		feedstats.whenLastCloudRenew = now;
1967 | 		feedstats.ctCloudRenew++;
1968 | 		flFeedsArrayChanged = true; //because we modified feedstats
1969 | 		
1970 | 		request (theRequest, function (err, response, body) {
1971 | 			function recordErrorStats (message) {
1972 | 				feedstats.ctCloudRenewErrors++; //counts the number of communication errors
1973 | 				feedstats.ctConsecutiveCloudRenewErrors++;
1974 | 				feedstats.whenLastCloudRenewError = now;
1975 | 				feedstats.lastCloudRenewError = message;
1976 | 				flFeedsArrayChanged = true; 
1977 | 				}
1978 | 			try {
1979 | 				var flskip = false;
1980 | 				
1981 | 				if (err) {
1982 | 					flskip = true;
1983 | 					if (callback) {
1984 | 						callback (err.message);
1985 | 						}
1986 | 					}
1987 | 				else {
1988 | 					if (!body.success) {
1989 | 						flskip = true;
1990 | 						if (callback) {
1991 | 							callback (body.msg);
1992 | 							}
1993 | 						}
1994 | 					}
1995 | 				
1996 | 				if (flskip) {
1997 | 					recordErrorStats (err.message);
1998 | 					}
1999 | 				else {
2000 | 					feedstats.ctConsecutiveCloudRenewErrors = 0;
2001 | 					flFeedsArrayChanged = true; //because we modified feedstats
2002 | 					if (callback) {
2003 | 						callback ("It worked.");
2004 | 						}
2005 | 					}
2006 | 				}
2007 | 			catch (err) {
2008 | 				recordErrorStats (err.message);
2009 | 				if (callback) {
2010 | 					callback (err.message);
2011 | 					}
2012 | 				}
2013 | 			});
2014 | 		}
2015 | 	function renewNextSubscription () { //6/4/15 by DW
2016 | 		if (config.flRequestCloudNotify && config.flHttpEnabled) {
2017 | 			var theFeed;
2018 | 			for (var i = 0; i < feedsArray.length; i++) {
2019 | 				theFeed = feedsArray [i];
2020 | 				if (theFeed.cloud !== undefined) {
2021 | 					if (utils.secondsSince (theFeed.whenLastCloudRenew) > (23 * 60 * 60)) { //ready to be renewed
2022 | 						var urlCloudServer = "http://" + theFeed.cloud.domain + ":" + theFeed.cloud.port + theFeed.cloud.path;
2023 | 						
2024 | 						serverStats.ctRssCloudRenews++;
2025 | 						serverStats.whenLastRssCloudRenew = new Date ();
2026 | 						flStatsChanged = true;
2027 | 						
2028 | 						pleaseNotify (urlCloudServer, undefined, config.httpPort, "/feedupdated", theFeed.url, theFeed, function () {
2029 | 							});
2030 | 						return; //we renew at most one each time we're called
2031 | 						}
2032 | 					}
2033 | 				}
2034 | 			}
2035 | 		}
2036 | 	function rssCloudFeedUpdated (urlFeed) { //6/4/15 by DW
2037 | 		var feedstats = findInFeedsArray (urlFeed);
2038 | 		if (feedstats === undefined) {
2039 | 			myConsoleLog ("\nrssCloudFeedUpdated: url == " + urlFeed + ", but we're not subscribed to this feed, so it wasn't read.\n");
2040 | 			}
2041 | 		else {
2042 | 			var now = new Date ();
2043 | 			serverStats.whenLastRssCloudUpdate = now;
2044 | 			serverStats.ctRssCloudUpdates++;
2045 | 			serverStats.urlFeedLastCloudUpdate = urlFeed;
2046 | 			flStatsChanged = true;
2047 | 			myConsoleLog ("\nrssCloudFeedUpdated: " + urlFeed);
2048 | 			readFeedIfSubscribed (urlFeed, function () {
2049 | 				});
2050 | 			}
2051 | 		}
2052 | 	function renewThisFeedNow (urlFeed, callback) { //6/14/17 by DW
2053 | 		var theFeed = findInFeedsArray (urlFeed); 
2054 | 		if (theFeed.cloud === undefined) {
2055 | 			if (callback !== undefined) {
2056 | 				callback ("Can't renew the subscription because the feed, \"" + urlFeed + "\" is not cloud-aware.");
2057 | 				}
2058 | 			}
2059 | 		else {
2060 | 			var urlCloudServer = "http://" + theFeed.cloud.domain + ":" + theFeed.cloud.port + theFeed.cloud.path;
2061 | 			pleaseNotify (urlCloudServer, undefined, config.httpPort, "/feedupdated", theFeed.url, theFeed, function (message) {
2062 | 				if (callback !== undefined) {
2063 | 					callback ("We sent the request to " + urlCloudServer + ".");
2064 | 					}
2065 | 				});
2066 | 			}
2067 | 		}
2068 | //callbacks
2069 | 	var localStorage = {
2070 | 		};
2071 | 	var lastLocalStorageJson = "";
2072 | 	
2073 | 	function loadLocalStorage (callback) {
2074 | 		readFile (config.localStoragePath, function (data) {
2075 | 			if (data !== undefined) {
2076 | 				try {
2077 | 					var s = data.toString ();
2078 | 					localStorage = JSON.parse (s);
2079 | 					lastLocalStorageJson = s; 
2080 | 					}
2081 | 				catch (err) {
2082 | 					myConsoleLog ("loadLocalStorage: error reading localStorage == " + err.message);
2083 | 					}
2084 | 				}
2085 | 			if (callback != undefined) {
2086 | 				callback ();
2087 | 				}
2088 | 			});
2089 | 		}
2090 | 	function writeLocalStorageIfChanged () {
2091 | 		var s = utils.jsonStringify (localStorage);
2092 | 		if (s != lastLocalStorageJson) {
2093 | 			lastLocalStorageJson = s;
2094 | 			writeFile (config.localStoragePath, s);
2095 | 			}
2096 | 		}
2097 | 	function todaysRiverChanged () { //6/21/15 by DW -- callback scripts, call this to be sure your changes get saved
2098 | 		flRiverChanged = true;
2099 | 		}
2100 | 	function runUserScript (s, dataforscripts, scriptName) {
2101 | 		try {
2102 | 			if (dataforscripts !== undefined) {
2103 | 				with (dataforscripts) {
2104 | 					eval (s);
2105 | 					}
2106 | 				}
2107 | 			else {
2108 | 				eval (s);
2109 | 				}
2110 | 			}
2111 | 		catch (err) {
2112 | 			myConsoleLog ("runUserScript: error running \"" + scriptName + "\" == " + err.message);
2113 | 			}
2114 | 		}
2115 | 	function runScriptsInFolder (path, dataforscripts, callback) {
2116 | 		fsSureFilePath (path, function () {
2117 | 			fs.readdir (path, function (err, list) {
2118 | 				if (list !== undefined) { //3/29/17 by DW
2119 | 					for (var i = 0; i < list.length; i++) {
2120 | 						var fname = list [i];
2121 | 						if (utils.endsWith (fname.toLowerCase (), ".js")) {
2122 | 							var f = path + fname;
2123 | 							fs.readFile (f, function (err, data) {
2124 | 								if (err) {
2125 | 									myConsoleLog ("runScriptsInFolder: error == " + err.message);
2126 | 									}
2127 | 								else {
2128 | 									runUserScript (data.toString (), dataforscripts, f);
2129 | 									}
2130 | 								});
2131 | 							}
2132 | 						}
2133 | 					}
2134 | 				if (callback != undefined) {
2135 | 					callback ();
2136 | 					}
2137 | 				});
2138 | 			});
2139 | 		}
2140 | 	function callAddToRiverCallbacks (urlfeed, itemFromParser, itemFromRiver) {
2141 | 		var dataforscripts = {
2142 | 			urlfeed: urlfeed,
2143 | 			itemFromParser: itemFromParser,
2144 | 			itemFromRiver: itemFromRiver
2145 | 			};
2146 | 		runScriptsInFolder (config.addToRiverCallbacksFolder, dataforscripts, function () {
2147 | 			});
2148 | 		}
2149 | 	function callBuildRiverCallbacks (fname, jsontext) {
2150 | 		var dataforscripts = {
2151 | 			fname: fname,
2152 | 			jsontext: jsontext
2153 | 			};
2154 | 		runScriptsInFolder (config.buildRiverCallbacksFolder, dataforscripts, function () {
2155 | 			});
2156 | 		}
2157 | //websockets
2158 | 	var theWsServer;
2159 | 	
2160 | 	function countOpenSockets () {
2161 | 		if (theWsServer === undefined) { //12/18/15 by DW
2162 | 			return (0);
2163 | 			}
2164 | 		else {
2165 | 			return (theWsServer.connections.length);
2166 | 			}
2167 | 		}
2168 | 	
2169 | 	function notifyWebSocketListeners (s) {
2170 | 		if (theWsServer !== undefined) {
2171 | 			var ctUpdates = 0;
2172 | 			for (var i = 0; i < theWsServer.connections.length; i++) {
2173 | 				var conn = theWsServer.connections [i];
2174 | 				if (conn.riverServerData !== undefined) { //it's one of ours
2175 | 					try {
2176 | 						conn.sendText (s);
2177 | 						ctUpdates++;
2178 | 						}
2179 | 					catch (err) {
2180 | 						}
2181 | 					}
2182 | 				}
2183 | 			}
2184 | 		if (config.notifyListenersCallback !== undefined) { //3/25/17 by DW
2185 | 			config.notifyListenersCallback (s);
2186 | 			}
2187 | 		}
2188 | 	function handleWebSocketConnection (conn) { 
2189 | 		var now = new Date ();
2190 | 		
2191 | 		function logToConsole (conn, verb, value) {
2192 | 			getDomainName (conn.socket.remoteAddress, function (theName) { //log the request
2193 | 				var freemem = gigabyteString (os.freemem ()), method = "WS:" + verb, now = new Date (); 
2194 | 				if (theName === undefined) {
2195 | 					theName = conn.socket.remoteAddress;
2196 | 					}
2197 | 				myConsoleLog (now.toLocaleTimeString () + " " + freemem + " " + method + " " + value + " " + theName);
2198 | 				conn.chatLogData.domain = theName; 
2199 | 				});
2200 | 			}
2201 | 		
2202 | 		conn.riverServerData = {
2203 | 			whenStarted: now
2204 | 			};
2205 | 		conn.on ("text", function (s) {
2206 | 			
2207 | 			});
2208 | 		conn.on ("close", function () {
2209 | 			});
2210 | 		conn.on ("error", function (err) {
2211 | 			});
2212 | 		}
2213 | 	function startWebSocketServer () {
2214 | 		if (config.flWebSocketEnabled) {
2215 | 			if (config.webSocketPort !== undefined) { 
2216 | 				myConsoleLog ("startWebSocketServer: websockets port is " + config.webSocketPort);
2217 | 				try {
2218 | 					theWsServer = websocket.createServer (handleWebSocketConnection);
2219 | 					theWsServer.listen (config.webSocketPort);
2220 | 					}
2221 | 				catch (err) {
2222 | 					myConsoleLog ("startWebSocketServer: err.message == " + err.message);
2223 | 					}
2224 | 				}
2225 | 			}
2226 | 		}
2227 | 	
2228 | 	
2229 | //http server
2230 | 	function getServerStatsJson () { //3/25/17by DW
2231 | 		serverStats.ctSecsSinceLastStart = utils.secondsSince (serverStats.whenLastStart); 
2232 | 		serverStats.ctSecsSinceLastFeedReed = utils.secondsSince (serverStats.whenLastFeedRead); 
2233 | 		return (utils.jsonStringify (serverStats, true)); 
2234 | 		}
2235 | 	function returnThroughTemplate (htmltext, title, callback) {
2236 | 		fs.readFile (config.templatePath, function (err, data) {
2237 | 			var templatetext;
2238 | 			if (err) {
2239 | 				myConsoleLog ("returnThroughTemplate: error reading config.templatePath == " + config.templatePath + ", err.message == " + err.message);
2240 | 				templatetext = "";
2241 | 				}
2242 | 			else {
2243 | 				templatetext = data.toString ();
2244 | 				}
2245 | 			var pagetable = {
2246 | 				text: htmltext,
2247 | 				title: title
2248 | 				};
2249 | 			var pagetext = utils.multipleReplaceAll (templatetext, pagetable, false, "[%", "%]");
2250 | 			callback (pagetext);
2251 | 			});
2252 | 		}
2253 | 	function viewFeedList (callback) {
2254 | 		var htmltext = "", indentlevel = 0;
2255 | 		function dateString (d) {
2256 | 			d = new Date (d);
2257 | 			return ((d.getMonth () + 1) + "/" + d.getDate ());
2258 | 			}
2259 | 		function add (s) {
2260 | 			htmltext += utils.filledString ("\t", indentlevel) + s + "\n";
2261 | 			}
2262 | 		add ("<table class=\"feedTable\">"); indentlevel++;
2263 | 		
2264 | 		//column titles
2265 | 			add ("<tr>"); indentlevel++;
2266 | 			add ("<td class=\"tdFeedTitle\"><b>Title</b></td>");
2267 | 			add ("<td class=\"tdFeedCt\"><b>Stories</b></td>");
2268 | 			add ("<td class=\"tdFeedDate\"><b>When</b></td>");
2269 | 			add ("<td class=\"tdFeedCt\"><b>Reads</b></td>");
2270 | 			add ("<td class=\"tdFeedDate\"><b>When</b></td>");
2271 | 			add ("</tr>"); indentlevel--;
2272 | 		
2273 | 		for (var i = 0; i < feedsArray.length; i++) {
2274 | 			var item = feedsArray [i], title = item.title;
2275 | 			var urlFeedPage = "feed?url=" + encodeURIComponent (item.url);
2276 | 			//set title
2277 | 				if ((title === undefined) || (title === null)) {
2278 | 					title = "No title";
2279 | 					}
2280 | 				else {
2281 | 					title = utils.maxStringLength (title, 40);
2282 | 					}
2283 | 			add ("<tr>"); indentlevel++;
2284 | 			add ("<td class=\"tdFeedTitle\"><a href=\"" + urlFeedPage + "\">" + title + "</a></td>");
2285 | 			add ("<td class=\"tdFeedCt\">" + item.ctItems + "</td>");
2286 | 			add ("<td class=\"tdFeedDate\">" + dateString (item.whenLastNewItem) + "</td>");
2287 | 			add ("<td class=\"tdFeedCt\">" + item.ctReads + "</td>");
2288 | 			add ("<td class=\"tdFeedDate\">" + dateString (item.whenLastRead) + "</td>");
2289 | 			add ("</tr>"); indentlevel--;
2290 | 			}
2291 | 		add ("</table>"); indentlevel--;
2292 | 		returnThroughTemplate (htmltext, "Feed List", callback);
2293 | 		}
2294 | 	function viewFeed (urlfeed, callback) {
2295 | 		initFeed (urlfeed, function (feed) {
2296 | 			var htmltext = "", indentlevel = 0;
2297 | 			function add (s) {
2298 | 				htmltext += utils.filledString ("\t", indentlevel) + s + "\n";
2299 | 				}
2300 | 			function viewDate (d) {
2301 | 				var s = utils.viewDate (d);
2302 | 				if (s == "Wednesday, December 31, 1969") {
2303 | 					return ("");
2304 | 					}
2305 | 				return (s);
2306 | 				
2307 | 				}
2308 | 			function viewDescription () {
2309 | 				if (feed.feedInfo.description == null) {
2310 | 					return ("");
2311 | 					}
2312 | 				else {
2313 | 					return (feed.feedInfo.description);
2314 | 					}
2315 | 				}
2316 | 			
2317 | 			add ("<div class=\"divFeedPageTop\">"); indentlevel++;
2318 | 			add ("<div class=\"divFeedTitle\"><a href=\"" + feed.feedInfo.link + "\">" + feed.feedInfo.title + "</a></div>");
2319 | 			add ("<div class=\"divFeedDescription\">" + viewDescription () + "</div>");
2320 | 			add ("<div class=\"divFeedUrl\"><a href=\"" + feed.prefs.url + "\">" + feed.prefs.url + "</a></div>");
2321 | 			add ("</div>"); indentlevel--;
2322 | 			
2323 | 			add ("<table class=\"feedTable\">"); indentlevel++;
2324 | 			for (var i = 0; i < feed.history.length; i++) {
2325 | 				var item = feed.history [i];
2326 | 				add ("<tr>"); indentlevel++;
2327 | 				add ("<td class=\"tdFeedTitle\"><a href=\"" + item.link + "\" title=\"" + item.description + "\">" + item.title + "</a></td>");
2328 | 				add ("<td class=\"tdFeedDate\">" + utils.viewDate (item.when) + "</td>");
2329 | 				add ("</tr>"); indentlevel--;
2330 | 				}
2331 | 			add ("</table>"); indentlevel--;
2332 | 			
2333 | 			feed.stats.whenSubscribed = viewDate (feed.stats.whenSubscribed);
2334 | 			feed.stats.whenLastRead = viewDate (feed.stats.whenLastRead);
2335 | 			feed.stats.whenLastNewItem = viewDate (feed.stats.whenLastNewItem);
2336 | 			feed.stats.mostRecentPubDate = viewDate (feed.stats.mostRecentPubDate);
2337 | 			feed.stats.whenLastInfoWrite = viewDate (feed.stats.whenLastInfoWrite);
2338 | 			feed.stats.whenLastReadError = viewDate (feed.stats.whenLastReadError);
2339 | 			feed.stats.whenLastInfoRead = viewDate (feed.stats.whenLastInfoRead);
2340 | 			
2341 | 			add ("<div class=\"divFeedStatsJson\"><pre>" + utils.jsonStringify (feed.stats) + "</pre></div>");
2342 | 			
2343 | 			returnThroughTemplate (htmltext, "Feed", callback);
2344 | 			});
2345 | 		}
2346 | 	function configToJsonText () { //remove items whose name contains "password"
2347 | 		var theCopy = new Object ();
2348 | 		for (var x in config) {
2349 | 			if (!utils.stringContains (x, "password")) {
2350 | 				theCopy [x] = config [x];
2351 | 				}
2352 | 			}
2353 | 		return (utils.jsonStringify (theCopy));
2354 | 		}
2355 | 	function handleHttpRequest (httpRequest, httpResponse) {
2356 | 		function httpWriteHead (code, headers) { //4/29/18 by DW
2357 | 			headers ["User-Agent"] = config.httpUserAgent;
2358 | 			httpResponse.writeHead (code, headers);
2359 | 			}
2360 | 		function doHttpReturn (code, type, val) {
2361 | 			httpWriteHead (code, {"Content-Type": type});
2362 | 			httpResponse.end (val.toString ());    
2363 | 			}
2364 | 		function returnHtml (htmltext) {
2365 | 			httpWriteHead (200, {"Content-Type": "text/html"});
2366 | 			httpResponse.end (htmltext);    
2367 | 			}
2368 | 		function returnText (theText, flAnyOrigin) {
2369 | 			function getHeaders (type, flAnyOrigin) {
2370 | 				var headers = {"Content-Type": type};
2371 | 				if (flAnyOrigin) {
2372 | 					headers ["Access-Control-Allow-Origin"] = "*";
2373 | 					}
2374 | 				return (headers);
2375 | 				}
2376 | 			httpWriteHead (200, getHeaders ("text/plain", flAnyOrigin));
2377 | 			httpResponse.end (theText);    
2378 | 			}
2379 | 		function return404 (msgIfAny) {
2380 | 			function getHeaders (type) {
2381 | 				var headers = {"Content-Type": type};
2382 | 				return (headers);
2383 | 				}
2384 | 			httpWriteHead (404, getHeaders ("text/plain"));
2385 | 			if (msgIfAny !== undefined) {
2386 | 				httpResponse.end (msgIfAny);    
2387 | 				}
2388 | 			else {
2389 | 				httpResponse.end ("Not found");    
2390 | 				}
2391 | 			}
2392 | 		function returnRedirect (url, code) {
2393 | 			if (code === undefined) {
2394 | 				code = 302;
2395 | 				}
2396 | 			httpWriteHead (code, {"location": url, "Content-Type": "text/plain"});
2397 | 			httpResponse.end (code + " REDIRECT");    
2398 | 			}
2399 | 			
2400 | 		function returnError (message, code) {
2401 | 			if (code === undefined) {
2402 | 				code = 500;
2403 | 				}
2404 | 			httpWriteHead (code, {"location": url, "Content-Type": "text/plain"});
2405 | 			httpResponse.end (message);    
2406 | 			}
2407 | 			
2408 | 		function stringMustBeFilename (s, callback) {
2409 | 			if (utils.stringContains (s, "/")) {
2410 | 				returnError ("Illegal file name.", 403);
2411 | 				}
2412 | 			else {
2413 | 				callback ();
2414 | 				}
2415 | 			}
2416 | 		function writeHead (type) {
2417 | 			if (type == undefined) {
2418 | 				type = "text/plain";
2419 | 				}
2420 | 			httpWriteHead (200, {"Content-Type": type, "Access-Control-Allow-Origin": "*"});
2421 | 			}
2422 | 		function respondWithObject (obj) {
2423 | 			writeHead ("application/json");
2424 | 			httpResponse.end (utils.jsonStringify (obj));    
2425 | 			}
2426 | 		function returnServerHomePage () {
2427 | 			request (config.urlServerHomePageSource, function (error, response, templatetext) {
2428 | 				if (!error && response.statusCode == 200) {
2429 | 					var pagetable = {
2430 | 						config: configToJsonText (),
2431 | 						version: myVersion
2432 | 						};
2433 | 					var pagetext = utils.multipleReplaceAll (templatetext, pagetable, false, "[%", "%]");
2434 | 					returnHtml (pagetext);
2435 | 					}
2436 | 				});
2437 | 			
2438 | 			}
2439 | 		function returnFeedViewerPage () {
2440 | 			request (config.urlFeedViewerApp, function (error, response, templatetext) {
2441 | 				if (!error && response.statusCode == 200) {
2442 | 					var pagetable = {
2443 | 						config: configToJsonText (),
2444 | 						version: myVersion
2445 | 						};
2446 | 					var pagetext = utils.multipleReplaceAll (templatetext, pagetable, false, "[%", "%]");
2447 | 					returnHtml (pagetext);
2448 | 					}
2449 | 				});
2450 | 			}
2451 | 		function handleRequestLocally () {
2452 | 			switch (httpRequest.method) {
2453 | 				case "GET":
2454 | 					switch (lowerpath) {
2455 | 						case "/": //7/4/15 by DW
2456 | 							returnServerHomePage ();
2457 | 							break;
2458 | 						case "/feedviewer": //4/13/18 by DW
2459 | 							returnFeedViewerPage ();
2460 | 							break;
2461 | 						case "/version":
2462 | 							returnText (myVersion);
2463 | 							break;
2464 | 						case "/now":
2465 | 							returnText (now.toString ());    
2466 | 							break;
2467 | 						case "/stats": case "/serverdata":
2468 | 							returnText (getServerStatsJson (), true); //11/16/16 by DW -- set flAnyOrigin boolean
2469 | 							break;
2470 | 						case "/feedstats":
2471 | 							returnText (utils.jsonStringify (feedsArray, true));    
2472 | 							break;
2473 | 						case "/buildallrivers":
2474 | 							if (config.enabled) {
2475 | 								buildAllRivers ();
2476 | 								returnText ("Your rivers are building sir or madam.");    
2477 | 								}
2478 | 							else {
2479 | 								returnText ("Can't build the rivers because config.enabled is false.");    
2480 | 								}
2481 | 							break;
2482 | 						case "/loadlists":
2483 | 							loadListsFromFolder ();
2484 | 							returnText ("We're reading the lists, right now, as we speak.");    
2485 | 						case "/dashboard": 
2486 | 							request (config.urlDashboardSource, function (error, response, htmltext) {
2487 | 								if (!error && response.statusCode == 200) {
2488 | 									returnHtml (htmltext);    
2489 | 									}
2490 | 								});
2491 | 							break;
2492 | 						case "/ping": 
2493 | 							var url = parsedUrl.query.url;
2494 | 							if (url === undefined) {
2495 | 								returnText ("Ping received, but no url param was specified, so we couldn't do anything with it. Sorry.");    
2496 | 								}
2497 | 							else {
2498 | 								if (findInFeedsArray (url) === undefined) {
2499 | 									returnText ("Ping received, but we're not following this feed. Sorry.");    
2500 | 									}
2501 | 								else {
2502 | 									returnText ("Ping received, will read asap.");    
2503 | 									readFeedIfSubscribed (url, function () {
2504 | 										myConsoleLog ("Feed read.");
2505 | 										});
2506 | 									}
2507 | 								}
2508 | 							break;
2509 | 						case "/getlistnames": //11/11/14 by DW
2510 | 							httpWriteHead (200, {"Content-Type": "text/plain", "Access-Control-Allow-Origin": "*"});
2511 | 							httpResponse.end (utils.jsonStringify (serverStats.listNames));    
2512 | 							break;
2513 | 						case "/getalllists": 
2514 | 							getAllLists (function (theLists) {
2515 | 								returnText (utils.jsonStringify (theLists), true);
2516 | 								});
2517 | 							break;
2518 | 						case "/getonefeed":
2519 | 							getOneFeed (parsedUrl.query.url, function (theFeed) {
2520 | 								returnText (utils.jsonStringify (theFeed), true);    
2521 | 								});
2522 | 							break;
2523 | 						case "/getfeedriver": //6/29/17 by DW
2524 | 							getFeedRiverForServer (parsedUrl.query.url, function (jstruct) {
2525 | 								returnText (utils.jsonStringify (jstruct), true);    
2526 | 								});
2527 | 							break;
2528 | 						case "/getoneriver": //11/28/14 by DW 
2529 | 							getOneRiver (parsedUrl.query.fname, function (s) {
2530 | 								returnText (s, true);
2531 | 								});
2532 | 							break;
2533 | 						case "/getonelist": //2/3/16 by DW
2534 | 							var fname = parsedUrl.query.fname;
2535 | 							stringMustBeFilename (fname, function () {
2536 | 								getOneList (fname, function (s) {
2537 | 									if (s === undefined) {
2538 | 										return404 ();
2539 | 										}
2540 | 									else {
2541 | 										returnText (s, true);
2542 | 										}
2543 | 									});
2544 | 								});
2545 | 							break;
2546 | 						case "/getfeedmeta": //12/1/14 by DW -- for the list editor, just get the metadata about the feed
2547 | 							httpWriteHead (200, {"Content-Type": "text/plain", "Access-Control-Allow-Origin": "*"});
2548 | 							getFeedMetadata (parsedUrl.query.url, function (data) {
2549 | 								if (data == undefined) {
2550 | 									httpResponse.end ("");    
2551 | 									}
2552 | 								else {
2553 | 									httpResponse.end (utils.jsonStringify (data));    
2554 | 									}
2555 | 								});
2556 | 							break;
2557 | 						case "/readfile": //12/1/14 by DW
2558 | 							httpWriteHead (200, {"Content-Type": "text/plain", "Access-Control-Allow-Origin": "*"});
2559 | 							httpReadUrl (parsedUrl.query.url, function (s) { //xxx
2560 | 								if (s == undefined) {
2561 | 									httpResponse.end ("");    
2562 | 									}
2563 | 								else {
2564 | 									httpResponse.end (s);    
2565 | 									}
2566 | 								});
2567 | 							break;
2568 | 						case "/getprefs": //12/1/14 by DW
2569 | 							respondWithObject (config);
2570 | 							break;
2571 | 						case "/feedupdated": //6/4/15 by DW
2572 | 							var challenge = parsedUrl.query.challenge;
2573 | 							myConsoleLog ("/feedupdated: challenge == " + challenge);
2574 | 							httpWriteHead (200, {"Content-Type": "text/plain"});
2575 | 							httpResponse.end (challenge);    
2576 | 							break;
2577 | 						case "/renewfeed": //6/14/17 by DW
2578 | 							var url = parsedUrl.query.url;
2579 | 							renewThisFeedNow (parsedUrl.query.url, function (message) {
2580 | 								returnText (message);
2581 | 								});
2582 | 							break;
2583 | 						case "/favicon.ico": //7/19/15 by DW
2584 | 							returnRedirect (config.urlFavicon);
2585 | 							break;
2586 | 						
2587 | 						case "/feedlist": //1/27/16 by DW
2588 | 							viewFeedList (function (s) {
2589 | 								returnHtml (s);
2590 | 								});
2591 | 							break;
2592 | 						case "/feed": //1/27/16 by DW
2593 | 							var url = parsedUrl.query.url;
2594 | 							viewFeed (url, function (s) {
2595 | 								returnHtml (s);
2596 | 								});
2597 | 							break;
2598 | 						case "/test": //1/28/16 by DW
2599 | 							var theFeed = findInFeedsArray ("http://scripting.com/rss.xml");
2600 | 							returnText (utils.jsonStringify (theFeed));
2601 | 							
2602 | 							
2603 | 							
2604 | 							
2605 | 							
2606 | 							break;
2607 | 						
2608 | 						default: //404 not found
2609 | 							httpWriteHead (404, {"Content-Type": "text/plain", "Access-Control-Allow-Origin": "*"});
2610 | 							httpResponse.end ("\"" + lowerpath + "\" is not one of the endpoints defined by this server.");
2611 | 						}
2612 | 					break;
2613 | 				case "POST": //12/2/14 by DW
2614 | 					var body = "";
2615 | 					httpRequest.on ("data", function (data) {
2616 | 						body += data;
2617 | 						});
2618 | 					httpRequest.on ("end", function () {
2619 | 						var flPostAllowed = false;
2620 | 						
2621 | 						//set flPostAllowed -- 12/4/14 by DW
2622 | 							if (flLocalRequest) {
2623 | 								flPostAllowed = true;
2624 | 								}
2625 | 							else {
2626 | 								if (lowerpath == "/feedupdated") {
2627 | 									flPostAllowed = true;
2628 | 									}
2629 | 								else {
2630 | 									if (config.remotePassword.length > 0) { //must have password set
2631 | 										flPostAllowed = (parsedUrl.query.password === config.remotePassword);
2632 | 										}
2633 | 									}
2634 | 								}
2635 | 						if (flPostAllowed) {
2636 | 							myConsoleLog ("POST body length: " + body.length);
2637 | 							switch (lowerpath) {
2638 | 								case "/savelist": 
2639 | 									var listname = parsedUrl.query.listname;
2640 | 									stringMustBeFilename (listname, function () {
2641 | 										saveSubscriptionList (listname, body);
2642 | 										returnText ("", true);    
2643 | 										});
2644 | 									break;
2645 | 								case "/feedupdated": //6/4/15 by DW
2646 | 									var postbody = qs.parse (body);
2647 | 									rssCloudFeedUpdated (postbody.url);
2648 | 									httpWriteHead (200, {"Content-Type": "text/plain"});
2649 | 									httpResponse.end ("Thanks for the update! :-)");    
2650 | 									break;
2651 | 								default: //404 not found
2652 | 									httpWriteHead (404, {"Content-Type": "text/plain", "Access-Control-Allow-Origin": "*"});
2653 | 									httpResponse.end ("\"" + lowerpath + "\" is not one of the endpoints defined by this server.");
2654 | 								}
2655 | 							}
2656 | 						else {
2657 | 							httpWriteHead (403, {"Content-Type": "text/plain", "Access-Control-Allow-Origin": "*"});
2658 | 							httpResponse.end ("This feature can only be accessed locally.");    
2659 | 							}
2660 | 						});
2661 | 					break;
2662 | 				}
2663 | 			}
2664 | 		try {
2665 | 			var parsedUrl = urlpack.parse (httpRequest.url, true), now = new Date (), startTime = now;
2666 | 			var lowerpath = parsedUrl.pathname.toLowerCase (), host, port = 80, flLocalRequest = false;
2667 | 			
2668 | 			//set host, port, flLocalRequest
2669 | 				host = httpRequest.headers.host;
2670 | 				if (utils.stringContains (host, ":")) {
2671 | 					port = utils.stringNthField (host, ":", 2);
2672 | 					host = utils.stringNthField (host, ":", 1);
2673 | 					}
2674 | 				flLocalRequest = utils.beginsWith (host, "localhost");
2675 | 			//show the request on the console
2676 | 				var localstring = "";
2677 | 				if (flLocalRequest) {
2678 | 					localstring = "* ";
2679 | 					}
2680 | 				myConsoleLog (localstring + httpRequest.method + " " + host + ":" + port + " " + lowerpath);
2681 | 			
2682 | 			//stats
2683 | 				serverStats.ctHits++;
2684 | 				serverStats.ctHitsToday++;
2685 | 				serverStats.ctHitsThisRun++;
2686 | 			
2687 | 			if (config.handleHttpRequestCallback !== undefined) {
2688 | 				var myRequest = { //bundle things up for the callback
2689 | 					method: httpRequest.method,
2690 | 					path: parsedUrl.pathname,
2691 | 					lowerpath: lowerpath,
2692 | 					params: {},
2693 | 					host: host,
2694 | 					lowerhost: host.toLowerCase (),
2695 | 					port: port,
2696 | 					referrer: undefined,
2697 | 					flLocalRequest: flLocalRequest,
2698 | 					client: httpRequest.connection.remoteAddress,
2699 | 					now: new Date (),
2700 | 					sysRequest: httpRequest,
2701 | 					sysResponse: httpResponse,
2702 | 					httpReturn: doHttpReturn
2703 | 					};
2704 | 				for (var x in parsedUrl.query) {
2705 | 					myRequest.params [x] = parsedUrl.query [x];
2706 | 					}
2707 | 				config.handleHttpRequestCallback (myRequest, function (flConsumed) {
2708 | 					if (!flConsumed) {
2709 | 						handleRequestLocally ();
2710 | 						}
2711 | 					});
2712 | 				}
2713 | 			else {
2714 | 				handleRequestLocally ();
2715 | 				}
2716 | 			}
2717 | 		catch (tryError) {
2718 | 			httpWriteHead (503, {"Content-Type": "text/plain", "Access-Control-Allow-Origin": "*"});
2719 | 			httpResponse.end (tryError.message);    
2720 | 			}
2721 | 		}
2722 | 	function startHttpServer () {
2723 | 		if (config.flHttpEnabled) {
2724 | 			try {
2725 | 				http.createServer (handleHttpRequest).listen (config.httpPort);
2726 | 				}
2727 | 			catch (err) {
2728 | 				myConsoleLog ("startHttpServer: err.message == " + err.message);
2729 | 				}
2730 | 			}
2731 | 		}
2732 | //background processes
2733 | 	function everyQuarterSecond () {
2734 | 		if (config.enabled) {
2735 | 			findNextFeedToRead (function (urlFeed) {
2736 | 				readFeed (urlFeed, function () {
2737 | 					});
2738 | 				});
2739 | 			}
2740 | 		}
2741 | 	function everySecond () {
2742 | 		function checkStuff () {
2743 | 			var now = new Date ();
2744 | 			if (!flEveryMinuteScheduled) {
2745 | 				if (now.getSeconds () == 0) {
2746 | 					setInterval (everyMinute, 60000); 
2747 | 					everyMinute (); //do one right now
2748 | 					flEveryMinuteScheduled = true;
2749 | 					}
2750 | 				}
2751 | 			if (config.enabled) {
2752 | 				if (config.flMaintainCalendarStructure) {
2753 | 					if (flRiverChanged) {
2754 | 						saveTodaysRiver ();
2755 | 						flRiverChanged = false;
2756 | 						}
2757 | 					}
2758 | 				if (flStatsChanged) { 
2759 | 					saveStats ();
2760 | 					flStatsChanged = false;
2761 | 					if (config.statsChangedCallback !== undefined) { //3/25/17 by DW
2762 | 						config.statsChangedCallback (getServerStatsJson ());
2763 | 						}
2764 | 					}
2765 | 				if (flFeedsArrayChanged) {
2766 | 					saveFeedsArray ();
2767 | 					flFeedsArrayChanged = false;
2768 | 					}
2769 | 				if (flFeedsInListsChanged) {
2770 | 					flFeedsInListsChanged = false;
2771 | 					saveFeedsInLists ();
2772 | 					}
2773 | 				if (flFileWriteQueueChanged) {
2774 | 					flFileWriteQueueChanged = false;
2775 | 					checkFileWriteQueue ();
2776 | 					}
2777 | 				checkPodcastQueue (); //4/18/17 by DW
2778 | 				}
2779 | 			if (config.everySecondCallback !== undefined) { //6/20/17 by DW
2780 | 				config.everySecondCallback ();
2781 | 				}
2782 | 			}
2783 | 		if (config.flWatchAppDateChange) { 
2784 | 			utils.getFileModDate (config.fnameApp, function (theModDate) {
2785 | 				if (theModDate != origAppModDate) {
2786 | 					myConsoleLog ("everySecond: " + config.fnameApp + " has been updated. " + myProductName + " is quitting now.");
2787 | 					process.exit (0);
2788 | 					}
2789 | 				else {
2790 | 					checkStuff ();
2791 | 					}
2792 | 				});
2793 | 			}
2794 | 		else {
2795 | 			checkStuff ();
2796 | 			}
2797 | 		}
2798 | 	function everyFiveSeconds () {
2799 | 		if (config.enabled) {
2800 | 			renewNextSubscription (); 
2801 | 			writeLocalStorageIfChanged (); 
2802 | 			saveChangedRiverStructs ();
2803 | 			saveChangedFeedRivers (); //6/29/17 by DW
2804 | 			if (config.flBuildEveryFiveSeconds) { //3/29/17 by DW
2805 | 				buildChangedRivers ();
2806 | 				}
2807 | 			}
2808 | 		}
2809 | 	function everyMinute () {
2810 | 		var now = new Date ();
2811 | 		function doConsoleMessage () {
2812 | 			var ctsockets = countOpenSockets (), portmsg = "";
2813 | 			if (ctsockets == 1) {
2814 | 				ctsockets = ctsockets + " open socket"
2815 | 				}
2816 | 			else {
2817 | 				ctsockets = ctsockets + " open sockets"
2818 | 				}
2819 | 			
2820 | 			if (config.flHttpEnabled) {
2821 | 				portmsg = ", port: " + config.httpPort;
2822 | 				}
2823 | 			
2824 | 			myConsoleLog ("\n" + myProductName + " v" + myVersion + ": " + now.toLocaleTimeString () + ", " + feedsArray.length + " feeds, " + serverStats.ctFeedReadsThisRun + " reads, " + serverStats.ctStoriesAddedThisRun + " stories, " + ctsockets + portmsg + ".");
2825 | 			}
2826 | 		
2827 | 		if (config.enabled) {
2828 | 			buildChangedRivers (function () {
2829 | 				doConsoleMessage ();
2830 | 				loadListsFromFolder ();
2831 | 				checkRiverRollover ();
2832 | 				tossOldFeedRivers (); //6/29/17 by DW
2833 | 				//check for hour rollover
2834 | 					var thisHour = now.getHours ();
2835 | 					if (thisHour != lastEveryMinuteHour) {
2836 | 						serverStats.ctFeedReadsLastHour = 0;
2837 | 						flStatsChanged = true;
2838 | 						lastEveryMinuteHour = thisHour;
2839 | 						}
2840 | 				});
2841 | 			}
2842 | 		else {
2843 | 			doConsoleMessage ();
2844 | 			}
2845 | 		
2846 | 		if (config.everyMinuteCallback !== undefined) { //6/20/17 by DW
2847 | 			config.everyMinuteCallback ();
2848 | 			}
2849 | 		}
2850 | 
2851 | function init (userConfig, callback) {
2852 | 	var now = new Date ();
2853 | 	for (x in userConfig) {
2854 | 		config [x] = userConfig [x];
2855 | 		}
2856 | 		
2857 | 	loadTodaysRiver (function () {
2858 | 		readStats (config.statsFilePath, serverStats, function () {
2859 | 			serverStats.aggregator = myProductName + " v" + myVersion;
2860 | 			serverStats.whenLastStart = now;
2861 | 			serverStats.ctStarts++;
2862 | 			serverStats.ctFeedReadsThisRun = 0;
2863 | 			serverStats.ctStoriesAddedThisRun = 0;
2864 | 			serverStats.ctHitsThisRun = 0;
2865 | 			serverStats.ctFeedReadsLastHour = 0;
2866 | 			
2867 | 			if (serverStats.listModDates !== undefined) {
2868 | 				delete serverStats.listModDates;
2869 | 				}
2870 | 			if (serverStats.ctCloudRenews !== undefined) {
2871 | 				delete serverStats.ctCloudRenews;
2872 | 				}
2873 | 			if (serverStats.ctReadsSkipped !== undefined) {
2874 | 				delete serverStats.ctReadsSkipped;
2875 | 				}
2876 | 			if (serverStats.ctActiveThreads !== undefined) {
2877 | 				delete serverStats.ctActiveThreads;
2878 | 				}
2879 | 			
2880 | 			flStatsChanged = true;
2881 | 			
2882 | 			readStats (fnameFeedsStats, feedsArray, function () {
2883 | 				loadListsFromFolder (function () {
2884 | 					loadListBasedRivers (function () {
2885 | 						loadLocalStorage (function () {
2886 | 							utils.getFileModDate (config.fnameApp, function (theDate) { //set origAppModDate
2887 | 								origAppModDate = theDate;
2888 | 								
2889 | 								var portmsg = "";
2890 | 								if (config.flHttpEnabled) {
2891 | 									portmsg = " running on port " + config.httpPort;
2892 | 									}
2893 | 								
2894 | 								myConsoleLog ("\n" + configToJsonText ()); 
2895 | 								myConsoleLog ("\n" + myProductName + " v" + myVersion + portmsg + ".\n"); 
2896 | 								
2897 | 								setInterval (everyQuarterSecond, 250); 
2898 | 								setInterval (everySecond, 1000); 
2899 | 								setInterval (everyFiveSeconds, 5000); 
2900 | 								startHttpServer ();
2901 | 								startWebSocketServer ();
2902 | 								if (callback !== undefined) {
2903 | 									callback ();
2904 | 									}
2905 | 								});
2906 | 							});
2907 | 						});
2908 | 					});
2909 | 				});
2910 | 			});
2911 | 		});
2912 | 	}
2913 | 
2914 | 


--------------------------------------------------------------------------------
/examples/feedFiler/README.md:
--------------------------------------------------------------------------------
 1 | #### Hello World for davereader
 2 | 
 3 | A very simple app, that models a very common kind of feed-based app. 
 4 | 
 5 | For every new item posted to one of a set of feeds, it writes the JSON representation of that feed to a calendar-structured sub-folder.
 6 | 
 7 | This would be a good starting point for an app that sent a link to stories to a Twitter or Slack account, for example. 
 8 | 
 9 | All the data maintained by davereader is stored in the reader_data folder. 
10 | 
11 | In the startup function the first thing we do is add the addToRiverCallback function to config. The engine will call this function for every new item discovered in one of the subscribed-to feeds. 
12 | 
13 | And that's about it. ;-)
14 | 
15 | 


--------------------------------------------------------------------------------
/examples/feedFiler/filer.js:
--------------------------------------------------------------------------------
 1 | const reader = require ("davereader");
 2 | const utils = require ("daveutils");
 3 | const fs = require ("fs");
 4 | 
 5 | const readerDataFolder = "reader_data/";
 6 | const myOutputFolder = "myOutputFolder/";
 7 | 
 8 | var config = {
 9 | 	flHttpEnabled: false,
10 | 	flWebSocketEnabled: false,
11 | 	
12 | 	listsFolder: readerDataFolder + "lists/",
13 | 	riversFolder: readerDataFolder + "rivers/",
14 | 	podcastsFolder: readerDataFolder + "podcasts/", 
15 | 	dataFolder: readerDataFolder + "data/",
16 | 	
17 | 	addToRiverCallbacksFolder: readerDataFolder + "callbacks/addToRiver/",
18 | 	buildRiverCallbacksFolder: readerDataFolder + "callbacks/buildRiver/",
19 | 	
20 | 	flRequestCloudNotify: false, 
21 | 	flDownloadPodcasts: false
22 | 	};
23 | 
24 | var feeds = [
25 | 	"http://scripting.com/rss.xml",
26 | 	"http://status.aws.amazon.com/rss/ec2-us-east-1.rss",
27 | 	"http://www.aclu.org/blog/feed/",
28 | 	"http://www.bart.gov/news/rss/rss.xml",
29 | 	"http://www.memeorandum.com/feed.xml",
30 | 	"http://code4lib.org/node/feed",
31 | 	"http://www.nytimes.com/pages/technology/index.html?partner=rssnyt",
32 | 	"http://hn.geekity.com/newstories.xml",
33 | 	"http://www.npr.org/rss/rss.php?id=1045"
34 | 	];
35 | function writeFeedsList (callback) { 
36 | 	var f = readerDataFolder + "lists/feeds.json", jsontext = utils.jsonStringify (feeds);
37 | 	utils.sureFilePath (f, function () {
38 | 		fs.writeFile (f, jsontext, function (err) {
39 | 			if (err) {
40 | 				console.log ("writeFeedsList: err.message == " + err.message);
41 | 				}
42 | 			else {
43 | 				callback ();
44 | 				}
45 | 			});
46 | 		});
47 | 	}
48 | 
49 | function startup () {
50 | 	config.newItemCallback = function (urlfeed, itemFromParser, item) { //called for each new item
51 | 		var f = myOutputFolder + utils.getDatePath () + utils.padWithZeros (item.id, 4) + ".json";
52 | 		utils.sureFilePath (f, function () {
53 | 			fs.writeFile (f, utils.jsonStringify (item))
54 | 			});
55 | 		};
56 | 	writeFeedsList (function () {
57 | 		reader.init (config);
58 | 		});
59 | 	}
60 | startup ();
61 | 


--------------------------------------------------------------------------------
/examples/feedFiler/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 | 	"name": "feedFiler",
 3 | 	"description": "Example app.",
 4 | 	"author": "Dave Winer <dave@smallpicture.com>",
 5 | 	"license": "MIT",
 6 | 	"version": "0.4.0",
 7 | 	"main": "filer.js",
 8 | 	"dependencies" : {
 9 | 		"daveutils": "*",
10 | 		"davereader": "*"
11 | 		} 
12 | 	}
13 | 


--------------------------------------------------------------------------------
/examples/river5/config.json:
--------------------------------------------------------------------------------
1 | config.json
2 | 	{
3 | 		"maxRiverItems": 300
4 | 		}
5 | 


--------------------------------------------------------------------------------
/examples/river5/lists/myopmlfeeds.opml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="ISO-8859-1"?>
 2 | <opml version="2.0">
 3 | 	<head>
 4 | 		<title>examples/river5/lists/myopmlfeeds.opml</title>
 5 | 		<dateModified>&lt;%dateModified%></dateModified>
 6 | 		<expansionState></expansionState>
 7 | 		<vertScrollState>1</vertScrollState>
 8 | 		<windowTop>300</windowTop>
 9 | 		<windowLeft>700</windowLeft>
10 | 		<windowBottom>900</windowBottom>
11 | 		<windowRight>1500</windowRight>
12 | 		</head>
13 | 	<body text="examples/river5/lists/myopmlfeeds.opml">
14 | 		<outline text="ACLU feeds" type="include" url="http://rss2.io/lists/aclu.opml"></outline>
15 | 		<outline description="" htmlUrl="" name="advertisingAgeDigital" text="Ad Age Digital" title="Advertising Age - Digital" type="rss" xmlUrl="http://adage.com/rss-feed?section_id=32&amp;xml=RSS2"></outline>
16 | 		<outline description="The Art of Technology" htmlUrl="http://arstechnica.com" name="arsTechnica" text="Ars Technica" title="Ars Technica" type="rss" xmlUrl="http://feeds.arstechnica.com/arstechnica/everything/"></outline>
17 | 		<outline htmlUrl="http://scripting.com/" name="daveWiner" text="Dave's linkblog feed" title="Dave Winer" type="rss" xmlUrl="http://radio3.io/users/davewiner/rss.xml"></outline>
18 | 		<outline name="davesIncludedFeeds" text="Dave-selected podcasts" type="include" url="http://podcatch.com/users/davewiner/podcasts.opml"></outline>
19 | 		<outline htmlUrl="" name="fivethirtyeightfivethirtyeightFeatures" text="FiveThirtyEight" title="FiveThirtyEightFiveThirtyEight | Features" type="rss" xmlUrl="http://fivethirtyeight.com/features/feed/"></outline>
20 | 		<outline htmlUrl="http://www.guardian.co.uk/world/usa" name="worldNewsUnitedStatesGuardiancouk" text="Guardian" title="World news: United States | guardian.co.uk" type="rss" xmlUrl="http://www.guardian.co.uk/world/usa/rss"></outline>
21 | 		<outline description="Must-read media news" htmlUrl="http://mediagazer.com/" language="en-us" name="mediagazer" text="Mediagazer" title="Mediagazer" type="rss" version="RSS2" xmlUrl="http://mediagazer.com/feed.xml"></outline>
22 | 		<outline description="Political Web, page A1" htmlUrl="http://www.memeorandum.com/" language="en-us" name="memeorandum" text="memeorandum" title="memeorandum" type="rss" version="RSS2" xmlUrl="http://www.memeorandum.com/feed.xml"></outline>
23 | 		<outline description="Media, Tech &amp; Business Models" htmlUrl="http://www.mondaynote.com" text="Monday Note" title="Monday Note" type="rss" xmlUrl="http://feeds2.feedburner.com/monday-note"></outline>
24 | 		<outline description="Business, Innovation, Technology, Society" htmlUrl="" text="NYT Bits" title="NYT Bits" type="rss" xmlUrl="http://bits.blogs.nytimes.com/feed/"></outline>
25 | 		<outline htmlUrl="" name="quartz" text="Quartz" title="Quartz" type="rss" xmlUrl="http://qz.com/feed/"></outline>
26 | 		<outline htmlUrl="http://scripting.com/" text="Scripting News" title="Scripting News" type="rss" xmlUrl="http://scripting.com/rss.xml"></outline>
27 | 		<outline htmlUrl="http://politicalwire.com/" name="taeganGoddardsPoliticalWire" text="Taegan Goddard's Political Wire" title="Taegan Goddard's Political Wire" type="rss" xmlUrl="http://feeds.feedburner.com/PoliticalWire"></outline>
28 | 		<outline htmlUrl="http://talkingpointsmemo.com" name="talkingPointsMemo" text="Talking Points Memo" title="Talking Points Memo" type="rss" xmlUrl="http://feeds.feedburner.com/talking-points-memo"></outline>
29 | 		<outline description="Startup and Technology News" htmlUrl="" name="techcrunch" text="TechCrunch" title="TechCrunch" type="rss" xmlUrl="http://techcrunch.com/feed/"></outline>
30 | 		<outline description="Easily digestible tech news..." htmlUrl="http://www.techdirt.com/" name="techdirt" text="Techdirt" title="Techdirt." type="rss" xmlUrl="http://feeds.feedburner.com/techdirt/feed"></outline>
31 | 		<outline htmlUrl="http://www.techmeme.com/" text="Techmeme" title="Techmeme" type="rss" xmlUrl="http://www.techmeme.com/index.xml"></outline>
32 | 		<outline htmlUrl="" name="theCannabist" text="The Cannabist" title="The Cannabist" type="rss" xmlUrl="http://www.thecannabist.co/feed/"></outline>
33 | 		<outline description="Stay up to date on everything happening in The New Yorker and on newyorker.com.&lt;img src=&quot;http://www.newyorker.com/rss_views/everything.gif&quot;&gt;" htmlUrl="http://www.newyorker.com/rss/feeds/everything.xml" language="en-US" name="theNewYorker" text="The New Yorker" title="The New Yorker" type="rss" version="RSS2" xmlUrl="http://www.newyorker.com/services/rss/feeds/everything.xml"></outline>
34 | 		<outline htmlUrl="http://thewirecutter.com" name="theWirecutter" text="The Wirecutter" title="The Wirecutter" type="rss" xmlUrl="http://feeds.feedburner.com/TheWirecutter"></outline>
35 | 		<outline description="Breaking File-sharing, Copyright and Privacy News" htmlUrl="http://torrentfreak.com" name="torrentfreak" text="TorrentFreak" title="TorrentFreak" type="rss" xmlUrl="http://feeds.feedburner.com/Torrentfreak"></outline>
36 | 		<outline htmlUrl="http://www.vox.com/" name="voxAll" text="Vox -  All" title="Vox -  All" type="rss" xmlUrl="http://www.vox.com/rss/index.xml"></outline>
37 | 		</body>
38 | 	</opml>
39 | 


--------------------------------------------------------------------------------
/examples/river5/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 | 	"name": "river5",
 3 | 	"description": "Example app.",
 4 | 	"author": "Dave Winer <dave@smallpicture.com>",
 5 | 	"license": "MIT",
 6 | 	"version": "0.4.0",
 7 | 	"main": "river5.js",
 8 | 	"dependencies" : {
 9 | 		"davereader": "*"
10 | 		} 
11 | 	}
12 | 


--------------------------------------------------------------------------------
/examples/river5/river5.js:
--------------------------------------------------------------------------------
 1 | var reader = require ("davereader"); 
 2 | var fs = require ("fs");
 3 | function readConfig (callback) {
 4 | 	fs.readFile ("config.json", function (err, data) {
 5 | 		var myConfig = new Object ();
 6 | 		if (!err) {
 7 | 			try {
 8 | 				myConfig = JSON.parse (data.toString ());
 9 | 				}
10 | 			catch (err) {
11 | 				console.log ("readConfig: err == " + err.message);
12 | 				}
13 | 			}
14 | 		callback (myConfig);
15 | 		});
16 | 	}
17 | readConfig (function (myConfig) {
18 | 	reader.init (myConfig, function () {
19 | 		});
20 | 	});
21 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 | 	"name": "davereader",
 3 | 	"description": "Feed reading and aggregating for Node.",
 4 | 	"author": "Dave Winer <dave@smallpicture.com>",
 5 | 	"license": "MIT",
 6 | 	"version": "0.6.10", 
 7 | 	"main": "davereader.js",
 8 | 	"repository": {
 9 | 		"type": "git",
10 | 		"url": "https://github.com/scripting/reader.git"
11 | 		},
12 | 	"files": [
13 | 		"davereader.js"
14 | 		],
15 | 	"dependencies" : {
16 | 		"daveutils": "*",
17 | 		"request": "*",
18 | 		"mime": "*",
19 | 		"md5": "*",
20 | 		"nodejs-websocket": "*",
21 | 		"opmlparser": "*",
22 | 		"feedparser": "*",
23 | 		"davefeedread": "*"
24 | 		} 
25 | 	}
26 | 


--------------------------------------------------------------------------------