├── package.json
├── LICENSE
├── README.md
└── publisher.js


/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 | 	"name": "fargoPublisher",
 3 | 	"description": "A JavaScript project to connect to Fargo to publish a folder in Dropbox.",
 4 | 	"author": "Dave Winer <dave@smallpicture.com>",
 5 | 	"version": "0.0.97",
 6 | 	"scripts": {
 7 | 		"start": "node publisher.js"
 8 | 			},
 9 | 	"repository": {
10 | 		"type": "git",
11 | 		"url": "https://github.com/scripting/fargoPublisher.git"
12 | 		},
13 | 	"dependencies" : {
14 | 		"aws-sdk": "*", 
15 | 		"request": "*"
16 | 		}, 
17 | 	"license": "MIT",
18 | 	"engines": {
19 | 		"node": "0.10.x"
20 | 		}
21 | 	}
22 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2014 Dave Winer
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of
 6 | this software and associated documentation files (the "Software"), to deal in
 7 | the Software without restriction, including without limitation the rights to
 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 9 | the Software, and to permit persons to whom the Software is furnished to do so,
10 | subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | ### What is Fargo Publisher? 
  2 | 
  3 | Fargo Publisher is a <a href="http://nodejs.org/">node.js</a> app that connects to <a href="http://fargo.io/">Fargo</a> to publish a folder of HTML docs.
  4 | 
  5 | <a href="http://fargo.io/docs/whatsComingInFargo2.html">Fargo 2</a>, released in February 2014, has a new built-in content management system that creates static HTML files. All that's needed is a way to flow those files to a static HTTP server. That's what Fargo Publisher does. 
  6 | 
  7 | It defines an open protocol that any app can use to connect to static storage, no matter where the content originates. This implementation stores the files in Amazon S3, but a fork of this project could store them in files, or in another content server. 
  8 | 
  9 | It's open source, using the MIT License. 
 10 | 
 11 | This document includes information you need to install a server, and technical information for implementers.
 12 | 
 13 | ### How names work
 14 | 
 15 | Each outline can be given a public name.
 16 | 
 17 | Rather than invent our own naming system, we're using the Internet's -- DNS.
 18 | 
 19 | So if I call my outline "dave" -- you can refer to it as <a href="http://dave.smallpict.com/">dave.smallpict.com</a>.
 20 | 
 21 | When you go to that address, you get the rendered content. 
 22 | 
 23 | There's only one element of the &lt;head> section of that outline that we care about, &lt;linkHosting>. It's automatically maintained by Fargo. 
 24 | 
 25 | The rest of the outline contents is  up to the user.
 26 | 
 27 | ### User experience (set up)
 28 | 
 29 | The Fargo user brings the outline he or she wants to make public to the front.
 30 | 
 31 | Choose the <i>Name Outline</i> command in the File menu.
 32 | 
 33 | A <a href="http://static.scripting.com/larryKing/images/2013/05/14/choosePublicName.gif">dialog</a> appears.
 34 | 
 35 | Type a name. As you type the software tells you whether the name is taken. It does this with a call to Fargo Publisher.
 36 | 
 37 | When you click OK, a message is sent to Fargo Publisher to associate that name with the public URL of the outline. Getting a public URL for a file is a feature of Dropbox.
 38 | 
 39 | ### User experience (publishing)
 40 | 
 41 | When the user publishes an outline, or a portion of an outline, it could cause many files to be rendered. 
 42 | 
 43 | The text of those files is saved to a <i>package</i> file, which is linked into the user's outline, automatically by the CMS.
 44 | 
 45 | When the text is fully rendered, it sends a message to the Publisher server with the name of the outline.
 46 | 
 47 | Publisher then reads the outline (it was registered in the previous step) locates the package file, reads it, breaks the package up into its component files and saves them to the user's folder in the S3 bucket. 
 48 | 
 49 | Publisher sends back to Fargo the base URL of the folder, which is then hooked into the Eye icon, which can be used to view the headline the cursor points to.
 50 | 
 51 | ### User experience (viewing rendered content)
 52 | 
 53 | The Eye icon in the left margin of Fargo is super-important.
 54 | 
 55 | It connects the editing context with the viewing context.
 56 | 
 57 | Put the cursor on any headline in a public outline, and click the <a href="http://static.scripting.com/larryKing/images/2014/01/23/theAllImportantEye.gif">Eye</a>.
 58 | 
 59 | A new tab opens with the rendered view of that section of the outline.
 60 | 
 61 | ### Before deploy
 62 | 
 63 | You're going to need to make a few decisions before you deploy. 
 64 | 
 65 | 1. The user files will be stored in an S3 bucket. There is an S3 path for that location. What should it be?
 66 | 
 67 | 2. Where do you want to store the FP data? This includes information about the user's outlines, and stats generated by Fargo Publisher. It's also an S3 path. You might not want this to be public, although there is no personal information stored about the user outlines. It's all designed for publishing. These are <i>public</i> outlines, by definition. 
 68 | 
 69 | 3. What domain for the user names? You have to own the domain and the DNS should point to the node server so it can redirect to the user's content when subdomains are accessed.
 70 | 
 71 | 4. What port do you want the server to run on? If you don't specify it, the server will boot on port 80.
 72 | 
 73 | 5. Do you want Fargo Publisher to redirect to the content or serve it directly? (The default is to redirect.)
 74 | 
 75 | For my deployment I went with (Unix shell commands):
 76 | 
 77 | 1. export fpHostingPath=/beta.fargo.io/users/
 78 | 
 79 | 2. export fpDataPath=/beta.fargo.io/data/
 80 | 
 81 | 3. export fpDomain=smallpict.com
 82 | 
 83 | 4. export fpServerPort=80
 84 | 
 85 | 5. export fpRedirect=false
 86 | 
 87 | ### How to deploy
 88 | 
 89 | You must have a current node.js installation.
 90 | 
 91 | Install <a href="https://github.com/mikeal/request">request</a>, <a href="http://aws.amazon.com/sdkfornodejs/">AWS</a> and if necessary <a href="http://nodejs.org/api/url.html">url</a>. 
 92 | 
 93 | Set environment variables for AWS.
 94 | 
 95 | 1. AWS_ACCESS_KEY_ID
 96 | 
 97 | 2. AWS_SECRET_ACCESS_KEY
 98 | 
 99 | Set environment variables with your S3 paths and the domain you're using.
100 | 
101 | 1. fpHostingPath -- where the users' files will be stored.
102 | 
103 | 2. fpDataPath -- where you want names and stats to be stored. 
104 | 
105 | 3. fpDomain -- the domain we're allocating.
106 | 
107 | The app is in package.js. package.json already contains all the info that node needs to run it.
108 | 
109 | ### Updates
110 | 
111 | #### v0.97 5/11/15 by DW
112 | 
113 | It's been over a year since the last update to Fargo Publisher, and I've learned a lot about Node and JavaScript since then.
114 | 
115 | The major change in this release is that you can now specify all the environment variables in a file called config.json in the same folder as the publisher.js file. The values in this file take precedence over the environment variables. You still have to set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY through environment variables because that's where the AWS software will look for them.
116 | 
117 | I'm no longer maintaining the OPML versions of the files, so I removed the folder from the repository.
118 | 
119 | Same with worknotes.md.
120 | 
121 | I was previously running my server on Heroku, but now am running it along with my other server software on a Ubuntu instance on AWS.
122 | 
123 | ### Notes
124 | 
125 | There's a folder of OPML files in the repository that contains the outlines that the JavaScript and Markdown come from.
126 | 
127 | We use the MIT License. Nothing proprietary about the protocol or the code. You are encouraged to clone, innovate, enjoy.
128 | 
129 | I love programming in this mode. The tools are great, and node is a wonderful environment, made that way by programmers who share their work so generously. The quality of the work is very impressive. And the commitment to no breakage also refreshing in this day and age. 
130 | 
131 | Thanks to Brent Simmons for his Hello World server <a href="http://inessential.com/2013/12/09/getting_started_with_node_js_for_cocoa_">example</a>. It helped this node newbie get started. He says his example is for Cocoa developers, but that's not true. I understood it, and I've never written a line of Cocoa in my life, and think it's a silly name for a programming environment. ;-)
132 | 
133 | 


--------------------------------------------------------------------------------
/publisher.js:
--------------------------------------------------------------------------------
  1 | //Copyright 2014-2015, Small Picture, Inc.
  2 | 	//Last update: &lt;%now%&gt; Eastern.
  3 | var myVersion = "0.97", myProductName = "Fargo Publisher"; 
  4 | 
  5 | var http = require ("http");
  6 | var request = require ("request");
  7 | var urlpack = require ("url");
  8 | var AWS = require ("aws-sdk");
  9 | var s3 = new AWS.S3 ();
 10 | var dns = require ("dns");
 11 | var os = require ("os");
 12 | var fs = require ("fs"); //5/10/15 by DW
 13 | 
 14 | var fnameConfig = "config.json"; //5/9/15 by DW
 15 | 
 16 | var s3HostingPath = process.env.fpHostingPath; //where we store all the users' HTML and XML files
 17 | var s3defaultType = "text/plain";
 18 | var s3defaultAcl = "public-read";
 19 | 
 20 | var s3DataPath = process.env.fpDataPath;
 21 | var s3NamesPath; 
 22 | var s3StatsPath; 
 23 | var s3SPrefsPath; 
 24 | var s3SScriptsPath; 
 25 | 
 26 | var myDomain = process.env.fpDomain; //something like smallpict.com
 27 | 
 28 | var flRedirect = process.env.fpRedirect; //if false, we just return the content from the s3 cache, instead of redirecting to it -- 2/17/14 by DW
 29 | if (flRedirect == undefined) {
 30 | 	flRedirect = true;
 31 | 	}
 32 | else {
 33 | 	flRedirect = getBoolean (flRedirect);
 34 | 	}
 35 | 
 36 | var myPort;
 37 | if (process.env.PORT == undefined) { //it's not Heroku -- 2/1/14 by DW
 38 | 	myPort = process.env.fpServerPort;
 39 | 	}
 40 | else {
 41 | 	myPort = process.env.PORT;
 42 | 	}
 43 | if (myPort == undefined) {
 44 | 	myPort = 80;
 45 | 	}
 46 | 
 47 | var maxChanges = 100, nameChangesFile = "changes.json";
 48 | var maxHttpLog = 500, nameHttpLogFile = "httpLog.json";
 49 | 
 50 | var serverPrefs; //loaded at startup from prefs.json in the prefs folder on S3 -- 2/11/14 by DW
 51 | var namePrefsFile = "prefs.json";
 52 | 
 53 | var serverStats = {
 54 | 	today: 0,
 55 | 	ctHits: 0, 
 56 | 	ctHitsThisRun: 0,
 57 | 	ctHitsToday: 0,
 58 | 	whenServerStart: 0,
 59 | 	httpLog: []
 60 | 	};
 61 | 
 62 | var urlsChangedData = []; //used in the /httpurlschanged endpoint -- 2/13/14 by DW
 63 | 
 64 | 
 65 | function consoleLog (s) {
 66 | 	console.log (new Date ().toLocaleTimeString () + " -- " + s);
 67 | 	}
 68 | function stringLower (s) {
 69 | 	return (s.toLowerCase ());
 70 | 	}
 71 | function endsWith (s, possibleEnding, flUnicase) {
 72 | 	if ((s == undefined) || (s.length == 0)) { 
 73 | 		return (false);
 74 | 		}
 75 | 	var ixstring = s.length - 1;
 76 | 	if (flUnicase == undefined) {
 77 | 		flUnicase = true;
 78 | 		}
 79 | 	if (flUnicase) {
 80 | 		for (var i = possibleEnding.length - 1; i >= 0; i--) {
 81 | 			if (stringLower (s [ixstring--]) != stringLower (possibleEnding [i])) {
 82 | 				return (false);
 83 | 				}
 84 | 			}
 85 | 		}
 86 | 	else {
 87 | 		for (var i = possibleEnding.length - 1; i >= 0; i--) {
 88 | 			if (s [ixstring--] != possibleEnding [i]) {
 89 | 				return (false);
 90 | 				}
 91 | 			}
 92 | 		}
 93 | 	return (true);
 94 | 	}
 95 | function padWithZeros (num, ctplaces) {
 96 | 	var s = num.toString ();
 97 | 	while (s.length < ctplaces) {
 98 | 		s = "0" + s;
 99 | 		}
100 | 	return (s);
101 | 	}
102 | function stringContains (s, whatItMightContain, flUnicase) { //11/9/14 by DW
103 | 	if (flUnicase === undefined) {
104 | 		flUnicase = true;
105 | 		}
106 | 	if (flUnicase) {
107 | 		s = s.toLowerCase ();
108 | 		whatItMightContain = whatItMightContain.toLowerCase ();
109 | 		}
110 | 	return (s.indexOf (whatItMightContain) != -1);
111 | 	}
112 | function stringCountFields (s, chdelim) {
113 | 	var ct = 1;
114 | 	if (s.length == 0) {
115 | 		return (0);
116 | 		}
117 | 	for (var i = 0; i < s.length; i++) {
118 | 		if (s [i] == chdelim) {
119 | 			ct++;
120 | 			}
121 | 		}
122 | 	return (ct)
123 | 	}
124 | function stringNthField (s, chdelim, n) {
125 | 	var splits = s.split (chdelim);
126 | 	if (splits.length >= n) {
127 | 		return splits [n-1];
128 | 		}
129 | 	return ("");
130 | 	}
131 | function isAlpha (ch) {
132 | 	return (((ch >= 'a') && (ch <= 'z')) || ((ch >= 'A') && (ch <= 'Z')));
133 | 	}
134 | function isNumeric (ch) {
135 | 	return ((ch >= '0') && (ch <= '9'));
136 | 	}
137 | function secondsSince (when) { //2/24/14 by DW
138 | 	var now = new Date ();
139 | 	return ((now - when) / 1000);
140 | 	}
141 | function kilobyteString (num) { //1/24/15 by DW
142 | 	num = Number (num) / 1024;
143 | 	return (num.toFixed (2) + "K");
144 | 	}
145 | function megabyteString (num) { //1/24/15 by DW
146 | 	var onemeg = 1024 * 1024;
147 | 	if (num <= onemeg) {
148 | 		return (kilobyteString (num));
149 | 		}
150 | 	num = Number (num) / onemeg;
151 | 	return (num.toFixed (2) + "MB");
152 | 	}
153 | function gigabyteString (num) { //1/24/15 by DW
154 | 	var onegig = 1024 * 1024 * 1024;
155 | 	if (num <= onegig) {
156 | 		return (megabyteString (num));
157 | 		}
158 | 	num = Number (num) / onegig;
159 | 	return (num.toFixed (2) + "GB");
160 | 	}
161 | function cleanName (name) {
162 | 	var s = "";
163 | 	for (var i = 0; i < name.length; i++) {
164 | 		var ch = name [i];
165 | 		if (isAlpha (ch) || isNumeric (ch)) {
166 | 			s += ch;
167 | 			}
168 | 		}
169 | 	return (s.toLowerCase (s));
170 | 	}
171 | function getNameFromSubdomain (subdomain) {
172 | 	var sections = subdomain.split (".");
173 | 	return (sections [0]);
174 | 	}
175 | function getBoolean (val) { 
176 | 	switch (typeof (val)) {
177 | 		case "string":
178 | 			if (stringLower (val) == "true") {
179 | 				return (true);
180 | 				}
181 | 			break;
182 | 		case "boolean":
183 | 			return (val);
184 | 			break;
185 | 		case "number":
186 | 			if (val == 1) {
187 | 				return (true);
188 | 				}
189 | 			break;
190 | 		}
191 | 	return (false);
192 | 	}
193 | function tcpGetMyIpAddress () {
194 | 	var interfaces = require ("os").networkInterfaces ();
195 | 	for (var devName in interfaces) {
196 | 		var iface = interfaces [devName];
197 | 		for (var i = 0; i < iface.length; i++) {
198 | 			var alias = iface [i];
199 | 			if (alias.family === 'IPv4' && alias.address !== '127.0.0.1' && !alias.internal)
200 | 				return (alias.address);
201 | 				}
202 | 		}
203 | 	return ("0.0.0.0");
204 | 	}
205 | function tcpGetDomainName (ip, callback) {
206 | 	dns.reverse (ip, function (err, domains) {
207 | 		if (err != null) {
208 | 			callback (ip); //use the IP address in place of the domain name
209 | 			}
210 | 		else {
211 | 			callback (domains); 
212 | 			}
213 | 		});
214 | 	}
215 | function scrapeTagValue (sourcestring, tagname) {
216 | 	var s = sourcestring; //work with a copy
217 | 	var opentag = "<" + tagname + ">", closetag = "</" + tagname + ">";
218 | 	var ix = s.indexOf (opentag);
219 | 	if (ix >= 0) {
220 | 		s = s.substr (ix + opentag.length);
221 | 		ix = s.indexOf (closetag);
222 | 		if (ix >= 0) {
223 | 			s = s.substr (0, ix);
224 | 			return (s);
225 | 			}
226 | 		}
227 | 	return ("");
228 | 	}
229 | function sameDay (d1, d2) { 
230 | 	//returns true if the two dates are on the same day
231 | 	d1 = new Date (d1);
232 | 	d2 = new Date (d2);
233 | 	return ((d1.getFullYear () == d2.getFullYear ()) && (d1.getMonth () == d2.getMonth ()) && (d1.getDate () == d2.getDate ()));
234 | 	}
235 | function httpReadUrl (url, callback) {
236 | 	request (url, function (error, response, body) {
237 | 		if (!error && response.statusCode == 200) {
238 | 			callback (body) 
239 | 			}
240 | 		});
241 | 	}
242 | function s3SplitPath (path) { //split path like this: /tmp.scripting.com/testing/one.txt -- into bucketname and path.
243 | 	var bucketname = "";
244 | 	if (path.length > 0) {
245 | 		if (path [0] == "/") { //delete the slash
246 | 			path = path.substr (1); 
247 | 			}
248 | 		var ix = path.indexOf ("/");
249 | 		bucketname = path.substr (0, ix);
250 | 		path = path.substr (ix + 1);
251 | 		}
252 | 	return ({Bucket: bucketname, Key: path});
253 | 	}
254 | function s3NewObject (path, data, type, acl, callback) {
255 | 	var splitpath = s3SplitPath (path);
256 | 	if (type == undefined) {
257 | 		type = s3defaultType;
258 | 		}
259 | 	if (acl == undefined) {
260 | 		acl = s3defaultAcl;
261 | 		}
262 | 	var params = {
263 | 		ACL: acl,
264 | 		ContentType: type,
265 | 		Body: data,
266 | 		Bucket: splitpath.Bucket,
267 | 		Key: splitpath.Key
268 | 		};
269 | 	s3.putObject (params, function (err, data) { 
270 | 		if (callback != undefined) {
271 | 			callback (err, data);
272 | 			}
273 | 		});
274 | 	}
275 | function s3Redirect (path, url) { //1/30/14 by DW -- doesn't appear to work -- don't know why
276 | 	var splitpath = s3SplitPath (path);
277 | 	var params = {
278 | 		WebsiteRedirectLocation: url,
279 | 		Bucket: splitpath.Bucket,
280 | 		Key: splitpath.Key,
281 | 		Body: " "
282 | 		};
283 | 	s3.putObject (params, function (err, data) { 
284 | 		if (err != null) {
285 | 			consoleLog ("s3Redirect: err.message = " + err.message + ".");
286 | 			}
287 | 		else {
288 | 			consoleLog ("s3Redirect: path = " + path + ", url = " + url + ", data = ", JSON.stringify (data));
289 | 			}
290 | 		});
291 | 	}
292 | function s3GetObjectMetadata (path, callback) {
293 | 	var params = s3SplitPath (path);
294 | 	s3.headObject (params, function (err, data) {
295 | 		callback (data);
296 | 		});
297 | 	}
298 | function s3GetObject (path, callback) {
299 | 	var params = s3SplitPath (path);
300 | 	s3.getObject (params, function (err, data) {
301 | 		callback (data);
302 | 		});
303 | 	}
304 | function updateNameRecord (name, obj, callback) { 
305 | 	s3NewObject (s3NamesPath + name + ".json", JSON.stringify (obj, undefined, 3), "text/plain", "public-read", function (err, data) {
306 | 		if (callback != undefined) {
307 | 			callback (err, data);
308 | 			}
309 | 		});
310 | 	}
311 | function addNameRecord (name, opmlUrl, callback) { 
312 | 	var data = {
313 | 		"name": name,
314 | 		"opmlUrl": opmlUrl,
315 | 		"whenCreated": new Date ().toString ()
316 | 		};
317 | 	updateNameRecord (name, data, callback);
318 | 	}
319 | function isNameDefined (name, callback) {
320 | 	s3GetObjectMetadata (s3NamesPath + name + ".json", function (metadata) {
321 | 		callback (metadata != null);
322 | 		});
323 | 	}
324 | function getNameRecord (name, callback) {
325 | 	s3GetObject (s3NamesPath + name + ".json", function (data) {
326 | 		if (data == null) {
327 | 			callback (null);
328 | 			}
329 | 		else {
330 | 			callback (data.Body);
331 | 			}
332 | 		});
333 | 	}
334 | function statsAddToChanges (url) { //add an item to changes.json -- 1/29/14 by DW
335 | 	var path = s3StatsPath + nameChangesFile;
336 | 	s3GetObject (path, function (data) {
337 | 		var changes, obj = new Object (), ctupdates = 0;
338 | 		
339 | 		if (data == null) {
340 | 			changes = new Array ();
341 | 			}
342 | 		else {
343 | 			changes = JSON.parse (data.Body);
344 | 			}
345 | 		
346 | 		for (var i = changes.length - 1; i >= 0; i--) { //delete all other instances of the url in the array
347 | 			if (changes [i].url == url) {
348 | 				if (changes [i].ct != undefined) {
349 | 					ctupdates = changes [i].ct;
350 | 					}
351 | 				changes.splice (i, 1);
352 | 				}
353 | 			}
354 | 		
355 | 		obj.url = url;  //add at beginning of array
356 | 		obj.when = new Date ().toString ();
357 | 		obj.ct = ++ctupdates;
358 | 		
359 | 		changes.unshift (obj);
360 | 		
361 | 		while (changes.length > maxChanges) { //keep array within max size
362 | 			changes.pop ();
363 | 			}
364 | 		
365 | 		s3NewObject (path, JSON.stringify (changes, undefined, 3));
366 | 		});
367 | 	}
368 | function statsAddToHttpLog (httpRequest, urlRedirect, errorMessage, startTime) { //2/11/14 by DW
369 | 	var host = httpRequest.headers.host, url = httpRequest.url, ip = httpRequest.connection.remoteAddress, now = new Date ();
370 | 	if (startTime == undefined) {
371 | 		startTime = new Date ();
372 | 		}
373 | 	serverStats.ctHits++;
374 | 	serverStats.ctHitsThisRun++;
375 | 	serverStats.version = myVersion;  //2/24/14 by DW
376 | 	if (!sameDay (serverStats.today, now)) { //date rollover
377 | 		serverStats.today = now;
378 | 		serverStats.ctHitsToday = 0;
379 | 		}
380 | 	serverStats.ctHitsToday++;
381 | 	
382 | 	var obj = new Object ();
383 | 	obj.when = now.toUTCString ();
384 | 	obj.secs = secondsSince (startTime); //2/24/14 by DW
385 | 	obj.url = "http://" + host + url;
386 | 	if (urlRedirect != undefined) {
387 | 		obj.urlRedirect = urlRedirect;
388 | 		}
389 | 	if (errorMessage != undefined) {
390 | 		obj.errorMessage = errorMessage;
391 | 		}
392 | 	
393 | 	function finishLogAdd () {
394 | 		serverStats.httpLog.unshift (obj);  //add at beginning of array
395 | 		while (serverStats.httpLog.length > maxHttpLog) { //keep array within max size
396 | 			serverStats.httpLog.pop ();
397 | 			}
398 | 		s3NewObject (s3StatsPath + nameHttpLogFile, JSON.stringify (serverStats, undefined, 3));
399 | 		}
400 | 	
401 | 	if (ip != undefined) {
402 | 		tcpGetDomainName (ip, function (domains) {
403 | 			obj.clientIp = ip;
404 | 			obj.clientNames = domains;
405 | 			finishLogAdd ();
406 | 			});
407 | 		}
408 | 	else {
409 | 		finishLogAdd ();
410 | 		}
411 | 	
412 | 	}
413 | function loadServerStats (callback) {
414 | 	s3GetObject (s3StatsPath + nameHttpLogFile, function (data) {
415 | 		if (data != null) {
416 | 			serverStats = JSON.parse (data.Body);
417 | 			serverStats.ctHitsThisRun = 0;
418 | 			if (serverStats.ctHitsToday == undefined) {
419 | 				serverStats.ctHitsToday = 0;
420 | 				}
421 | 			if (serverStats.today == undefined) {
422 | 				serverStats.today = new Date ().toUTCString ();
423 | 				}
424 | 			serverStats.whenServerStart = new Date ().toUTCString ();
425 | 			}
426 | 		if (callback !== undefined) {
427 | 			callback ();
428 | 			}
429 | 		});
430 | 	}
431 | function loadServerPrefs (callback) {
432 | 	s3GetObject (s3SPrefsPath + namePrefsFile, function (data) {
433 | 		if (data != null) {
434 | 			serverPrefs = JSON.parse (data.Body);
435 | 			}
436 | 		if (callback !== undefined) {
437 | 			callback ();
438 | 			}
439 | 		});
440 | 	}
441 | function parsePackages (name, s) { //name is something like "dave"
442 | 	var magicpattern = "<[{~#--- ", ix, path, htmltext, ctfiles = 0, ctchars = 0;
443 | 	while (s.length > 0) {
444 | 		ix = s.indexOf (magicpattern);
445 | 		if (ix < 0) {
446 | 			break;
447 | 			}
448 | 		s = s.substr (ix + magicpattern.length);
449 | 		ix = s.indexOf ("\n");
450 | 		path = s.substr (0, ix);
451 | 		s = s.substr (ix + 1);
452 | 		ix = s.indexOf (magicpattern);
453 | 		if (ix < 0) {
454 | 			htmltext = s;
455 | 			}
456 | 		else {
457 | 			htmltext = s.substr (0, ix);
458 | 			s = s.substr (ix);
459 | 			}
460 | 		
461 | 		if (path.length > 0) {
462 | 			if (path [0] == "/") { //delete leading slash, if present
463 | 				path = path.substr (1);
464 | 				}
465 | 			s3NewObject (s3HostingPath + name + "/" + path, htmltext, "text/html");
466 | 			ctfiles++;
467 | 			ctchars += htmltext.length;
468 | 			}
469 | 		}
470 | 	consoleLog (ctfiles + " files written, " + ctchars + " chars.");
471 | 	}
472 | function handlePackagePing (subdomain) { //something like http://dave.smallpict.com/
473 | 	var parsedUrl = urlpack.parse (subdomain, true);
474 | 	var host = parsedUrl.host;
475 | 	
476 | 	if (host == undefined) { //1/31/14 by DW
477 | 		return;
478 | 		}
479 | 	if (!endsWith (host, myDomain)) { //1/29/14 by DW -- not one of our domains
480 | 		return;
481 | 		}
482 | 	
483 | 	var sections = host.split (".");
484 | 	var name = sections [0];
485 | 	
486 | 	consoleLog ("Ping received: " + host);
487 | 	
488 | 	getNameRecord (name, function (jsontext) {
489 | 		if (jsontext == null) {
490 | 			consoleLog ("Can't handle the package ping for the outline named \"" + name + "\" because there is no outline with that name.");
491 | 			}
492 | 		else {
493 | 			var obj = JSON.parse (jsontext);
494 | 			httpReadUrl (obj.opmlUrl, function (httptext) {
495 | 				var urlpackage = scrapeTagValue (httptext, "linkHosting");
496 | 				httpReadUrl (urlpackage, function (packagetext) {
497 | 					parsePackages (name, packagetext);
498 | 					
499 | 					obj.whenLastUpdate = new Date ().toString ();
500 | 					obj.urlRedirect = "http:/" + s3HostingPath + name + "/"; 
501 | 					
502 | 					if (obj.ctUpdates == undefined) { //1/31/14 by DW
503 | 						obj.ctUpdates = 0;
504 | 						}
505 | 					obj.ctUpdates++;
506 | 					
507 | 					updateNameRecord (name, obj);
508 | 					
509 | 					statsAddToChanges (subdomain); //add it to changes.json -- 1/29/14 by DW
510 | 					});
511 | 				});
512 | 			}
513 | 		});
514 | 	}
515 | function handleRequest (httpRequest, httpResponse) { //5/10/15 by DW
516 | 	try {
517 | 		var parsedUrl = urlpack.parse (httpRequest.url, true);
518 | 		var lowercasepath = parsedUrl.pathname.toLowerCase ();
519 | 		var now = new Date (), nowstring = now.toString ();
520 | 		var host, port, lowerhost, referrer;
521 | 		
522 | 		//set host, port -- 5/10/15 by DW
523 | 			host = httpRequest.headers.host;
524 | 			if (stringContains (host, ":")) {
525 | 				port = stringNthField (host, ":", 2);
526 | 				host = stringNthField (host, ":", 1);
527 | 				}
528 | 			else {
529 | 				port = 80;
530 | 				}
531 | 			lowerhost = host.toLowerCase ();
532 | 		//handle HEAD request
533 | 			if (httpRequest.method == "HEAD") {
534 | 				httpRequest.end ("");
535 | 				return;
536 | 				}
537 | 		//handle redirect through the prefs/redirects table -- 2/11/14 by DW
538 | 			if ((serverPrefs != undefined) && (serverPrefs.redirects != undefined)) {
539 | 				if (serverPrefs.redirects [lowerhost] != undefined) {
540 | 					var newurl = "http://" + serverPrefs.redirects [lowerhost] + parsedUrl.pathname;
541 | 					httpResponse.writeHead (302, {"location": newurl});
542 | 					statsAddToHttpLog (httpRequest, newurl, undefined, now); 
543 | 					httpResponse.end ("302 REDIRECT");    
544 | 					return;
545 | 					}
546 | 				}
547 | 			
548 | 		//handle redirect through the domain we're managing -- 5/10/15 by DW
549 | 			var flhosted = false, lowerdomain = myDomain.toLowerCase (), usethishost;
550 | 			if (endsWith (lowerhost, lowerdomain)) {
551 | 				flhosted = true;
552 | 				usethishost = host;
553 | 				}
554 | 			else {
555 | 				var forwardedhost = httpRequest.headers ["x-forwarded-host"];
556 | 				if (forwardedhost !== undefined) {
557 | 					if (endsWith (forwardedhost.toLowerCase (), lowerdomain)) {
558 | 						flhosted = true;
559 | 						usethishost = forwardedhost;
560 | 						}
561 | 					}
562 | 				}
563 | 			if (flhosted) { //something like dave.smallpict.com
564 | 				var s3path = s3HostingPath + getNameFromSubdomain (usethishost) + parsedUrl.pathname;
565 | 				if (flRedirect) { //2/17/14 by DW
566 | 					var newurl = "http:/" + s3path;
567 | 					httpResponse.writeHead (302, {"location": newurl});
568 | 					statsAddToHttpLog (httpRequest, newurl, undefined, now); 
569 | 					httpResponse.end ("302 REDIRECT");    
570 | 					}
571 | 				else {
572 | 					var contentType = "text/html";
573 | 					
574 | 					if (endsWith (s3path, "/")) { //2/19/14 by DW
575 | 						s3path += "index.html";
576 | 						}
577 | 					
578 | 					if (parsedUrl.pathname.toLowerCase () == "/favicon.ico") { //2/26/14 by DW
579 | 						s3path = "/fargo.io/favicon.ico";
580 | 						contentType = "image/gif";
581 | 						}
582 | 					
583 | 					s3GetObject (s3path, function (data) {
584 | 						if (data == null) {
585 | 							s3GetObject (s3path + "/index.html", function (data) {
586 | 								if (data == null) {
587 | 									httpResponse.writeHead (404, {"Content-Type": "text/plain"});
588 | 									statsAddToHttpLog (httpRequest, undefined, undefined, now); 
589 | 									httpResponse.end ("There is no content to display at \"" + s3path + "\".");
590 | 									}
591 | 								else {
592 | 									httpResponse.writeHead (200, {"Content-Type": contentType});
593 | 									statsAddToHttpLog (httpRequest, undefined, undefined, now); 
594 | 									httpResponse.end (data.Body);    
595 | 									}
596 | 								});
597 | 							}
598 | 						else {
599 | 							httpResponse.writeHead (200, {"Content-Type": contentType});
600 | 							statsAddToHttpLog (httpRequest, undefined, undefined, now); 
601 | 							httpResponse.end (data.Body);    
602 | 							}
603 | 						});
604 | 					}
605 | 				console.log (now.toLocaleTimeString () + ": " + s3HostingPath + getNameFromSubdomain (usethishost) + parsedUrl.pathname);
606 | 				return;
607 | 				}
608 | 		//set referrer -- 5/10/15 by DW
609 | 			referrer = httpRequest.headers.referer;
610 | 			if (referrer == undefined) {
611 | 				referrer = "";
612 | 				}
613 | 			
614 | 		//log the request -- 5/10/15 by DW
615 | 			var client = httpRequest.connection.remoteAddress;
616 | 			if (httpRequest.headers ["x-forwarded-for"] !== undefined) {
617 | 				client = httpRequest.headers ["x-forwarded-for"];
618 | 				}
619 | 			dns.reverse (client, function (err, domains) {
620 | 				var freemem = gigabyteString (os.freemem ()); //1/24/15 by DW
621 | 				if (!err) {
622 | 					if (domains.length > 0) {
623 | 						client = domains [0];
624 | 						}
625 | 					}
626 | 				console.log (now.toLocaleTimeString () + " " + freemem + " " + httpRequest.method + " " + host + ":" + port + " " + lowercasepath + " " + referrer + " " + client);
627 | 				});
628 | 		
629 | 		switch (lowercasepath) {
630 | 			case "/pingpackage":
631 | 				httpResponse.writeHead (200, {"Content-Type": "application/json", "Access-Control-Allow-Origin": "fargo.io"});
632 | 				
633 | 				handlePackagePing (parsedUrl.query.link);
634 | 				
635 | 				var x = {"url": parsedUrl.query.link};
636 | 				var s = "getData (" + JSON.stringify (x) + ")";
637 | 				
638 | 				statsAddToHttpLog (httpRequest, undefined, undefined, now); 
639 | 				httpResponse.end (s);    
640 | 				
641 | 				break;
642 | 			case "/isnameavailable":
643 | 				function sendStringBack (s) {
644 | 					var x = {"message": s};
645 | 					statsAddToHttpLog (httpRequest, undefined, undefined, now); 
646 | 					httpResponse.end ("getData (" + JSON.stringify (x) + ")");    
647 | 					}
648 | 				httpResponse.writeHead (200, {"Content-Type": "application/json", "Access-Control-Allow-Origin": "fargo.io"});
649 | 				var name = cleanName (parsedUrl.query.name);
650 | 				if (name.length == 0) {
651 | 					sendStringBack ("");    
652 | 					}
653 | 				else {
654 | 					if (name.length < 4) {
655 | 						sendStringBack ("Name must be 4 or more characters.");
656 | 						}
657 | 					else {
658 | 						isNameDefined (name, function (fldefined) {
659 | 							var color, answer;
660 | 							if (fldefined) {
661 | 								color = "red";
662 | 								answer = "is not";
663 | 								}
664 | 							else {
665 | 								color = "green";
666 | 								answer = "is";
667 | 								}
668 | 							sendStringBack ("<span style=\"color: " + color + ";\">" + name + "." + myDomain + " " + answer + " available.</span>")
669 | 							});
670 | 						}
671 | 					}
672 | 				break;
673 | 			case "/newoutlinename":
674 | 				var recordkey = cleanName (parsedUrl.query.name), url = parsedUrl.query.url;
675 | 				
676 | 				consoleLog ("Create new outline name: " + recordkey + ", url=" + url);
677 | 				
678 | 				httpResponse.writeHead (200, {"Content-Type": "application/json", "Access-Control-Allow-Origin": "fargo.io"});
679 | 				
680 | 				if (url == undefined) {
681 | 					var x = {flError: true, errorString: "Can't assign the name because there is no <i>url</i> parameter provided."};
682 | 					statsAddToHttpLog (httpRequest, undefined, x.errorString, now); 
683 | 					httpResponse.end ("getData (" + JSON.stringify (x) + ")");    
684 | 					}
685 | 				else {
686 | 					isNameDefined (recordkey, function (fldefined) {
687 | 						if (fldefined) {
688 | 							var x = {flError: true, errorString: "Can't assign the name '" + recordkey + "' to the outline because there already is an outline with that name."};
689 | 							statsAddToHttpLog (httpRequest, undefined, x.errorString, now); 
690 | 							httpResponse.end ("getData (" + JSON.stringify (x) + ")");    
691 | 							}
692 | 						else {
693 | 							addNameRecord (recordkey, url, function (err, data) {
694 | 								if (err) {
695 | 									statsAddToHttpLog (httpRequest, undefined, err, now); 
696 | 									httpResponse.end ("getData (" + JSON.stringify (err) + ")");    
697 | 									}
698 | 								else {
699 | 									var x = {flError: false, name: recordkey + "." + myDomain};
700 | 									statsAddToHttpLog (httpRequest, undefined, undefined, now); 
701 | 									httpResponse.end ("getData (" + JSON.stringify (x) + ")");    
702 | 									}
703 | 								});
704 | 							}
705 | 						});
706 | 					}
707 | 				break;
708 | 			case "/geturlfromname":
709 | 				var name = cleanName (parsedUrl.query.name);
710 | 				httpResponse.writeHead (200, {"Content-Type": "application/json", "Access-Control-Allow-Origin": "fargo.io"});
711 | 				getNameRecord (name, function (jsontext) {
712 | 					if (jsontext == null) {
713 | 						var x = {flError: true, errorString: "Can't open the outline named '" + name + "' because there is no outline with that name."};
714 | 						httpResponse.end ("getData (" + JSON.stringify (x) + ")");    
715 | 						}
716 | 					else {
717 | 						var obj = JSON.parse (jsontext);
718 | 						var x = {flError: false, url: obj.opmlUrl};
719 | 						statsAddToHttpLog (httpRequest, undefined, undefined, now); 
720 | 						httpResponse.end ("getData (" + JSON.stringify (x) + ")");    
721 | 						}
722 | 					});
723 | 				break;
724 | 			case "/version":
725 | 				httpResponse.writeHead (200, {"Content-Type": "text/plain"});
726 | 				statsAddToHttpLog (httpRequest, undefined, undefined, now); 
727 | 				httpResponse.end (myVersion);    
728 | 				break;
729 | 			case "/now": //2/9/14 by DW
730 | 				httpResponse.writeHead (200, {"Content-Type": "text/plain", "Access-Control-Allow-Origin": "*"});
731 | 				httpResponse.end (nowstring);    
732 | 				statsAddToHttpLog (httpRequest, undefined, undefined, now); 
733 | 				break;
734 | 			case "/httpreadurl": //2/10/14 by DW
735 | 				var type = "text/plain";
736 | 				httpReadUrl (parsedUrl.query.url, function (s) {
737 | 					if (parsedUrl.query.type != undefined) {
738 | 						type = parsedUrl.query.type;
739 | 						}
740 | 					httpResponse.writeHead (200, {"Content-Type": type, "Access-Control-Allow-Origin": "*"});
741 | 					statsAddToHttpLog (httpRequest, undefined, undefined, now); 
742 | 					httpResponse.end (s);    
743 | 					});
744 | 				break;
745 | 			case "/status": //2/11/14 by DW
746 | 				var myStatus = {
747 | 					version: myVersion, 
748 | 					now: now.toUTCString (), 
749 | 					whenServerStart: new Date (serverStats.whenServerStart).toUTCString (), 
750 | 					hits: serverStats.ctHits, 
751 | 					hitsToday: serverStats.ctHitsToday
752 | 					};
753 | 				httpResponse.writeHead (200, {"Content-Type": "text/plain", "Access-Control-Allow-Origin": "*"});
754 | 				httpResponse.end (JSON.stringify (myStatus, undefined, 4));    
755 | 				statsAddToHttpLog (httpRequest, undefined, undefined, now); 
756 | 				break;
757 | 			case "/httpurlschanged": //2/13/14 by DW
758 | 				var urlarray = [], returnstruct = {url: []}, ct = 1;
759 | 				function doArrayCheck (urlarray, ix, callback) {
760 | 					if (ix < urlarray.length) {
761 | 						var url = urlarray [ix], eTag = "";
762 | 						if (urlsChangedData [url] != undefined) {
763 | 							if (urlsChangedData [url].etag != undefined) {
764 | 								eTag = urlsChangedData [url].etag;
765 | 								}
766 | 							}
767 | 						var options = {
768 | 							uri: url,
769 | 							headers: {}
770 | 							};
771 | 						if (eTag.length > 0) {
772 | 							options.headers ["If-None-Match"] = eTag;
773 | 							}
774 | 						request (options, function (error, response, body) {
775 | 							if (!error) {
776 | 								if ((response.statusCode == 200) || (response.statusCode == 304)) {
777 | 									if (callback != undefined) {
778 | 										callback (ix, response, body);
779 | 										}
780 | 									doArrayCheck (urlarray, ix + 1, callback);
781 | 									}
782 | 								}
783 | 							});
784 | 						}
785 | 					else {
786 | 						httpResponse.writeHead (200, {"Content-Type": "application/json"});
787 | 						statsAddToHttpLog (httpRequest, undefined, undefined, now); 
788 | 						httpResponse.end ("getData (" + JSON.stringify (returnstruct) + ")");    
789 | 						
790 | 						}
791 | 					}
792 | 				while (true) { //get the urlX params into urlarray
793 | 					var paramname = "url" + ct++;
794 | 					if (parsedUrl.query [paramname] == undefined) { //ran out of urls
795 | 						break;
796 | 						}
797 | 					urlarray [urlarray.length] = parsedUrl.query [paramname];
798 | 					}
799 | 				
800 | 				doArrayCheck (urlarray, 0, function (ixarray, response, body) {
801 | 					var url = urlarray [ixarray], now = new Date ();
802 | 					if (urlsChangedData [url] == undefined) {
803 | 						var obj = new Object ();
804 | 						obj.whenLastCheck = now;
805 | 						obj.ctChecks = 0;
806 | 						obj.ctChanges = 0;
807 | 						urlsChangedData [url] = obj;
808 | 						}
809 | 					if (response.headers.etag != undefined) {
810 | 						urlsChangedData [url].etag = response.headers.etag;
811 | 						}
812 | 					urlsChangedData [url].whenLastCheck = now;
813 | 					urlsChangedData [url].ctChecks++;
814 | 					consoleLog ("callback: urlsChangedData [" + url + "] == " + JSON.stringify (urlsChangedData [url]));
815 | 					if (response.statusCode == 304) { //no change
816 | 						returnstruct.url [ixarray] = urlsChangedData [url].whenLastChange.toUTCString ();
817 | 						}
818 | 					else {
819 | 						urlsChangedData [url].ctChanges++;
820 | 						urlsChangedData [url].whenLastChange = now;
821 | 						returnstruct.url [ixarray] = now.toUTCString ();
822 | 						}
823 | 					});
824 | 				
825 | 				break;
826 | 			case "/getenclosureinfo": //2/15/14 by DW
827 | 				if (parsedUrl.query.url != undefined) {
828 | 					var options = {
829 | 						uri: parsedUrl.query.url,
830 | 						method: "HEAD"
831 | 						};
832 | 					request (options, function (error, response, body) {
833 | 						var flhaveresult = false;
834 | 						if (!error) {
835 | 							if (response.statusCode == 200) {
836 | 								httpResponse.writeHead (200, {"Content-Type": "application/json"});
837 | 								statsAddToHttpLog (httpRequest, undefined, undefined, now); 
838 | 								httpResponse.end ("getData (" + JSON.stringify ({length: response.headers ["content-length"], type: response.headers ["content-type"]}) + ")");    
839 | 								flhaveresult = true;
840 | 								}
841 | 							}
842 | 						if (!flhaveresult) {
843 | 							httpResponse.writeHead (200, {"Content-Type": "application/json"});
844 | 							httpResponse.end ("getData (" + JSON.stringify ({flError: true}) + ")");
845 | 							}
846 | 						});
847 | 					}
848 | 				break;
849 | 			default: //see if it's in the scripts folder, if not 404 -- 4/5/14 by DW
850 | 				var scriptpath = s3SScriptsPath + lowercasepath.substr (1) + ".js"; //drop leading / on lowercasepath
851 | 				s3GetObject (scriptpath, function (data) {
852 | 					if (data == null) {
853 | 						httpResponse.writeHead (404, {"Content-Type": "text/plain"});
854 | 						httpResponse.end ("\"" + parsedUrl.pathname + "\" is not one of the endpoints defined by the Fargo Publisher API.");
855 | 						}
856 | 					else {
857 | 						try {
858 | 							var val = eval (data.Body.toString ());
859 | 							statsAddToHttpLog (httpRequest, undefined, undefined, now); 
860 | 							httpResponse.writeHead (200, {"Content-Type": "text/html"});
861 | 							httpResponse.end (val.toString ());    
862 | 							}
863 | 						catch (err) {
864 | 							httpResponse.writeHead (503, {"Content-Type": "text/plain"});
865 | 							httpResponse.end (err.message);    
866 | 							}
867 | 						}
868 | 					});
869 | 				break;
870 | 				
871 | 			}
872 | 		}
873 | 	catch (tryError) {
874 | 		statsAddToHttpLog (httpRequest, undefined, tryError.message, now); 
875 | 		httpResponse.writeHead (500, {"Content-Type": "text/plain", "Access-Control-Allow-Origin": "*"});
876 | 		httpResponse.end (tryError.message);    
877 | 		}
878 | 	}
879 | function loadConfig (callback) { //5/10/15 by DW
880 | 	fs.readFile (fnameConfig, function (err, data) {
881 | 		if (!err) {
882 | 			var config = JSON.parse (data.toString ());
883 | 			if (config.fpHostingPath !== undefined) {
884 | 				s3HostingPath = config.fpHostingPath;
885 | 				}
886 | 			if (config.fpDataPath !== undefined) {
887 | 				s3DataPath = config.fpDataPath;
888 | 				}
889 | 			if (config.fpDomain !== undefined) {
890 | 				myDomain = config.fpDomain;
891 | 				}
892 | 			if (config.fpRedirect !== undefined) {
893 | 				flRedirect = config.fpRedirect;
894 | 				}
895 | 			if (config.fpServerPort !== undefined) {
896 | 				myPort = config.fpServerPort;
897 | 				}
898 | 			}
899 | 		if (callback !== undefined) {
900 | 			callback ();
901 | 			}
902 | 		});
903 | 	}
904 | 
905 | function startup () {
906 | 	loadConfig (function () {
907 | 		s3NamesPath = s3DataPath + "names/"; 
908 | 		s3StatsPath = s3DataPath + "stats/"; 
909 | 		s3SPrefsPath = s3DataPath + "prefs/"; 
910 | 		s3SScriptsPath = s3DataPath + "scripts/"; 
911 | 		loadServerPrefs (function () {
912 | 			loadServerStats (function () {
913 | 				console.log ("\n" + myProductName + " v" + myVersion + " on port " + myPort + ".");
914 | 				console.log ("\nS3 data path == " + s3DataPath);
915 | 				console.log ("Domain == " + myDomain);
916 | 				console.log ("Redirect == " + getBoolean (flRedirect));
917 | 				console.log ("");
918 | 				http.createServer (handleRequest).listen (myPort);
919 | 				});
920 | 			});
921 | 		});
922 | 	}
923 | 
924 | startup ();
925 | 
926 | 


--------------------------------------------------------------------------------