├── package.json ├── config.json ├── LICENSE ├── worknotes.md ├── README.md ├── feedtomasto.js └── source.opml /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "feedtomasto", 3 | "version": "0.6.3", 4 | "author": "Dave Winer ", 5 | "main": "feedtomasto.js", 6 | "scripts": { 7 | "start": "node feedtomasto.js" 8 | }, 9 | "dependencies" : { 10 | "daveutils": "*", 11 | "request": "*", 12 | "websocket": "*", 13 | "reallysimple": "*", 14 | "wordpress": "*" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /config.json: -------------------------------------------------------------------------------- 1 | { 2 | "enabled": true, 3 | "feeds": [ 4 | "http://data.feedland.org/feeds/davewiner.xml", 5 | "http://nytimes.com/timeswire/feeds/" 6 | ], 7 | "masto": { 8 | "enabled": true, 9 | "appName": "feedToMasto", 10 | "urlMastodonServer": "https://my.masto.social/", 11 | "clientKey": "xxx", 12 | "clientSecret": "xxx", 13 | "accessToken": "xxx", 14 | "scopes": "write:statuses+read:accounts", 15 | "flServerSupportsMarkdown": false 16 | }, 17 | "bluesky": { 18 | "enabled": true, 19 | "username": "bullmancuso", 20 | "mailaddress": "bull@mancuso.com", 21 | "password": "xxx", 22 | "urlsite": "https://bsky.social/", 23 | "maxCtChars": 300, 24 | "flServerSupportsMarkdown": false 25 | }, 26 | "wordpress": { 27 | "enabled": true, 28 | "urlsite": "https://bullman.wordpress.com", 29 | "username": "bullmancuso", 30 | "password": "xxx" 31 | }, 32 | "ctSecsBetwChecks": 60, 33 | "maxCtChars": 500, 34 | "flOnlyPostNewItems": true, 35 | "maxGuids": 2500, 36 | "flServerSupportsMarkdown": false, 37 | "disclaimer": "" 38 | } 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Dave Winer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /worknotes.md: -------------------------------------------------------------------------------- 1 | #### 5/16/23 by DW 2 | 3 | Strings with emoji characters in them take up more space in a JSON file than those without. 4 | 5 | #### 5/14/23 by DW -- v0.6.3 6 | 7 | Reorg'd the code for bluesky and masto, to make them a single rountine that calls local routines. 8 | 9 | Then added support for WordPress. 10 | 11 | #### 5/13/23 by DW 12 | 13 | When we check for enabled in masto table, handle case where it doesn't exist. Avoid breakage with users of previous versions. 14 | 15 | Look for and use maxCtChars in masto or bluesky tables. 16 | 17 | Calling this version 0.6.0 because we added support for Bluesky. 18 | 19 | #### 5/12/23 by DW 20 | 21 | Add Bluesky support. 22 | 23 | added config.bluesky 24 | 25 | added config.masto.enabled, allows you to turn off one service 26 | 27 | there already was a config.enabled. we only do checkFeeds if it's true, so you can shut down an instance so it can be tested elsewhere 28 | 29 | Update readme.md to indicate new functionality 30 | 31 | Breakage in new version 32 | 33 | config.disclaimer no longer supported. 34 | 35 | #### 4/18/23 by DW 36 | 37 | Hook it up to FeedLand's websockets interface so we're notified instantly that feeds updated. Much simpler than the rssCloud interface. 38 | 39 | #### 4/17/23 by DW 40 | 41 | I'm going to use this to map my linkblog feed to Mastodon. 42 | 43 | #### 12/4/22 by DW 44 | 45 | Rewrote mastopost to send the params in the body of the post instead of as url params. 46 | 47 | This is the way forms work, and posts are basically emulating forms, so it seemed this was the most conservative way to go and should give the maximum interop. 48 | 49 | #### 12/3/22 by DW -- v0.4.3 50 | 51 | Follow redirects on HTTP requests. 52 | 53 | #### 12/2/22 by DW -- v0.4.2 54 | 55 | Strip HTML markup from the description element, if it's used. Mastodon neuters the HTML tags. Not a good look. 56 | 57 | Check feeds at startup. Shouldn't have to wait a minute or more for it to check. 58 | 59 | Added a missing step in the instructions in the readme. 60 | 61 | #### 12/1/22 by DW -- v0.4.1 62 | 63 | We now do a better job with items that have both titles and descriptions. Previously it was either/or -- either it had a title or a description but not both. Now we allocate for the disclaimer + title + link and whatever is left over we give to the description/markdown text. 64 | 65 | Added three more feeds to the initial config.json: my linkblog feed, NYT Most Recent Stories and the FeedLand Likes feed from all users. The NYT feed has a consistent format, and the Likes feed is all over the map, and it's easy to add an item to the feed, just like something in FeedLand, the feed is rebuilt immediately. 66 | 67 | Also emptied out the disclaimer in the initial config.json. 68 | 69 | #### 11/28/22 by DW 70 | 71 | Check a list of feeds periodically, post new items to Mastodon. 72 | 73 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # feedToMasto 2 | 3 | A Node app that checks a list of feeds periodically, posting new items to Mastodon and/or Bluesky. 4 | 5 | ### Bluesky, WordPress support added 6 | 7 | Where ever you see "Mastodon" in the docs here, replace it in your mind with "Mastodon and/or Bluesky and/or WordPress." 8 | 9 | I've started a separate section below for how to configure this app to post to Bluesky, and a section for WordPress. 10 | 11 | ### Why did you do this? 12 | 13 | These are things I believe, or goals I have. 14 | 15 | * The network of RSS publishers and readers should be part of the Fediverse. feedToMasto enables the flow of RSS content into the Fediverse via Mastodon. 16 | 17 | * To provide good simple working code for the Mastodon API. I had to do too much work to figure out how to get what amounts to a Hello World script up and running. Now you don't have to do all that work. 18 | 19 | * You should be able to build feed-based utilities without giving any thought to reading feeds. Reading a feed should be as easy as reading a JSON file. This app illustrates how that works, using the reallySimple package. 20 | 21 | * I want to get some code out there into the Mastoverse, to start building a rep in the community. 22 | 23 | ### Requirements 24 | 25 | * You will need an account on a Mastodon server. 26 | 27 | * You will need a place to run a Node.js app. It can run behind a firewall, it does not have to run on a public-facing machine. 28 | 29 | * You will need one or more feeds, they could be RSS, Atom or RDF. 30 | 31 | * The feeds must have guids. This app depends on the guids being unique. 32 | 33 | * To get started download the feedToMasto folder. 34 | 35 | ### config.json 36 | 37 | Setting up the feedToMasto app is all done in config.json. Open it in a text editor. 38 | 39 | ### On your Mastodon account 40 | 41 | Click on Preferences, in the right margin. 42 | 43 | Click on Development in the left margin. 44 | 45 | Click on the New application button in the upper right corner. 46 | 47 | You should see a form that looks like this. We're going to fill in the form. 48 | 49 | * Application name -- anything you like, perhaps feedToMasto. 50 | 51 | * Application website -- anything you like. 52 | 53 | * Redirect URI -- leave it as-is. 54 | 55 | In the Scopes section do the following. 56 | 57 | * Uncheck read. 58 | 59 | * Check read:accounts. 60 | 61 | * Uncheck write. 62 | 63 | * Check write:statuses. 64 | 65 | * Uncheck follow. 66 | 67 | When you're done the Scopes section should look like this. 68 | 69 | Finally click the Submit button at the bottom of the page. 70 | 71 | You should then see a list of your applications, with this new app on the list. 72 | 73 | Click on the name of the new app to bring up a screen with various numeric strings you'll add to config.json in the next section. 74 | 75 | ### Back in config.json 76 | 77 | Open config.json in an editor. This is what you should see. 78 | 79 | On the screen in your browser you should see various numeric strings that you are going to copy into config.json. 80 | 81 | Copy the three items, Client key, Client secret and Your access token from the browser to the xxx's in config.json as shown in this screen shot. 82 | 83 | Enter the URL of your Mastodon server in urlMastodonServer in config.json. 84 | 85 | ### Screen shot 86 | 87 | 88 | 89 | ### Using with Bluesky 90 | 91 | In config.json, at the top level, create a new object called bluesky. 92 | 93 | Set it up as shown in the example file. Basically you're providing the same info you provide when you log in personally, your username, email address and password. 94 | 95 | Why Bluesky? I wanted to start fanning out to support new social networks other than Mastodon. I was using Bluesky and had some good developer docs from my friend Manton Reece, so I decided the second platform would be Bluesky. This is not an endorsement of the product. 96 | 97 | ### Using with WordPress 98 | 99 | In config.json, at the top level, create a new object called wordpress. 100 | 101 | Set it up as shown in the example file. Basically you're providing the same info you provide when you log in personally, your username, email address and password. 102 | 103 | Why WordPress? It's like the Mount Everest of social media, so huge you often miss it. It also supports all the features of textcasting, so it's a way to test all the features, flowing from RSS, that also supports all the features of textcasting. 104 | 105 | ### What the other items in config.json are for 106 | 107 | The other values in config.json have default values that work well for a first run of the app. The only ones you should come back before running it for real are feeds and disclaimer. 108 | 109 | * enabled -- if you set it false and reboot the app it will do everything but check the feeds. 110 | 111 | * feeds -- an array of feed URLs. I've included two feeds to get you started. You should change them to your feed addresses of course. 112 | 113 | * ctSecsBetwChecks -- the amount of time between feed checks. We only check every minute at the top of the minute. Default value is 60. 114 | 115 | * maxCtChars -- the number of characters in a toot. Maybe you have a higher number on your server? 116 | 117 | * flOnlyPostNewItems -- If false, the first time we read a feed we'll dump all the items into the Mastodon account. Probably not what you had in mind, that's why it defaults to true. 118 | 119 | * maxGuids -- we use the guids in feed items to determine if we've already pushed the item to Mastodon. At some point we no longer need to keep it around because it's no longer in the feed, but we never know for sure what that is. It depends on how many feeds you have and how big they tend to get. The default of 2500 seemed a good balance between performance and the risk of posting items twice. 120 | 121 | * flServerSupportsMarkdown -- if it does, we'll look for source:markdown elements in the feed item and transmit that in place of the description element. 122 | 123 | * disclaimer -- text that appears at the begining of every toot. If you don't want it, make it the empty string. 124 | 125 | ### Notes 126 | 127 | Bug reports, not pull requests. 128 | 129 | Comments, questions, feature requests, bug reports here. 130 | 131 | -------------------------------------------------------------------------------- /feedtomasto.js: -------------------------------------------------------------------------------- 1 | const myVersion = "0.6.3", myProductName = "feedToMasto"; 2 | 3 | const fs = require ("fs"); 4 | const request = require ("request"); 5 | const websocket = require ("websocket").w3cwebsocket; 6 | const utils = require ("daveutils"); 7 | const reallysimple = require ("reallysimple"); 8 | const wordpress = require ("wordpress"); //5/14/23 by DW 9 | 10 | var config = { 11 | enabled: true, 12 | feeds: [ 13 | "http://data.feedland.org/feeds/davewiner.xml" 14 | ], 15 | masto: { 16 | enabled: false, //5/12/23 by DW 17 | appName: "feedToMasto", 18 | urlMastodonServer: undefined, 19 | access_token: undefined, 20 | created_at: undefined, 21 | scope: "write:statuses read:accounts", 22 | token_type: "Bearer", 23 | maxCtChars: 500, 24 | flServerSupportsMarkdown: false 25 | }, 26 | bluesky: { //5/12/23 by DW 27 | enabled: false, 28 | urlsite: undefined, 29 | username: undefined, 30 | password: undefined, 31 | maxCtChars: 300, 32 | flServerSupportsMarkdown: false 33 | }, 34 | wordpress: { //5/14/23 by DW 35 | enabled: false, 36 | urlsite: undefined, 37 | username: undefined, 38 | password: undefined, 39 | maxCtChars: undefined, 40 | flServerSupportsMarkdown: false 41 | }, 42 | dataFolder: "data/", 43 | ctSecsBetwChecks: 60, 44 | maxCtChars: 500, 45 | flOnlyPostNewItems: true, //if false when we start up we'll post all the items in the feed 46 | maxGuids: 2500, //we don't store the guids forever, after we have this number of guids, we start deleting the oldest ones 47 | disclaimer: "*This is a test. This came out of the archive of my blog. None of this happened today or yesterday. Still diggin!*", 48 | urlSocketServer: "wss://feedland.org/" //4/18/23 by DW 49 | }; 50 | const fnameConfig = "config.json"; 51 | 52 | var stats = { 53 | guids: new Object () 54 | }; 55 | const fnameStats = "stats.json"; 56 | var flStatsChanged = false; 57 | 58 | var whenLastCheck = new Date (0); 59 | 60 | function statsChanged () { 61 | flStatsChanged = true; 62 | } 63 | function deleteOldGuids () { 64 | function countGuids () { 65 | var ct = 0; 66 | for (var x in stats.guids) { 67 | ct++ 68 | } 69 | return (ct); 70 | } 71 | function deleteOldestGuid () { 72 | var oldestWhen = new Date (), oldestx; 73 | function dateLessThan (d1, d2) { 74 | return (new Date (d1) < new Date (d2)); 75 | } 76 | for (var x in stats.guids) { 77 | var theGuid = stats.guids [x]; 78 | if (dateLessThan (theGuid.when, oldestWhen)) { 79 | oldestWhen = theGuid.when; 80 | oldestx = x; 81 | } 82 | } 83 | if (oldestx !== undefined) { 84 | delete stats.guids [oldestx]; 85 | statsChanged (); 86 | } 87 | } 88 | var ct = countGuids () - config.maxGuids; 89 | if (ct > 0) { 90 | console.log ("deleteOldGuids: ct == " + ct); 91 | for (var i = 1; i <= ct; i++) { 92 | deleteOldestGuid (); 93 | } 94 | } 95 | } 96 | function isNewFeed (feedUrl) { 97 | var flnew = true; 98 | for (var x in stats.guids) { 99 | if (stats.guids [x].feedUrl == feedUrl) { 100 | flnew = false; 101 | break; 102 | } 103 | } 104 | return (flnew); 105 | } 106 | function buildParamList (paramtable) { //8/4/21 by DW 107 | if (paramtable === undefined) { 108 | return (""); 109 | } 110 | else { 111 | var s = ""; 112 | for (var x in paramtable) { 113 | if (paramtable [x] !== undefined) { //8/4/21 by DW 114 | if (s.length > 0) { 115 | s += "&"; 116 | } 117 | s += x + "=" + encodeURIComponent (paramtable [x]); 118 | } 119 | } 120 | return (s); 121 | } 122 | } 123 | function saveItemForDebugging (item) { 124 | const f = "data/items/" + utils.random (1000, 9999) + ".json"; 125 | utils.sureFilePath (f, function () { 126 | fs.writeFile (f, utils.jsonStringify (item), function (err) { 127 | if (err) { 128 | console.log (err.message); 129 | } 130 | }); 131 | }); 132 | } 133 | function getDebuggingItem (num, callback) { 134 | const f = "data/items/" + num + ".json"; 135 | fs.readFile (f, function (err, jsontext) { 136 | if (err) { 137 | callback (err); 138 | } 139 | else { 140 | callback (undefined, JSON.parse (jsontext)); 141 | } 142 | }); 143 | } 144 | function getStringBytes (theString) { //5/17/23 by DW 145 | const ctbytes = Buffer.byteLength (theString); 146 | return (ctbytes); 147 | } 148 | 149 | //mastodon 150 | function mastoPostNewItem (item) { 151 | if (utils.getBoolean (config.masto.enabled)) { 152 | function getStatusText (item, maxCtChars=config.maxCtChars) { 153 | var statustext = ""; 154 | function add (s) { 155 | statustext += s; 156 | } 157 | function addText (desc) { 158 | desc = utils.trimWhitespace (utils.stripMarkup (desc)); 159 | if (desc.length > 0) { 160 | const maxcount = maxCtChars - (statustext.length + desc.length + 2); //the 2 is for the two newlines after the description 161 | desc = utils.maxStringLength (desc, maxcount, false, true) + "\n\n"; 162 | add (desc); 163 | } 164 | } 165 | function notEmpty (s) { 166 | if (s === undefined) { 167 | return (false); 168 | } 169 | if (s.length == 0) { 170 | return (false); 171 | } 172 | return (true); 173 | } 174 | if (notEmpty (item.title)) { 175 | addText (item.title); 176 | } 177 | else { 178 | addText (item.description); 179 | } 180 | if (notEmpty (item.link)) { 181 | add (item.link); 182 | } 183 | return (statustext); 184 | } 185 | function mastocall (path, params, callback) { 186 | var headers = undefined; 187 | if (config.masto.accessToken !== undefined) { 188 | headers = { 189 | Authorization: "Bearer " + config.masto.accessToken 190 | }; 191 | } 192 | const theRequest = { 193 | url: config.masto.urlMastodonServer + path + "?" + buildParamList (params), 194 | method: "GET", 195 | followAllRedirects: true, //12/3/22 by DW 196 | maxRedirects: 5, 197 | headers, 198 | }; 199 | request (theRequest, function (err, response, jsontext) { 200 | function sendBackError (defaultMessage) { 201 | var flcalledback = false; 202 | if (data !== undefined) { 203 | try { 204 | jstruct = JSON.parse (data); 205 | if (jstruct.error !== undefined) { 206 | callback ({message: jstruct.error}); 207 | flcalledback = true; 208 | } 209 | } 210 | catch (err) { 211 | } 212 | } 213 | 214 | if (!flcalledback) { 215 | callback ({message: defaultMessage}); 216 | } 217 | } 218 | if (err) { 219 | sendBackError (err.message); 220 | } 221 | else { 222 | var code = response.statusCode; 223 | if ((code < 200) || (code > 299)) { 224 | const message = "The request returned a status code of " + response.statusCode + "."; 225 | sendBackError (message); 226 | } 227 | else { 228 | try { 229 | callback (undefined, JSON.parse (jsontext)) 230 | } 231 | catch (err) { 232 | callback (err); 233 | } 234 | } 235 | } 236 | }); 237 | } 238 | function mastopost (path, params, callback) { 239 | const theRequest = { 240 | url: config.masto.urlMastodonServer + path, 241 | method: "POST", 242 | followAllRedirects: true, //12/3/22 by DW 243 | maxRedirects: 5, 244 | headers: { 245 | "Authorization": "Bearer " + config.masto.accessToken, 246 | "Content-Type": "application/x-www-form-urlencoded" 247 | }, 248 | body: buildParamList (params) 249 | }; 250 | request (theRequest, function (err, response, jsontext) { 251 | if (err) { 252 | console.log ("mastopost: err.message == " + err.message); 253 | callback (err); 254 | } 255 | else { 256 | var code = response.statusCode; 257 | if ((code < 200) || (code > 299)) { 258 | const message = "The request returned a status code of " + response.statusCode + "."; 259 | callback ({message}); 260 | } 261 | else { 262 | try { 263 | callback (undefined, JSON.parse (jsontext)) 264 | } 265 | catch (err) { 266 | callback (err); 267 | } 268 | } 269 | } 270 | }); 271 | } 272 | function getUserInfo (callback) { 273 | mastocall ("api/v1/accounts/verify_credentials", undefined, callback); 274 | } 275 | function tootStatus (statusText, inReplyTo, callback) { 276 | const params = { 277 | status: statusText, 278 | in_reply_to_id: inReplyTo 279 | }; 280 | mastopost ("api/v1/statuses", params, callback); 281 | } 282 | const statustext = getStatusText (item, config.masto.maxCtChars); //5/12/23 by DW 283 | tootStatus (statustext, undefined, function (err, data) { 284 | if (err) { 285 | console.log (err.message); 286 | } 287 | else { 288 | console.log (new Date ().toLocaleTimeString () + ": " + data.url); 289 | } 290 | }); 291 | } 292 | } 293 | //bluesky 294 | function blueskyPostNewItem (item) { 295 | if (utils.getBoolean (config.bluesky.enabled)) { 296 | function getAccessToken (options, callback) { 297 | const url = options.urlsite + "xrpc/com.atproto.server.createSession"; 298 | const bodystruct = { 299 | identifier: options.mailaddress, 300 | password: options.password 301 | }; 302 | var theRequest = { 303 | method: "POST", 304 | url: url, 305 | body: utils.jsonStringify (bodystruct), 306 | headers: { 307 | "User-Agent": options.userAgent, 308 | "Content-Type": "application/json" 309 | } 310 | }; 311 | request (theRequest, function (err, response, body) { 312 | var jstruct = undefined; 313 | if (err) { 314 | callback (err); 315 | } 316 | else { 317 | try { 318 | callback (undefined, JSON.parse (body)); 319 | } 320 | catch (err) { 321 | callback (err); 322 | } 323 | } 324 | }); 325 | } 326 | function newPost (options, authorization, item, callback) { 327 | const url = options.urlsite + "xrpc/com.atproto.repo.createRecord"; 328 | const nowstring = new Date ().toISOString (); 329 | 330 | function notEmpty (s) { 331 | if (s === undefined) { 332 | return (false); 333 | } 334 | if (s.length == 0) { 335 | return (false); 336 | } 337 | return (true); 338 | } 339 | function decodeForBluesky (s) { 340 | var replacetable = { 341 | "#39": "'" 342 | }; 343 | s = utils.multipleReplaceAll (s, replacetable, true, "&", ";"); 344 | return (s); 345 | } 346 | function getStatusText (item) { //special for bluesky, just get the text, no link 347 | var statustext = ""; 348 | function add (s) { 349 | statustext += s; 350 | } 351 | function addText (desc) { 352 | desc = decodeForBluesky (desc); 353 | desc = utils.trimWhitespace (utils.stripMarkup (desc)); 354 | if (desc.length > 0) { 355 | const maxcount = config.bluesky.maxCtChars - (statustext.length + desc.length + 2); //the 2 is for the two newlines after the description 356 | desc = utils.maxStringLength (desc, maxcount, false, true) + "\n\n"; 357 | add (desc); 358 | } 359 | } 360 | if (notEmpty (item.title)) { 361 | addText (item.title); 362 | } 363 | else { 364 | addText (item.description); 365 | } 366 | return (statustext); 367 | } 368 | function getRecord (item) { 369 | var theRecord = { 370 | text: getStatusText (item), 371 | createdAt: nowstring 372 | } 373 | if (notEmpty (item.link)) { 374 | const linkword = utils.getDomainFromUrl (item.link); 375 | theRecord.text += linkword; 376 | const ctbytes = getStringBytes (theRecord.text); //5/16/23 by DW 377 | theRecord.facets = [ 378 | { 379 | features: [ 380 | { 381 | uri: item.link, 382 | "$type": "app.bsky.richtext.facet#link" 383 | } 384 | ], 385 | index: { 386 | byteStart: ctbytes - linkword.length, //5/16/23 by DW 387 | byteEnd: ctbytes 388 | } 389 | } 390 | ]; 391 | } 392 | console.log ("bluesky/getRecord: theRecord == " + utils.jsonStringify (theRecord)); 393 | return (theRecord); 394 | } 395 | 396 | const bodystruct = { 397 | repo: authorization.did, 398 | collection: "app.bsky.feed.post", 399 | validate: true, 400 | record: getRecord (item) 401 | }; 402 | var theRequest = { 403 | method: "POST", 404 | url: url, 405 | body: utils.jsonStringify (bodystruct), 406 | headers: { 407 | "User-Agent": options.userAgent, 408 | "Content-Type": "application/json", 409 | Authorization: "Bearer " + authorization.accessJwt 410 | } 411 | }; 412 | request (theRequest, function (err, response, body) { 413 | if (err) { 414 | callback (err); 415 | } 416 | else { 417 | try { 418 | callback (undefined, JSON.parse (body)); 419 | } 420 | catch (err) { 421 | callback (err); 422 | } 423 | } 424 | }); 425 | } 426 | getAccessToken (config.bluesky, function (err, authorization) { 427 | if (err) { 428 | console.log ("blueskyPostNewItem: err.message == " + err.message); 429 | } 430 | else { 431 | newPost (config.bluesky, authorization, item, function (err, data) { 432 | if (err) { 433 | console.log ("blueskyPostNewItem: err.message == " + err.message); 434 | } 435 | else { 436 | } 437 | }); 438 | } 439 | }); 440 | } 441 | } 442 | //wordpress -- 5/14/23 by DW 443 | function wordpressPostNewItem (item) { 444 | if (utils.getBoolean (config.wordpress.enabled)) { 445 | const client = wordpress.createClient ({ 446 | url: config.wordpress.urlsite, 447 | username: config.wordpress.username, 448 | password: config.wordpress.password 449 | }); 450 | 451 | function getPostInfo (client, idPost, callback) { 452 | client.getPost (idPost, function (err, thePost) { 453 | if (err) { 454 | callback (err); 455 | } 456 | else { 457 | callback (undefined, thePost); 458 | } 459 | }); 460 | } 461 | function newPost (client, title, content, link, callback) { 462 | if (link !== undefined) { 463 | content += "\n\n" + link; 464 | } 465 | 466 | const thePost = { 467 | title, 468 | content, 469 | status: "publish" //omit this to create a draft that isn't published 470 | }; 471 | client.newPost (thePost, function (err, idNewPost) { 472 | if (err) { 473 | callback (err); 474 | } 475 | else { 476 | getPostInfo (client, idNewPost, function (err, theNewPost) { 477 | if (err) { 478 | callback (err); 479 | } 480 | else { 481 | callback (undefined, theNewPost); 482 | } 483 | }); 484 | } 485 | }); 486 | } 487 | 488 | newPost (client, item.title, item.description, item.link, function (err, thePost) { 489 | if (err) { 490 | console.log ("wordpressPostNewItem: err.message == " + err.message); 491 | } 492 | else { 493 | console.log ("wordpressPostNewItem: thePost.link == " + thePost.link); 494 | } 495 | }); 496 | } 497 | } 498 | 499 | function checkFeed (feedUrl, callback) { 500 | const flNewFeed = isNewFeed (feedUrl); 501 | var flPost = (flNewFeed && config.flOnlyPostNewItems) ? false : true; 502 | reallysimple.readFeed (feedUrl, function (err, theFeed) { 503 | if (err) { 504 | callback (err); 505 | } 506 | else { 507 | theFeed.items.forEach (function (item) { 508 | if (item.guid !== undefined) { //we ignore items without guids 509 | var flfound = false; 510 | for (var x in stats.guids) { 511 | if (x == item.guid) { 512 | flfound = true; 513 | break; 514 | } 515 | } 516 | if (!flfound) { 517 | stats.guids [item.guid] = { 518 | when: new Date (), 519 | feedUrl 520 | }; 521 | statsChanged (); 522 | if (flPost) { 523 | mastoPostNewItem (item); 524 | blueskyPostNewItem (item); //5/12/23 by DW 525 | wordpressPostNewItem (item); //5/14/23 by DW 526 | saveItemForDebugging (item); //5/12/23 by DW 527 | } 528 | } 529 | } 530 | }); 531 | } 532 | }); 533 | } 534 | function writeStats () { 535 | fs.writeFile (fnameStats, utils.jsonStringify (stats), function (err) { 536 | }); 537 | } 538 | function checkFeeds () { 539 | if (config.enabled) { //5/12/23 by DW 540 | whenLastCheck = new Date (); 541 | config.feeds.forEach (function (feedUrl) { 542 | checkFeed (feedUrl, function (err, data) { 543 | if (err) { 544 | console.log ("everySecond: feedUrl == " +feedUrl + ", err.message == " + err.message); 545 | } 546 | }); 547 | }); 548 | } 549 | } 550 | 551 | function startSocket () { //4/18/23 by DW 552 | function wsConnectUserToServer (itemReceivedCallback) { 553 | var mySocket = undefined; 554 | function checkConnection () { 555 | if (mySocket === undefined) { 556 | mySocket = new websocket (config.urlSocketServer); 557 | mySocket.onopen = function (evt) { 558 | }; 559 | mySocket.onmessage = function (evt) { 560 | function getPayload (jsontext) { 561 | var thePayload = undefined; 562 | try { 563 | thePayload = JSON.parse (jsontext); 564 | } 565 | catch (err) { 566 | } 567 | return (thePayload); 568 | } 569 | if (evt.data !== undefined) { //no error 570 | var theCommand = utils.stringNthField (evt.data, "\r", 1); 571 | var jsontext = utils.stringDelete (evt.data, 1, theCommand.length + 1); 572 | var thePayload = getPayload (jsontext); 573 | switch (theCommand) { 574 | case "newItem": 575 | itemReceivedCallback (thePayload); 576 | break; 577 | } 578 | } 579 | }; 580 | mySocket.onclose = function (evt) { 581 | mySocket = undefined; 582 | }; 583 | mySocket.onerror = function (evt) { 584 | }; 585 | } 586 | } 587 | setInterval (checkConnection, 1000); 588 | } 589 | wsConnectUserToServer (function (thePayload) { 590 | config.feeds.forEach (function (feedUrl) { 591 | if (feedUrl == thePayload.theFeed.feedUrl) { 592 | console.log (new Date ().toLocaleTimeString () + ": title == " + thePayload.theFeed.title + ", feedUrl == " + thePayload.theFeed.feedUrl); 593 | checkFeed (feedUrl, function (err, data) { 594 | if (err) { 595 | console.log ("startSocket: feedUrl == " +feedUrl + ", err.message == " + err.message); 596 | } 597 | }); 598 | } 599 | }); 600 | }); 601 | } 602 | 603 | function everyMinute () { 604 | if (utils.secondsSince (whenLastCheck) > config.ctSecsBetwChecks) { //check feeds at most once a minute 605 | checkFeeds (); 606 | } 607 | deleteOldGuids (); 608 | } 609 | function everySecond () { 610 | if (flStatsChanged) { 611 | flStatsChanged = false; 612 | writeStats (); 613 | } 614 | } 615 | 616 | function readConfig (fname, data, callback) { 617 | fs.readFile (fname, function (err, jsontext) { 618 | if (!err) { 619 | var jstruct; 620 | try { 621 | jstruct = JSON.parse (jsontext); 622 | for (var x in jstruct) { 623 | data [x] = jstruct [x]; 624 | } 625 | } 626 | catch (err) { 627 | console.log ("readConfig: fname == " + fname + ", err.message == " + utils.jsonStringify (err.message)); 628 | } 629 | } 630 | callback (); 631 | }); 632 | } 633 | 634 | function testGetStatusText () { 635 | const theNum = "8313"; 636 | getDebuggingItem (theNum, function (err, item) { 637 | if (err) { 638 | console.log (err.message); 639 | } 640 | else { 641 | console.log ("testGetStatusText: item == " + utils.jsonStringify (item)); 642 | console.log ("testGetStatusText: statustext == " + getStatusText (item)); 643 | } 644 | }); 645 | } 646 | function testBlueskyPost () { 647 | const theNum = "8313"; 648 | getDebuggingItem (theNum, function (err, item) { 649 | if (err) { 650 | console.log (err.message); 651 | } 652 | else { 653 | console.log ("testBlueskyPost: item == " + utils.jsonStringify (item)); 654 | blueskyPostNewItem (item); 655 | } 656 | }); 657 | } 658 | 659 | 660 | function startup () { 661 | console.log ("startup"); 662 | readConfig (fnameStats, stats, function () { 663 | readConfig (fnameConfig, config, function () { 664 | console.log ("config == " + utils.jsonStringify (config)); 665 | checkFeeds (); //check at startup 666 | utils.runEveryMinute (everyMinute); 667 | setInterval (everySecond, 1000); 668 | startSocket (); //4/18/23 by DW 669 | }); 670 | }); 671 | } 672 | startup (); 673 | 674 | -------------------------------------------------------------------------------- /source.opml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 14 | 15 | 16 | 17 | nodeEditor: feedToMasto 18 | Mon, 28 Nov 2022 22:00:42 GMT 19 | Wed, 17 May 2023 13:39:27 GMT 20 | Dave Winer 21 | http://davewiner.com/ 22 | 1, 4, 32, 33, 35, 37, 44, 46, 55, 57, 98 23 | 17 24 | 269 25 | 225 26 | 1284 27 | 1504 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | 338 | 339 | 340 | 341 | 342 | 343 | 344 | 345 | 346 | 347 | 348 | 349 | 350 | 351 | 352 | 353 | 354 | 355 | 356 | 357 | 358 | 359 | 360 | 361 | 362 | 363 | 364 | 365 | 366 | 367 | 368 | 369 | 370 | 371 | 372 | 373 | 374 | 375 | 376 | 377 | 378 | 379 | 380 | 381 | 382 | 383 | 384 | 385 | 386 | 387 | 388 | 389 | 390 | 391 | 392 | 393 | 394 | 395 | 396 | 397 | 398 | 399 | 400 | 401 | 402 | 403 | 404 | 405 | 406 | 407 | 408 | 409 | 410 | 411 | 412 | 413 | 414 | 415 | 416 | 417 | 418 | 419 | 420 | 421 | 422 | 423 | 424 | 425 | 426 | 427 | 428 | 429 | 430 | 431 | 432 | 433 | 434 | 435 | 436 | 437 | 438 | 439 | 440 | 441 | 442 | 443 | 444 | 445 | 446 | 447 | 448 | 449 | 450 | 451 | 452 | 453 | 454 | 455 | 456 | 457 | 458 | 459 | 460 | 461 | 462 | 463 | 464 | 465 | 466 | 467 | 468 | 469 | 470 | 471 | 472 | 473 | 474 | 475 | 476 | 477 | 478 | 479 | 480 | 481 | 482 | 483 | 484 | 485 | 486 | 487 | 488 | 489 | 490 | 491 | 492 | 493 | 494 | 495 | 496 | 497 | 498 | 499 | 500 | 501 | 502 | 503 | 504 | 505 | 506 | 507 | 508 | 509 | 510 | 511 | 512 | 513 | 514 | 515 | 516 | 517 | 518 | 519 | 520 | 521 | 522 | 523 | 524 | 525 | 526 | 527 | 528 | 529 | 530 | 531 | 532 | 533 | 534 | 535 | 536 | 537 | 538 | 539 | 540 | 541 | 542 | 543 | 544 | 545 | 546 | 547 | 548 | 549 | 550 | 551 | 552 | 553 | 554 | 555 | 556 | 557 | 558 | 559 | 560 | 561 | 562 | 563 | 564 | 565 | 566 | 567 | 568 | 569 | 570 | 571 | 572 | 573 | 574 | 575 | 576 | 577 | 578 | 579 | 580 | 581 | 582 | 583 | 584 | 585 | 586 | 587 | 588 | 589 | 590 | 591 | 592 | 593 | 594 | 595 | 596 | 597 | 598 | 599 | 600 | 601 | 602 | 603 | 604 | 605 | 606 | 607 | 608 | 609 | 610 | 611 | 612 | 613 | 614 | 615 | 616 | 617 | 618 | 619 | 620 | 621 | 622 | 623 | 624 | 625 | 626 | 627 | 628 | 629 | 630 | 631 | 632 | 633 | 634 | 635 | 636 | 637 | 638 | 639 | 640 | 641 | 642 | 643 | 644 | 645 | 646 | 647 | 648 | 649 | 650 | 651 | 652 | 653 | 654 | 655 | 656 | 657 | 658 | 659 | 660 | 661 | 662 | 663 | 664 | 665 | 666 | 667 | 668 | 669 | 670 | 671 | 672 | 673 | 674 | 675 | 676 | 677 | 678 | 679 | 680 | 681 | 682 | 683 | 684 | 685 | 686 | 687 | 688 | 689 | 690 | 691 | 692 | 693 | 694 | 695 | 696 | 697 | 698 | 699 | 700 | 701 | 702 | 703 | 704 | 705 | 706 | 707 | 708 | 709 | 710 | 711 | 712 | 713 | 714 | 715 | 716 | 717 | 718 | 719 | 720 | 721 | 722 | 723 | 724 | 725 | 726 | 727 | 728 | 729 | 730 | 731 | 732 | 733 | 734 | 735 | 736 | 737 | 738 | 739 | 740 | 741 | 742 | 743 | 744 | 745 | 746 | 747 | 748 | 749 | 750 | 751 | 752 | 753 | 754 | 755 | 756 | 757 | 758 | 759 | 760 | 761 | 762 | 763 | 764 | 765 | 766 | 767 | 768 | 769 | 770 | 771 | 772 | 773 | 774 | 775 | 776 | 777 | 778 | 779 | 780 | 781 | 782 | 783 | 784 | 785 | 786 | 787 | 788 | 789 | 790 | 791 | 792 | 793 | 794 | 795 | 796 | 797 | 798 | 799 | 800 | 801 | 802 | 803 | 804 | 805 | 806 | 807 | 808 | 809 | 810 | 811 | 812 | 813 | 814 | 815 | 816 | 817 | 818 | 819 | 820 | 821 | 822 | 823 | 824 | 825 | 826 | 827 | 828 | 829 | 830 | 831 | 832 | 833 | 834 | 835 | 836 | 837 | 838 | 839 | 840 | 841 | 842 | 843 | 844 | 845 | 846 | 847 | 848 | 849 | 850 | 851 | 852 | 853 | 854 | 855 | 856 | 857 | 858 | 859 | 860 | 861 | 862 | 863 | 864 | 865 | 866 | 867 | 868 | 869 | 870 | 871 | 872 | 873 | 874 | 875 | 876 | 877 | 878 | 879 | 880 | 881 | 882 | 883 | 884 | 885 | 886 | 887 | 888 | 889 | 890 | 891 | 892 | 893 | 894 | 895 | 896 | 897 | 898 | 899 | 900 | 901 | 902 | 903 | 904 | 905 | 906 | 907 | 908 | 909 | 910 | 911 | 912 | 913 | 914 | 915 | 916 | 917 | 918 | 919 | 920 | 921 | 922 | 923 | 924 | 925 | 926 | 927 | 928 | 929 | 930 | 931 | 932 | 933 | 934 | 935 | 936 | 937 | 938 | 939 | 940 | 941 | 942 | 943 | 944 | 945 | 946 | 947 | 948 | 949 | 950 | 951 | 952 | 953 | 954 | 955 | 956 | 957 | 958 | 959 | 960 | 961 | 962 | 963 | 964 | 965 | 966 | 967 | 968 | 969 | 970 | 971 | 972 | 973 | 974 | 975 | 976 | 977 | 978 | 979 | 980 | 981 | 982 | 983 | 984 | 985 | 986 | 987 | 988 | 989 | 990 | 991 | 992 | 993 | 994 | 995 | 996 | 997 | 998 | 999 | 1000 | 1001 | 1002 | 1003 | 1004 | 1005 | 1006 | 1007 | 1008 | 1009 | 1010 | 1011 | 1012 | 1013 | 1014 | 1015 | 1016 | 1017 | 1018 | 1019 | 1020 | 1021 | 1022 | 1023 | 1024 | 1025 | 1026 | 1027 | 1028 | 1029 | 1030 | 1031 | 1032 | 1033 | 1034 | 1035 | 1036 | 1037 | 1038 | 1039 | 1040 | 1041 | 1042 | 1043 | 1044 | 1045 | 1046 | 1047 | 1048 | 1049 | 1050 | 1051 | 1052 | 1053 | 1054 | 1055 | 1056 | 1057 | 1058 | 1059 | 1060 | 1061 | 1062 | 1063 | 1064 | 1065 | 1066 | 1067 | 1068 | 1069 | 1070 | 1071 | 1072 | 1073 | 1074 | 1075 | 1076 | 1077 | 1078 | 1079 | 1080 | 1081 | 1082 | 1083 | 1084 | 1085 | 1086 | 1087 | 1088 | 1089 | 1090 | 1091 | 1092 | 1093 | 1094 | 1095 | 1096 | 1097 | 1098 | 1099 | 1100 | 1101 | 1102 | 1103 | 1104 | 1105 | 1106 | 1107 | 1108 | 1109 | 1110 | 1111 | 1112 | 1113 | 1114 | 1115 | 1116 | 1117 | 1118 | 1119 | 1120 | 1121 | 1122 | 1123 | 1124 | 1125 | 1126 | 1127 | 1128 | 1129 | 1130 | 1131 | 1132 | 1133 | 1134 | 1135 | 1136 | 1137 | 1138 | 1139 | 1140 | 1141 | 1142 | 1143 | 1144 | 1145 | 1146 | 1147 | 1148 | 1149 | 1150 | 1151 | 1152 | 1153 | 1154 | 1155 | 1156 | 1157 | 1158 | 1159 | 1160 | 1161 | 1162 | 1163 | 1164 | 1165 | 1166 | 1167 | 1168 | 1169 | 1170 | 1171 | 1172 | 1173 | 1174 | 1175 | 1176 | 1177 | 1178 | 1179 | 1180 | 1181 | 1182 | 1183 | 1184 | 1185 | 1186 | 1187 | 1188 | 1189 | 1190 | 1191 | 1192 | 1193 | 1194 | 1195 | 1196 | 1197 | 1198 | 1199 | 1200 | 1201 | 1202 | 1203 | 1204 | 1205 | 1206 | 1207 | 1208 | 1209 | 1210 | 1211 | 1212 | 1213 | 1214 | 1215 | 1216 | 1217 | 1218 | 1219 | 1220 | 1221 | 1222 | 1223 | 1224 | 1225 | 1226 | 1227 | 1228 | 1229 | 1230 | 1231 | 1232 | 1233 | 1234 | 1235 | 1236 | 1237 | 1238 | 1239 | 1240 | 1241 | 1242 | 1243 | 1244 | 1245 | 1246 | 1247 | 1248 | 1249 | 1250 | 1251 | 1252 | 1253 | 1254 | 1255 | 1256 | 1257 | 1258 | 1259 | 1260 | 1261 | 1262 | 1263 | 1264 | 1265 | 1266 | 1267 | 1268 | 1269 | 1270 | 1271 | 1272 | 1273 | 1274 | 1275 | 1276 | 1277 | 1278 | 1279 | 1280 | 1281 | 1282 | 1283 | 1284 | 1285 | 1286 | 1287 | 1288 | 1289 | 1290 | 1291 | 1292 | 1293 | 1294 | 1295 | 1296 | 1297 | 1298 | 1299 | 1300 | 1301 | 1302 | 1303 | 1304 | 1305 | 1306 | 1307 | 1308 | 1309 | 1310 | 1311 | 1312 | 1313 | 1314 | 1315 | 1316 | 1317 | 1318 | 1319 | 1320 | 1321 | 1322 | 1323 | 1324 | 1325 | 1326 | 1327 | 1328 | 1329 | 1330 | 1331 | 1332 | 1333 | 1334 | 1335 | 1336 | 1337 | 1338 | 1339 | 1340 | 1341 | 1342 | 1343 | 1344 | 1345 | 1346 | 1347 | 1348 | 1349 | 1350 | 1351 | --------------------------------------------------------------------------------