')
96 |
97 | if (scribe === 'component:card') {
98 | const photo = el.children('[data-scribe="element:photo"]')
99 | const photoGrid = el.children('[data-scribe="element:photo_grid"]')
100 | const photos = photo.length ? photo : photoGrid
101 |
102 | if (photos.length) {
103 | const images = photos.find('img')
104 |
105 | images.each(function () {
106 | const img = $(this)
107 | const alt = img.attr('alt')
108 | const url = img.attr('data-image')
109 | const format = img.attr('data-image-format')
110 | const height = img.attr('height')
111 | const width = img.attr('width')
112 |
113 | this.attribs = {
114 | 'data-type': 'media-image',
115 | 'data-url': tweetUrl,
116 | src: `${url}?format=${format}`,
117 | height,
118 | width
119 | }
120 | if (alt) {
121 | this.attribs.alt = alt
122 | }
123 | // Move the media img to a new container
124 | media.append(img)
125 | })
126 | media.attr('data-type', `image-container ${images.length}`)
127 | mediaHtml = $('
').append(media).html()
128 | }
129 | }
130 | })
131 |
132 | tweetContent.children('img').each(function () {
133 | const props = this.attribs
134 |
135 | // Handle emojis inside the text
136 | if (props.class?.includes('Emoji--forText')) {
137 | this.attribs = {
138 | 'data-type': 'emoji-for-text',
139 | src: props.src,
140 | alt: props.alt
141 | }
142 | return
143 | }
144 |
145 | console.error(
146 | 'An image with the following props is not being handled:',
147 | props
148 | )
149 | })
150 |
151 | tweetContent.children('a').each(function () {
152 | const props = this.attribs
153 | const scribe = props['data-scribe']
154 | const el = $(this)
155 | const asTwitterLink = (type) => {
156 | this.attribs = {
157 | 'data-type': type,
158 | href: props.href
159 | }
160 | // Replace custom tags inside the anchor with text
161 | el.text(el.text())
162 | }
163 |
164 | // @mention
165 | if (scribe === 'element:mention') {
166 | return asTwitterLink('mention')
167 | }
168 |
169 | // #hashtag
170 | if (scribe === 'element:hashtag') {
171 | // A hashtag may be a $cashtag too
172 | const type =
173 | props['data-query-source'] === 'cashtag_click' ? 'cashtag' : 'hashtag'
174 | return asTwitterLink(type)
175 | }
176 |
177 | if (scribe === 'element:url') {
178 | const url = props['data-expanded-url']
179 | // const quotedTweetId = props['data-tweet-id']
180 |
181 | // Remove link to quoted tweet to leave the card only
182 | // if (quotedTweetId && quotedTweetId === quotedTweet?.id) {
183 | // el.remove();
184 | // return;
185 | // }
186 |
187 | // Handle normal links
188 | const text = { type: 'text', data: url }
189 | // Replace the link with plain text and markdown will take care of it
190 | el.replaceWith(text)
191 | }
192 | })
193 |
194 | content.html = tweetContent.html()
195 |
196 | if (quotedTweet) content.quotedTweet = quotedTweet
197 | if (mediaHtml) content.mediaHtml = mediaHtml
198 |
199 | return content
200 | }
201 |
202 | export function getTweetData(html) {
203 | const $ = cheerio.load(html, {
204 | decodeEntities: false,
205 | xmlMode: false
206 | })
207 | const tweetContent = getTweetContent($)
208 |
209 | return tweetContent
210 | }
211 |
--------------------------------------------------------------------------------
/packages/static-tweets/src/twitter/getTweetHtml.ts:
--------------------------------------------------------------------------------
1 | import { getVideo } from './tweet-html'
2 | import {
3 | fetchUserStatus,
4 | getEmbeddedTweetHtml,
5 | fetchTweetWithPoll
6 | } from './api'
7 | import { fetchTweetAst } from '../fetchTweetAst'
8 | import markdownToAst from '../markdown/markdownToAst'
9 |
10 | function getVideoData(userStatus) {
11 | const video = userStatus.extended_entities.media[0]
12 | const poster = video.media_url_https
13 | // Find the first mp4 video in the array, if the results are always properly sorted, then
14 | // it should always be the mp4 video with the lowest bitrate
15 | const mp4Video = video.video_info.variants.find(
16 | (v) => v.content_type === 'video/mp4'
17 | )
18 |
19 | if (!mp4Video) return
20 |
21 | return { poster, ...mp4Video }
22 | }
23 |
24 | function getPollData(tweet) {
25 | const polls = tweet.includes && tweet.includes.polls
26 | return polls && polls[0]
27 | }
28 |
29 | async function getMediaHtml(tweet) {
30 | let media = tweet.mediaHtml
31 |
32 | if (tweet.hasVideo) {
33 | const userStatus = await fetchUserStatus(tweet.meta.id)
34 | const video = userStatus && getVideoData(userStatus)
35 |
36 | media = video ? getVideo(media, video) : null
37 | }
38 |
39 | return media
40 | }
41 |
42 | async function getQuotedTweetHtml({ quotedTweet }, context) {
43 | if (!quotedTweet) return
44 |
45 | if (process.env.NEXT_PUBLIC_TWITTER_LOAD_WIDGETS === 'true') {
46 | const data = await getEmbeddedTweetHtml(quotedTweet.url)
47 | return data?.html
48 | } else {
49 | const ast = await fetchTweetAst(quotedTweet.id)
50 | // The AST of embedded tweets is always sent as data
51 | return ast && `
`
52 | }
53 | }
54 |
55 | async function getPollHtml(tweet, context) {
56 | if (!tweet.hasPoll) return null
57 |
58 | const tweetData = await fetchTweetWithPoll(tweet.meta.id)
59 | const poll = tweetData && getPollData(tweetData)
60 |
61 | if (poll) {
62 | const meta = {
63 | type: 'poll-container',
64 | endsAt: poll.end_datetime,
65 | duration: poll.duration_minutes,
66 | status: poll.voting_status,
67 | options: poll.options
68 | }
69 |
70 | return `
`
71 | }
72 |
73 | return null
74 | }
75 |
76 | export default async function getTweetHtml(tweet, context) {
77 | const meta = { ...tweet.meta, type: 'tweet' }
78 | const md: any = await markdownToAst(tweet.html)
79 |
80 | const html = [
81 | // md.children is the markdown content, which is later added as children to the container
82 | `
`,
83 | (await getMediaHtml(tweet)) || '',
84 | (await getQuotedTweetHtml(tweet, context)) || '',
85 | (await getPollHtml(tweet, context)) || '',
86 | `
`
87 | ].join('')
88 |
89 | return html
90 | }
91 |
--------------------------------------------------------------------------------
/packages/static-tweets/src/twitter/tweet-html.ts:
--------------------------------------------------------------------------------
1 | import path from 'path'
2 | import cheerio from 'cheerio'
3 | import { URL } from 'url'
4 |
5 | const TWEET_VIDEO_URL = 'https://video.twimg.com/tweet_video'
6 |
7 | // Could we use rehype directly and remove cheerio?
8 | function getTweetContent($, tweet, isMainTweet = true) {
9 | if (!tweet.length) return
10 |
11 | const meta: any = {}
12 | const content: any = { meta }
13 | const tweetContent = tweet.children('.js-tweet-text-container').children('p')
14 | const actions = tweet
15 | .children('.stream-item-footer')
16 | .children('.ProfileTweet-actionCountList')
17 | .children()
18 | const hasCard =
19 | tweet.children('.js-tweet-details-fixer').children('.card2').length > 0
20 |
21 | let quotedTweet
22 | let mediaHtml
23 | let hasVideo = false
24 |
25 | // Add the user avatar and date to the tweet only if it's the main tweet
26 | if (isMainTweet) {
27 | const avatar = tweet.find('.account-group').children('.avatar')
28 | const time = tweet.find('a.tweet-timestamp').children('span')
29 |
30 | meta.avatar = { bigger: avatar.attr('src') }
31 | meta.createdAt = Number(time.attr('data-time-ms'))
32 | }
33 |
34 | tweetContent.children('img').each(function () {
35 | const props = this.attribs
36 |
37 | // Handle emojis inside the text
38 | if (props.class && props.class.includes('Emoji--forText')) {
39 | this.attribs = {
40 | 'data-type': 'emoji-for-text',
41 | src: props.src,
42 | alt: props.alt
43 | }
44 | return
45 | }
46 |
47 | console.error(
48 | 'An image with the following props is not being handled:',
49 | props
50 | )
51 | })
52 |
53 | tweetContent.children('a').each(function () {
54 | const props = this.attribs
55 | const el = $(this)
56 |
57 | if (props['data-expanded-url']) {
58 | const url = props['data-expanded-url']
59 | const quotedTweetPath = tweet
60 | .children('.QuoteTweet')
61 | .find('.QuoteTweet-link')
62 | .attr('href')
63 |
64 | // Embedded Tweet
65 | if (quotedTweetPath && url.endsWith(quotedTweetPath)) {
66 | quotedTweet = { url }
67 | el.remove()
68 | return
69 | }
70 |
71 | // If Twitter is hiding the link, it's because it's adding a card with a preview
72 | const isLinkPreview = props.class && props.class.includes('u-hidden')
73 |
74 | if (isLinkPreview) {
75 | // In the case of a preview we only add a line break between the link and the paragraph.
76 | // TODO: Add the preview HTML and remove the link
77 | el.before('
')
78 | }
79 |
80 | // Handle normal links
81 | const text = { type: 'text', data: url }
82 | // Replace the link with plain text and markdown will take care of it
83 | el.replaceWith(text)
84 |
85 | return
86 | }
87 |
88 | // Embedded media
89 | if (props['data-pre-embedded'] === 'true') {
90 | const adaptiveMedia = tweet
91 | .children('.AdaptiveMediaOuterContainer')
92 | .children('.AdaptiveMedia')
93 | const isVideo = adaptiveMedia.hasClass('is-video')
94 | const media = $('
')
95 |
96 | // Videos and gifs
97 | if (isVideo) {
98 | const img = adaptiveMedia
99 | .find('.PlayableMedia-player')
100 | .css('background-image')
101 | const url = new URL(img.slice(4, -1).replace(/['"]/g, ''))
102 | const fileName = path.basename(url.pathname)
103 | const ext = path.extname(fileName)
104 | const videoUrl = `${TWEET_VIDEO_URL}/${fileName.replace(ext, '.mp4')}`
105 |
106 | // Gifs
107 | if (url.pathname.startsWith('/tweet_video_thumb')) {
108 | const video = $(
109 | `