├── .dockerignore
├── .gitattributes
├── .github
└── workflows
│ └── docker.yml
├── Dockerfile
├── LICENSE.txt
├── README.md
├── cmd
└── unfurlist
│ └── unfurlist.go
├── conf.go
├── data
└── providers.json
├── favicon.go
├── favicon_test.go
├── fetcher.go
├── go.mod
├── go.sum
├── googlemaps.go
├── googlemaps_test.go
├── html_meta_parser.go
├── html_meta_parser_test.go
├── image.go
├── internal
└── useragent
│ ├── LICENSE.txt
│ └── useragent.go
├── oembed_parser.go
├── opengraph_parser.go
├── prefixmap.go
├── prefixmap_test.go
├── remote-data-update.go
├── testdata
├── japanese
├── korean
├── no-charset-in-first-1024bytes
└── remote-dump.json
├── unfurlist.go
├── unfurlist_test.go
├── url_parser.go
├── url_parser_test.go
└── youtube.go
/.dockerignore:
--------------------------------------------------------------------------------
1 | Dockerfile
2 | .github
3 |
--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
1 | testdata/* -diff
2 |
--------------------------------------------------------------------------------
/.github/workflows/docker.yml:
--------------------------------------------------------------------------------
1 | name: Publish Docker image
2 | on:
3 | release:
4 | types: [published]
5 | push:
6 | branches:
7 | - master
8 | workflow_dispatch:
9 |
10 | env:
11 | IMAGE_NAME: unfurlist
12 |
13 | jobs:
14 | test:
15 | name: Test suite
16 | runs-on: ubuntu-latest
17 | timeout-minutes: 60
18 | steps:
19 | - uses: actions/setup-go@v5
20 | with:
21 | go-version: 'stable'
22 | - uses: actions/checkout@v4
23 | - name: Get dependencies
24 | run: go mod download
25 | - name: Run tests
26 | run: go test -v -race ./...
27 | push:
28 | needs: test
29 | name: Build and push Docker image
30 | runs-on: ubuntu-latest
31 | timeout-minutes: 60
32 | steps:
33 | - uses: actions/checkout@v4
34 | - name: Build image
35 | run: |
36 | docker build . --tag image
37 | - name: Log into registry
38 | run: echo "${{ secrets.GH_PACKAGES_TOKEN }}" | docker login ghcr.io -u ${GITHUB_ACTOR} --password-stdin
39 | - name: Push image
40 | run: |
41 | set -u
42 | # https://help.github.com/en/actions/automating-your-workflow-with-github-actions/using-environment-variables
43 | IMAGE_ID=ghcr.io/doist/$IMAGE_NAME
44 |
45 | # Strip git ref prefix from version
46 | VERSION=${GITHUB_REF##*/}
47 |
48 | # Strip "v" prefix from tag name
49 | case "${GITHUB_REF}" in refs/tags/*) VERSION=${VERSION#v} ;; esac
50 |
51 | case "$VERSION" in master) VERSION=latest ;; esac
52 |
53 | echo IMAGE_ID=$IMAGE_ID
54 | echo VERSION=$VERSION
55 | echo GITHUB_REF=$GITHUB_REF
56 |
57 | docker tag image $IMAGE_ID:$VERSION
58 | docker push $IMAGE_ID:$VERSION
59 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM public.ecr.aws/docker/library/golang:alpine AS builder
2 | RUN apk add git
3 | WORKDIR /app
4 | ENV GOPROXY=https://proxy.golang.org CGO_ENABLED=0
5 | COPY go.mod go.sum ./
6 | RUN go mod download
7 | COPY . ./
8 | RUN go build -ldflags='-s -w' -o main ./cmd/unfurlist
9 |
10 | FROM scratch
11 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
12 | COPY --from=builder /app/main /bin/main
13 | EXPOSE :8080
14 | CMD ["/bin/main", "-pprof=''", "-listen=:8080"]
15 |
--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2016 Doist
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | Package unfurlist implements a service that unfurls URLs and provides more
2 | information about them.
3 |
4 | To install ready-to-use http service:
5 |
6 | go get -u github.com/Doist/unfurlist/...
7 |
8 | See [documentation](https://godoc.org/github.com/Doist/unfurlist).
9 |
--------------------------------------------------------------------------------
/cmd/unfurlist/unfurlist.go:
--------------------------------------------------------------------------------
1 | // Command unfurlist implements http server exposing API endpoint
2 | package main
3 |
4 | import (
5 | "bufio"
6 | "bytes"
7 | "context"
8 | "errors"
9 | "flag"
10 | "io"
11 | "log"
12 | "net"
13 | "net/http"
14 | _ "net/http/pprof"
15 | "net/url"
16 | "os"
17 | "path"
18 | "regexp"
19 | "strings"
20 | "time"
21 |
22 | "github.com/Doist/unfurlist"
23 | "github.com/Doist/unfurlist/internal/useragent"
24 | "github.com/artyom/autoflags"
25 | "github.com/artyom/oembed"
26 | "github.com/bradfitz/gomemcache/memcache"
27 | )
28 |
29 | func main() {
30 | args := struct {
31 | Listen string `flag:"listen,address to listen, set both -sslcert and -sslkey for HTTPS"`
32 | Pprof string `flag:"pprof,address to serve pprof data"`
33 | Cert string `flag:"sslcert,path to certificate file (PEM format)"`
34 | Key string `flag:"sslkey,path to certificate file (PEM format)"`
35 | Cache string `flag:"cache,address of memcached, disabled if empty"`
36 | Blocklist string `flag:"blocklist,file with url prefixes to block, one per line"`
37 | WithDimensions bool `flag:"withDimensions,return image dimensions if possible (extra request to fetch image)"`
38 | Timeout time.Duration `flag:"timeout,timeout for remote i/o"`
39 | GoogleMapsKey string `flag:"googlemapskey,Google Static Maps API key to generate map previews"`
40 | VideoDomains string `flag:"videoDomains,comma-separated list of domains that host video+thumbnails"`
41 | MaxResults int `flag:"max,maximum number of results to get for single request"`
42 | Ping bool `flag:"ping,respond with 200 OK on /ping path (for health checks)"`
43 | OembedProviders string `flag:"oembedProviders,custom oembed providers list in json format"`
44 | }{
45 | Listen: "localhost:8080",
46 | Timeout: 30 * time.Second,
47 | MaxResults: unfurlist.DefaultMaxResults,
48 | }
49 | var discard string
50 | flag.StringVar(&discard, "image.proxy.url", "", "DEPRECATED and unused")
51 | flag.StringVar(&discard, "image.proxy.secret", "", "DEPRECATED and unused")
52 | flag.StringVar(&args.Blocklist, "blacklist", args.Blocklist, "DEPRECATED: use -blocklist instead")
53 | autoflags.Define(&args)
54 | flag.Parse()
55 |
56 | if args.Timeout < 0 {
57 | args.Timeout = 0
58 | }
59 | httpClient := &http.Client{
60 | CheckRedirect: failOnLoginPages,
61 | Timeout: args.Timeout,
62 | Transport: useragent.Set(&http.Transport{
63 | Proxy: http.ProxyFromEnvironment,
64 | DialContext: (&net.Dialer{
65 | Timeout: 10 * time.Second,
66 | KeepAlive: 30 * time.Second,
67 | DualStack: true,
68 | }).DialContext,
69 | MaxIdleConns: 100,
70 | IdleConnTimeout: 90 * time.Second,
71 | TLSHandshakeTimeout: 10 * time.Second,
72 | ExpectContinueTimeout: 1 * time.Second,
73 | }, "unfurlist (https://github.com/Doist/unfurlist)"),
74 | }
75 | logFlags := log.LstdFlags
76 | if os.Getenv("AWS_EXECUTION_ENV") != "" {
77 | logFlags = 0
78 | }
79 | configs := []unfurlist.ConfFunc{
80 | unfurlist.WithExtraHeaders(map[string]string{
81 | "Accept-Language": "en;q=1, *;q=0.5",
82 | }),
83 | unfurlist.WithLogger(log.New(os.Stderr, "", logFlags)),
84 | unfurlist.WithHTTPClient(httpClient),
85 | unfurlist.WithImageDimensions(args.WithDimensions),
86 | unfurlist.WithBlocklistTitles(titleBlocklist),
87 | unfurlist.WithMaxResults(args.MaxResults),
88 | }
89 | if args.OembedProviders != "" {
90 | data, err := os.ReadFile(args.OembedProviders)
91 | if err != nil {
92 | log.Fatal(err)
93 | }
94 | fn, err := oembed.Providers(bytes.NewReader(data))
95 | if err != nil {
96 | log.Fatal(err)
97 | }
98 | configs = append(configs, unfurlist.WithOembedLookupFunc(fn))
99 | }
100 | if args.Blocklist != "" {
101 | prefixes, err := readBlocklist(args.Blocklist)
102 | if err != nil {
103 | log.Fatal(err)
104 | }
105 | configs = append(configs, unfurlist.WithBlocklistPrefixes(prefixes))
106 | }
107 | if args.Cache != "" {
108 | log.Print("Enable cache at ", args.Cache)
109 | configs = append(configs, unfurlist.WithMemcache(memcache.New(args.Cache)))
110 | }
111 |
112 | var ff []unfurlist.FetchFunc
113 | if args.GoogleMapsKey != "" {
114 | ff = append(ff, unfurlist.GoogleMapsFetcher(args.GoogleMapsKey))
115 | }
116 | if args.VideoDomains != "" {
117 | ff = append(ff, videoThumbnailsFetcher(strings.Split(args.VideoDomains, ",")...))
118 | }
119 | if ff != nil {
120 | configs = append(configs, unfurlist.WithFetchers(ff...))
121 | }
122 |
123 | handler := unfurlist.New(configs...)
124 | if args.Pprof != "" {
125 | go func(addr string) { log.Println(http.ListenAndServe(addr, nil)) }(args.Pprof)
126 | }
127 | go func() {
128 | // on a highly used system unfurlist can accumulate a lot of
129 | // idle connections occupying memory; force periodic close of
130 | // them
131 | for range time.NewTicker(2 * time.Minute).C {
132 | if c, ok := httpClient.Transport.(interface {
133 | CloseIdleConnections()
134 | }); ok {
135 | c.CloseIdleConnections()
136 | }
137 | }
138 | }()
139 | mux := http.NewServeMux()
140 | mux.Handle("/", handler)
141 | if args.Ping {
142 | mux.HandleFunc("/ping", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })
143 | }
144 | srv := &http.Server{
145 | Addr: args.Listen,
146 | ReadTimeout: 5 * time.Second,
147 | WriteTimeout: 60 * time.Second,
148 | IdleTimeout: 30 * time.Second,
149 | Handler: mux,
150 | }
151 | if args.Cert != "" && args.Key != "" {
152 | log.Fatal(srv.ListenAndServeTLS(args.Cert, args.Key))
153 | } else {
154 | log.Fatal(srv.ListenAndServe())
155 | }
156 | }
157 |
158 | func readBlocklist(blocklist string) ([]string, error) {
159 | f, err := os.Open(blocklist)
160 | if err != nil {
161 | return nil, err
162 | }
163 | defer f.Close()
164 | s := bufio.NewScanner(io.LimitReader(f, 512*1024))
165 | var prefixes []string
166 | for s.Scan() {
167 | if bytes.HasPrefix(s.Bytes(), []byte("http")) {
168 | prefixes = append(prefixes, s.Text())
169 | }
170 | }
171 | if err := s.Err(); err != nil {
172 | return nil, err
173 | }
174 | return prefixes, nil
175 | }
176 |
177 | // failOnLoginPages can be used as http.Client.CheckRedirect to skip redirects
178 | // to login pages of most commonly used services or most commonly named login
179 | // pages. It also checks depth of redirect chain and stops on more then 10
180 | // consecutive redirects.
181 | func failOnLoginPages(req *http.Request, via []*http.Request) error {
182 | if len(via) >= 10 {
183 | return errors.New("stopped after 10 redirects")
184 | }
185 | if l := len(via); l > 0 && *req.URL == *via[l-1].URL {
186 | return errors.New("redirect loop")
187 | }
188 | if strings.Contains(strings.ToLower(req.URL.Host), "login") ||
189 | loginPathRe.MatchString(req.URL.Path) {
190 | return errWantLogin
191 | }
192 | u := *req.URL
193 | u.RawQuery, u.Fragment = "", ""
194 | if _, ok := loginPages[(&u).String()]; ok {
195 | return errWantLogin
196 | }
197 | return nil
198 | }
199 |
200 | var loginPathRe = regexp.MustCompile(`(?i)login|sign.?in`)
201 |
202 | var errWantLogin = errors.New("resource requires login")
203 |
204 | // loginPages is a set of popular services' known login pages
205 | var loginPages map[string]struct{}
206 |
207 | func init() {
208 | pages := []string{
209 | "https://bitbucket.org/account/signin/",
210 | "https://outlook.live.com/owa/",
211 | }
212 | loginPages = make(map[string]struct{}, len(pages))
213 | for _, u := range pages {
214 | loginPages[u] = struct{}{}
215 | }
216 |
217 | }
218 |
219 | var titleBlocklist = []string{
220 | "robot check", // Amazon
221 | }
222 |
223 | // videoThumbnailsFetcher return unfurlist.FetchFunc that returns metadata
224 | // with url to video thumbnail file for supported domains.
225 | func videoThumbnailsFetcher(domains ...string) func(context.Context, *http.Client, *url.URL) (*unfurlist.Metadata, bool) {
226 | doms := make(map[string]struct{})
227 | for _, d := range domains {
228 | doms[d] = struct{}{}
229 | }
230 | return func(_ context.Context, _ *http.Client, u *url.URL) (*unfurlist.Metadata, bool) {
231 | if _, ok := doms[u.Host]; !ok {
232 | return nil, false
233 | }
234 | switch strings.ToLower(path.Ext(u.Path)) {
235 | default:
236 | return nil, false
237 | case ".mp4", ".mov", ".m4v", ".3gp", ".webm", ".mkv":
238 | }
239 | u2 := &url.URL{
240 | Scheme: u.Scheme,
241 | Host: u.Host,
242 | Path: u.Path + ".thumb",
243 | }
244 | return &unfurlist.Metadata{
245 | Title: path.Base(u.Path),
246 | Type: "video",
247 | Image: u2.String(),
248 | }, true
249 | }
250 | }
251 |
--------------------------------------------------------------------------------
/conf.go:
--------------------------------------------------------------------------------
1 | package unfurlist
2 |
3 | import (
4 | "net/http"
5 | "strings"
6 |
7 | "github.com/artyom/oembed"
8 | "github.com/bradfitz/gomemcache/memcache"
9 | )
10 |
11 | // WithHTTPClient configures unfurl handler to use provided http.Client for
12 | // outgoing requests
13 | func WithHTTPClient(client *http.Client) ConfFunc {
14 | return func(h *unfurlHandler) *unfurlHandler {
15 | if client != nil {
16 | h.HTTPClient = client
17 | }
18 | return h
19 | }
20 | }
21 |
22 | // WithMemcache configures unfurl handler to cache metadata in memcached
23 | func WithMemcache(client *memcache.Client) ConfFunc {
24 | return func(h *unfurlHandler) *unfurlHandler {
25 | if client != nil {
26 | h.Cache = client
27 | }
28 | return h
29 | }
30 | }
31 |
32 | // WithExtraHeaders configures unfurl handler to add extra headers to each
33 | // outgoing http request
34 | func WithExtraHeaders(hdr map[string]string) ConfFunc {
35 | headers := make([]string, 0, len(hdr)*2)
36 | for k, v := range hdr {
37 | headers = append(headers, k, v)
38 | }
39 | return func(h *unfurlHandler) *unfurlHandler {
40 | h.Headers = headers
41 | return h
42 | }
43 | }
44 |
45 | // WithBlocklistPrefixes configures unfurl handler to skip unfurling urls
46 | // matching any provided prefix
47 | func WithBlocklistPrefixes(prefixes []string) ConfFunc {
48 | var pmap *prefixMap
49 | if len(prefixes) > 0 {
50 | pmap = newPrefixMap(prefixes)
51 | }
52 | return func(h *unfurlHandler) *unfurlHandler {
53 | if pmap != nil {
54 | h.pmap = pmap
55 | }
56 | return h
57 | }
58 | }
59 |
60 | // WithBlocklistTitles configures unfurl handler to skip unfurling urls that
61 | // return pages which title contains one of substrings provided
62 | func WithBlocklistTitles(substrings []string) ConfFunc {
63 | ss := make([]string, len(substrings))
64 | for i, s := range substrings {
65 | ss[i] = strings.ToLower(s)
66 | }
67 | return func(h *unfurlHandler) *unfurlHandler {
68 | if len(ss) > 0 {
69 | h.titleBlocklist = ss
70 | }
71 | return h
72 | }
73 | }
74 |
75 | // WithImageDimensions configures unfurl handler whether to fetch image
76 | // dimensions or not.
77 | func WithImageDimensions(enable bool) ConfFunc {
78 | return func(h *unfurlHandler) *unfurlHandler {
79 | h.FetchImageSize = enable
80 | return h
81 | }
82 | }
83 |
84 | // WithFetchers attaches custom fetchers to unfurl handler created by New().
85 | func WithFetchers(fetchers ...FetchFunc) ConfFunc {
86 | return func(h *unfurlHandler) *unfurlHandler {
87 | h.fetchers = fetchers
88 | return h
89 | }
90 | }
91 |
92 | // WithMaxResults configures unfurl handler to only process n first urls it
93 | // finds. n must be positive.
94 | func WithMaxResults(n int) ConfFunc {
95 | return func(h *unfurlHandler) *unfurlHandler {
96 | if n > 0 {
97 | h.maxResults = n
98 | }
99 | return h
100 | }
101 | }
102 |
103 | // WithOembedLookupFunc configures unfurl handler to use custom
104 | // oembed.LookupFunc for oembed lookups.
105 | func WithOembedLookupFunc(fn oembed.LookupFunc) ConfFunc {
106 | return func(h *unfurlHandler) *unfurlHandler {
107 | if fn != nil {
108 | h.oembedLookupFunc = fn
109 | }
110 | return h
111 | }
112 | }
113 |
114 | // WithLogger configures unfurl handler to use provided logger
115 | func WithLogger(l Logger) ConfFunc {
116 | return func(h *unfurlHandler) *unfurlHandler {
117 | if l != nil {
118 | h.Log = l
119 | }
120 | return h
121 | }
122 | }
123 |
124 | // Logger describes set of methods used by unfurl handler for logging; standard
125 | // lib *log.Logger implements this interface.
126 | type Logger interface {
127 | Print(v ...any)
128 | Printf(format string, v ...any)
129 | Println(v ...any)
130 | }
131 |
--------------------------------------------------------------------------------
/data/providers.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "provider_name": "23HQ",
4 | "provider_url": "http:\/\/www.23hq.com",
5 | "endpoints": [
6 | {
7 | "schemes": [
8 | "http:\/\/www.23hq.com\/*\/photo\/*"
9 | ],
10 | "url": "http:\/\/www.23hq.com\/23\/oembed"
11 | }
12 | ]
13 | },
14 | {
15 | "provider_name": "Abraia",
16 | "provider_url": "https:\/\/abraia.me",
17 | "endpoints": [
18 | {
19 | "schemes": [
20 | "https:\/\/store.abraia.me\/*"
21 | ],
22 | "url": "https:\/\/api.abraia.me\/oembed",
23 | "discovery": true
24 | }
25 | ]
26 | },
27 | {
28 | "provider_name": "Adways",
29 | "provider_url": "http:\/\/www.adways.com",
30 | "endpoints": [
31 | {
32 | "schemes": [
33 | "http:\/\/play.adpaths.com\/experience\/*"
34 | ],
35 | "url": "http:\/\/play.adpaths.com\/oembed\/*"
36 | }
37 | ]
38 | },
39 | {
40 | "provider_name": "Alpha App Net",
41 | "provider_url": "https:\/\/alpha.app.net\/browse\/posts\/",
42 | "endpoints": [
43 | {
44 | "schemes": [
45 | "https:\/\/alpha.app.net\/*\/post\/*",
46 | "https:\/\/photos.app.net\/*\/*"
47 | ],
48 | "url": "https:\/\/alpha-api.app.net\/oembed",
49 | "formats": [
50 | "json"
51 | ]
52 | }
53 | ]
54 | },
55 | {
56 | "provider_name": "Altru",
57 | "provider_url": "https:\/\/www.altrulabs.com",
58 | "endpoints": [
59 | {
60 | "schemes": [
61 | "https:\/\/app.altrulabs.com\/*\/*?answer_id=*"
62 | ],
63 | "url": "https:\/\/api.altrulabs.com\/social\/oembed",
64 | "formats": [
65 | "json"
66 | ]
67 | }
68 | ]
69 | },
70 | {
71 | "provider_name": "amCharts Live Editor",
72 | "provider_url": "https:\/\/live.amcharts.com\/",
73 | "endpoints": [
74 | {
75 | "schemes": [
76 | "http:\/\/live.amcharts.com\/*",
77 | "https:\/\/live.amcharts.com\/*"
78 | ],
79 | "url": "https:\/\/live.amcharts.com\/oembed"
80 | }
81 | ]
82 | },
83 | {
84 | "provider_name": "Animatron",
85 | "provider_url": "https:\/\/www.animatron.com\/",
86 | "endpoints": [
87 | {
88 | "schemes": [
89 | "https:\/\/www.animatron.com\/project\/*",
90 | "https:\/\/animatron.com\/project\/*"
91 | ],
92 | "url": "https:\/\/animatron.com\/oembed\/json",
93 | "discovery": true
94 | }
95 | ]
96 | },
97 | {
98 | "provider_name": "Animoto",
99 | "provider_url": "http:\/\/animoto.com\/",
100 | "endpoints": [
101 | {
102 | "schemes": [
103 | "http:\/\/animoto.com\/play\/*"
104 | ],
105 | "url": "http:\/\/animoto.com\/oembeds\/create"
106 | }
107 | ]
108 | },
109 | {
110 | "provider_name": "Apester",
111 | "provider_url": "https:\/\/www.apester.com",
112 | "endpoints": [
113 | {
114 | "schemes": [
115 | "https:\/\/renderer.apester.com\/v2\/*?preview=true&iframe_preview=true"
116 | ],
117 | "url": "https:\/\/display.apester.com\/oembed",
118 | "discovery": true
119 | }
120 | ]
121 | },
122 | {
123 | "provider_name": "ArcGIS StoryMaps",
124 | "provider_url": "https:\/\/storymaps.arcgis.com",
125 | "endpoints": [
126 | {
127 | "schemes": [
128 | "https:\/\/storymaps.arcgis.com\/stories\/*"
129 | ],
130 | "url": "https:\/\/storymaps.arcgis.com\/oembed",
131 | "discovery": true
132 | }
133 | ]
134 | },
135 | {
136 | "provider_name": "Archivos",
137 | "provider_url": "https:\/\/app.archivos.digital",
138 | "endpoints": [
139 | {
140 | "schemes": [
141 | "https:\/\/app.archivos.digital\/app\/view\/*"
142 | ],
143 | "url": "https:\/\/app.archivos.digital\/oembed\/"
144 | }
145 | ]
146 | },
147 | {
148 | "provider_name": "Audioboom",
149 | "provider_url": "https:\/\/audioboom.com",
150 | "endpoints": [
151 | {
152 | "schemes": [
153 | "https:\/\/audioboom.com\/channel\/*",
154 | "https:\/\/audioboom.com\/posts\/*"
155 | ],
156 | "url": "https:\/\/audioboom.com\/publishing\/oembed\/v4.{format}",
157 | "formats": [
158 | "json",
159 | "xml"
160 | ]
161 | }
162 | ]
163 | },
164 | {
165 | "provider_name": "AudioClip",
166 | "provider_url": "https:\/\/audioclip.naver.com",
167 | "endpoints": [
168 | {
169 | "schemes": [
170 | "https:\/\/audioclip.naver.com\/channels\/*\/clips\/*",
171 | "https:\/\/audioclip.naver.com\/audiobooks\/*"
172 | ],
173 | "url": "https:\/\/audioclip.naver.com\/oembed",
174 | "discovery": true
175 | }
176 | ]
177 | },
178 | {
179 | "provider_name": "Audiomack",
180 | "provider_url": "https:\/\/www.audiomack.com",
181 | "endpoints": [
182 | {
183 | "schemes": [
184 | "https:\/\/www.audiomack.com\/song\/*",
185 | "https:\/\/www.audiomack.com\/album\/*",
186 | "https:\/\/www.audiomack.com\/playlist\/*"
187 | ],
188 | "url": "https:\/\/www.audiomack.com\/oembed",
189 | "discovery": true
190 | }
191 | ]
192 | },
193 | {
194 | "provider_name": "AudioSnaps",
195 | "provider_url": "http:\/\/audiosnaps.com",
196 | "endpoints": [
197 | {
198 | "schemes": [
199 | "http:\/\/audiosnaps.com\/k\/*"
200 | ],
201 | "url": "http:\/\/audiosnaps.com\/service\/oembed",
202 | "discovery": true
203 | }
204 | ]
205 | },
206 | {
207 | "provider_name": "Avocode",
208 | "provider_url": "https:\/\/www.avocode.com\/",
209 | "endpoints": [
210 | {
211 | "schemes": [
212 | "https:\/\/app.avocode.com\/view\/*"
213 | ],
214 | "url": "https:\/\/stage-embed.avocode.com\/api\/oembed",
215 | "formats": [
216 | "json"
217 | ]
218 | }
219 | ]
220 | },
221 | {
222 | "provider_name": "Backtracks",
223 | "provider_url": "https:\/\/backtracks.fm",
224 | "endpoints": [
225 | {
226 | "schemes": [
227 | "https:\/\/backtracks.fm\/*\/*\/e\/*",
228 | "https:\/\/backtracks.fm\/*\/s\/*\/*",
229 | "https:\/\/backtracks.fm\/*\/*\/*\/*\/e\/*\/*",
230 | "https:\/\/backtracks.fm\/*",
231 | "http:\/\/backtracks.fm\/*"
232 | ],
233 | "url": "https:\/\/backtracks.fm\/oembed",
234 | "discovery": true
235 | }
236 | ]
237 | },
238 | {
239 | "provider_name": "Beautiful.AI",
240 | "provider_url": "https:\/\/www.beautiful.ai\/",
241 | "endpoints": [
242 | {
243 | "url": "https:\/\/www.beautiful.ai\/api\/oembed",
244 | "discovery": true
245 | }
246 | ]
247 | },
248 | {
249 | "provider_name": "Blackfire.io",
250 | "provider_url": "https:\/\/blackfire.io",
251 | "endpoints": [
252 | {
253 | "schemes": [
254 | "https:\/\/blackfire.io\/profiles\/*\/graph",
255 | "https:\/\/blackfire.io\/profiles\/compare\/*\/graph"
256 | ],
257 | "url": "https:\/\/blackfire.io\/oembed",
258 | "discovery": true
259 | }
260 | ]
261 | },
262 | {
263 | "provider_name": "Blogcast",
264 | "provider_url": "https:\/\/blogcast.host\/",
265 | "endpoints": [
266 | {
267 | "schemes": [
268 | "https:\/\/blogcast.host\/embed\/*",
269 | "https:\/\/blogcast.host\/embedly\/*"
270 | ],
271 | "url": "https:\/\/blogcast.host\/oembed",
272 | "discovery": true
273 | }
274 | ]
275 | },
276 | {
277 | "provider_name": "Box Office Buz",
278 | "provider_url": "http:\/\/boxofficebuz.com",
279 | "endpoints": [
280 | {
281 | "url": "http:\/\/boxofficebuz.com\/oembed",
282 | "discovery": true
283 | }
284 | ]
285 | },
286 | {
287 | "provider_name": "BrioVR",
288 | "provider_url": "https:\/\/view.briovr.com\/",
289 | "endpoints": [
290 | {
291 | "schemes": [
292 | "https:\/\/view.briovr.com\/api\/v1\/worlds\/oembed\/*"
293 | ],
294 | "url": "https:\/\/view.briovr.com\/api\/v1\/worlds\/oembed\/"
295 | }
296 | ]
297 | },
298 | {
299 | "provider_name": "Buttondown",
300 | "provider_url": "https:\/\/buttondown.email\/",
301 | "endpoints": [
302 | {
303 | "schemes": [
304 | "https:\/\/buttondown.email\/*"
305 | ],
306 | "url": "https:\/\/buttondown.email\/embed",
307 | "formats": [
308 | "json"
309 | ],
310 | "discovery": true
311 | }
312 | ]
313 | },
314 | {
315 | "provider_name": "Byzart Project",
316 | "provider_url": "https:\/\/cmc.byzart.eu",
317 | "endpoints": [
318 | {
319 | "schemes": [
320 | "https:\/\/cmc.byzart.eu\/files\/*"
321 | ],
322 | "url": "https:\/\/cmc.byzart.eu\/oembed\/",
323 | "discovery": false
324 | }
325 | ]
326 | },
327 | {
328 | "provider_name": "Cacoo",
329 | "provider_url": "https:\/\/cacoo.com",
330 | "endpoints": [
331 | {
332 | "schemes": [
333 | "https:\/\/cacoo.com\/diagrams\/*"
334 | ],
335 | "url": "http:\/\/cacoo.com\/oembed.{format}"
336 | }
337 | ]
338 | },
339 | {
340 | "provider_name": "Carbon Health",
341 | "provider_url": "https:\/\/carbonhealth.com",
342 | "endpoints": [
343 | {
344 | "schemes": [
345 | "https:\/\/carbonhealth.com\/practice\/*"
346 | ],
347 | "url": "http:\/\/carbonhealth.com\/oembed",
348 | "discovery": true
349 | }
350 | ]
351 | },
352 | {
353 | "provider_name": "CatBoat",
354 | "provider_url": "http:\/\/img.catbo.at\/",
355 | "endpoints": [
356 | {
357 | "schemes": [
358 | "http:\/\/img.catbo.at\/*"
359 | ],
360 | "url": "http:\/\/img.catbo.at\/oembed.json",
361 | "formats": [
362 | "json"
363 | ]
364 | }
365 | ]
366 | },
367 | {
368 | "provider_name": "Ceros",
369 | "provider_url": "http:\/\/www.ceros.com\/",
370 | "endpoints": [
371 | {
372 | "schemes": [
373 | "http:\/\/view.ceros.com\/*"
374 | ],
375 | "url": "http:\/\/view.ceros.com\/oembed",
376 | "discovery": true
377 | }
378 | ]
379 | },
380 | {
381 | "provider_name": "ChartBlocks",
382 | "provider_url": "http:\/\/www.chartblocks.com\/",
383 | "endpoints": [
384 | {
385 | "schemes": [
386 | "http:\/\/public.chartblocks.com\/c\/*"
387 | ],
388 | "url": "http:\/\/embed.chartblocks.com\/1.0\/oembed"
389 | }
390 | ]
391 | },
392 | {
393 | "provider_name": "chirbit.com",
394 | "provider_url": "http:\/\/www.chirbit.com\/",
395 | "endpoints": [
396 | {
397 | "schemes": [
398 | "http:\/\/chirb.it\/*"
399 | ],
400 | "url": "http:\/\/chirb.it\/oembed.{format}",
401 | "discovery": true
402 | }
403 | ]
404 | },
405 | {
406 | "provider_name": "CircuitLab",
407 | "provider_url": "https:\/\/www.circuitlab.com\/",
408 | "endpoints": [
409 | {
410 | "schemes": [
411 | "https:\/\/www.circuitlab.com\/circuit\/*"
412 | ],
413 | "url": "https:\/\/www.circuitlab.com\/circuit\/oembed\/",
414 | "discovery": true
415 | }
416 | ]
417 | },
418 | {
419 | "provider_name": "Clipland",
420 | "provider_url": "http:\/\/www.clipland.com\/",
421 | "endpoints": [
422 | {
423 | "schemes": [
424 | "http:\/\/www.clipland.com\/v\/*",
425 | "https:\/\/www.clipland.com\/v\/*"
426 | ],
427 | "url": "https:\/\/www.clipland.com\/api\/oembed",
428 | "discovery": true
429 | }
430 | ]
431 | },
432 | {
433 | "provider_name": "Clyp",
434 | "provider_url": "http:\/\/clyp.it\/",
435 | "endpoints": [
436 | {
437 | "schemes": [
438 | "http:\/\/clyp.it\/*",
439 | "http:\/\/clyp.it\/playlist\/*"
440 | ],
441 | "url": "http:\/\/api.clyp.it\/oembed\/",
442 | "discovery": true
443 | }
444 | ]
445 | },
446 | {
447 | "provider_name": "CodeHS",
448 | "provider_url": "http:\/\/www.codehs.com",
449 | "endpoints": [
450 | {
451 | "schemes": [
452 | "https:\/\/codehs.com\/editor\/share_abacus\/*"
453 | ],
454 | "url": "https:\/\/codehs.com\/api\/sharedprogram\/*\/oembed\/",
455 | "discovery": true
456 | }
457 | ]
458 | },
459 | {
460 | "provider_name": "Codepen",
461 | "provider_url": "https:\/\/codepen.io",
462 | "endpoints": [
463 | {
464 | "schemes": [
465 | "http:\/\/codepen.io\/*",
466 | "https:\/\/codepen.io\/*"
467 | ],
468 | "url": "http:\/\/codepen.io\/api\/oembed"
469 | }
470 | ]
471 | },
472 | {
473 | "provider_name": "Codepoints",
474 | "provider_url": "https:\/\/codepoints.net",
475 | "endpoints": [
476 | {
477 | "schemes": [
478 | "http:\/\/codepoints.net\/*",
479 | "https:\/\/codepoints.net\/*",
480 | "http:\/\/www.codepoints.net\/*",
481 | "https:\/\/www.codepoints.net\/*"
482 | ],
483 | "url": "https:\/\/codepoints.net\/api\/v1\/oembed",
484 | "discovery": true
485 | }
486 | ]
487 | },
488 | {
489 | "provider_name": "CodeSandbox",
490 | "provider_url": "https:\/\/codesandbox.io",
491 | "endpoints": [
492 | {
493 | "schemes": [
494 | "https:\/\/codesandbox.io\/s\/*",
495 | "https:\/\/codesandbox.io\/embed\/*"
496 | ],
497 | "url": "https:\/\/codesandbox.io\/oembed"
498 | }
499 | ]
500 | },
501 | {
502 | "provider_name": "CollegeHumor",
503 | "provider_url": "http:\/\/www.collegehumor.com\/",
504 | "endpoints": [
505 | {
506 | "schemes": [
507 | "http:\/\/www.collegehumor.com\/video\/*"
508 | ],
509 | "url": "http:\/\/www.collegehumor.com\/oembed.{format}",
510 | "discovery": true
511 | }
512 | ]
513 | },
514 | {
515 | "provider_name": "Commaful",
516 | "provider_url": "https:\/\/commaful.com",
517 | "endpoints": [
518 | {
519 | "schemes": [
520 | "https:\/\/commaful.com\/play\/*"
521 | ],
522 | "url": "https:\/\/commaful.com\/api\/oembed\/"
523 | }
524 | ]
525 | },
526 | {
527 | "provider_name": "Coub",
528 | "provider_url": "http:\/\/coub.com\/",
529 | "endpoints": [
530 | {
531 | "schemes": [
532 | "http:\/\/coub.com\/view\/*",
533 | "http:\/\/coub.com\/embed\/*"
534 | ],
535 | "url": "http:\/\/coub.com\/api\/oembed.{format}"
536 | }
537 | ]
538 | },
539 | {
540 | "provider_name": "Crowd Ranking",
541 | "provider_url": "http:\/\/crowdranking.com",
542 | "endpoints": [
543 | {
544 | "schemes": [
545 | "http:\/\/crowdranking.com\/*\/*"
546 | ],
547 | "url": "http:\/\/crowdranking.com\/api\/oembed.{format}"
548 | }
549 | ]
550 | },
551 | {
552 | "provider_name": "Cyrano Systems",
553 | "provider_url": "http:\/\/www.cyranosystems.com\/",
554 | "endpoints": [
555 | {
556 | "schemes": [
557 | "https:\/\/staging.cyranosystems.com\/msg\/*",
558 | "https:\/\/app.cyranosystems.com\/msg\/*"
559 | ],
560 | "url": "https:\/\/staging.cyranosystems.com\/oembed",
561 | "formats": [
562 | "json"
563 | ],
564 | "discovery": true
565 | }
566 | ]
567 | },
568 | {
569 | "provider_name": "Daily Mile",
570 | "provider_url": "http:\/\/www.dailymile.com",
571 | "endpoints": [
572 | {
573 | "schemes": [
574 | "http:\/\/www.dailymile.com\/people\/*\/entries\/*"
575 | ],
576 | "url": "http:\/\/api.dailymile.com\/oembed?format=json",
577 | "formats": [
578 | "json"
579 | ]
580 | }
581 | ]
582 | },
583 | {
584 | "provider_name": "Dailymotion",
585 | "provider_url": "https:\/\/www.dailymotion.com",
586 | "endpoints": [
587 | {
588 | "schemes": [
589 | "https:\/\/www.dailymotion.com\/video\/*"
590 | ],
591 | "url": "https:\/\/www.dailymotion.com\/services\/oembed",
592 | "discovery": true
593 | }
594 | ]
595 | },
596 | {
597 | "provider_name": "Deseret News",
598 | "provider_url": "https:\/\/www.deseret.com",
599 | "endpoints": [
600 | {
601 | "schemes": [
602 | "https:\/\/*.deseret.com\/*"
603 | ],
604 | "url": "https:\/\/embed.deseret.com\/"
605 | }
606 | ]
607 | },
608 | {
609 | "provider_name": "Deviantart.com",
610 | "provider_url": "http:\/\/www.deviantart.com",
611 | "endpoints": [
612 | {
613 | "schemes": [
614 | "http:\/\/*.deviantart.com\/art\/*",
615 | "http:\/\/*.deviantart.com\/*#\/d*",
616 | "http:\/\/fav.me\/*",
617 | "http:\/\/sta.sh\/*",
618 | "https:\/\/*.deviantart.com\/art\/*",
619 | "https:\/\/*.deviantart.com\/*\/art\/*",
620 | "https:\/\/sta.sh\/*\",",
621 | "https:\/\/*.deviantart.com\/*#\/d*\""
622 | ],
623 | "url": "http:\/\/backend.deviantart.com\/oembed"
624 | }
625 | ]
626 | },
627 | {
628 | "provider_name": "Didacte",
629 | "provider_url": "https:\/\/www.didacte.com\/",
630 | "endpoints": [
631 | {
632 | "schemes": [
633 | "https:\/\/*.didacte.com\/a\/course\/*"
634 | ],
635 | "url": "https:\/\/*.didacte.com\/cards\/oembed'",
636 | "discovery": true,
637 | "formats": [
638 | "json"
639 | ]
640 | }
641 | ]
642 | },
643 | {
644 | "provider_name": "Digiteka",
645 | "provider_url": "https:\/\/www.ultimedia.com\/",
646 | "endpoints": [
647 | {
648 | "schemes": [
649 | "https:\/\/www.ultimedia.com\/central\/video\/edit\/id\/*\/topic_id\/*\/",
650 | "https:\/\/www.ultimedia.com\/default\/index\/videogeneric\/id\/*\/showtitle\/1\/viewnc\/1",
651 | "https:\/\/www.ultimedia.com\/default\/index\/videogeneric\/id\/*"
652 | ],
653 | "url": "https:\/\/www.ultimedia.com\/api\/search\/oembed",
654 | "discovery": true
655 | }
656 | ]
657 | },
658 | {
659 | "provider_name": "Dipity",
660 | "provider_url": "http:\/\/www.dipity.com",
661 | "endpoints": [
662 | {
663 | "schemes": [
664 | "http:\/\/www.dipity.com\/*\/*\/"
665 | ],
666 | "url": "http:\/\/www.dipity.com\/oembed\/timeline\/"
667 | }
668 | ]
669 | },
670 | {
671 | "provider_name": "DocDroid",
672 | "provider_url": "https:\/\/www.docdroid.net\/",
673 | "endpoints": [
674 | {
675 | "schemes": [
676 | "https:\/\/*.docdroid.net\/*",
677 | "http:\/\/*.docdroid.net\/*",
678 | "https:\/\/docdro.id\/*",
679 | "http:\/\/docdro.id\/*"
680 | ],
681 | "url": "https:\/\/www.docdroid.net\/api\/oembed",
682 | "formats": [
683 | "json"
684 | ],
685 | "discovery": true
686 | }
687 | ]
688 | },
689 | {
690 | "provider_name": "Dotsub",
691 | "provider_url": "http:\/\/dotsub.com\/",
692 | "endpoints": [
693 | {
694 | "schemes": [
695 | "http:\/\/dotsub.com\/view\/*"
696 | ],
697 | "url": "http:\/\/dotsub.com\/services\/oembed"
698 | }
699 | ]
700 | },
701 | {
702 | "provider_name": "DTube",
703 | "provider_url": "https:\/\/d.tube\/",
704 | "endpoints": [
705 | {
706 | "schemes": [
707 | "https:\/\/d.tube\/v\/*"
708 | ],
709 | "url": "https:\/\/api.d.tube\/oembed",
710 | "discovery": true
711 | }
712 | ]
713 | },
714 | {
715 | "provider_name": "edocr",
716 | "provider_url": "http:\/\/www.edocr.com",
717 | "endpoints": [
718 | {
719 | "schemes": [
720 | "http:\/\/edocr.com\/docs\/*"
721 | ],
722 | "url": "http:\/\/edocr.com\/api\/oembed"
723 | }
724 | ]
725 | },
726 | {
727 | "provider_name": "eduMedia",
728 | "provider_url": "https:\/\/www.edumedia-sciences.com\/",
729 | "endpoints": [
730 | {
731 | "url": "https:\/\/www.edumedia-sciences.com\/oembed.json",
732 | "discovery": true
733 | },
734 | {
735 | "url": "https:\/\/www.edumedia-sciences.com\/oembed.xml",
736 | "discovery": true
737 | }
738 | ]
739 | },
740 | {
741 | "provider_name": "EgliseInfo",
742 | "provider_url": "http:\/\/egliseinfo.catholique.fr\/",
743 | "endpoints": [
744 | {
745 | "schemes": [
746 | "http:\/\/egliseinfo.catholique.fr\/*"
747 | ],
748 | "url": "http:\/\/egliseinfo.catholique.fr\/api\/oembed",
749 | "discovery": true
750 | }
751 | ]
752 | },
753 | {
754 | "provider_name": "Embed Articles",
755 | "provider_url": "http:\/\/embedarticles.com\/",
756 | "endpoints": [
757 | {
758 | "schemes": [
759 | "http:\/\/embedarticles.com\/*"
760 | ],
761 | "url": "http:\/\/embedarticles.com\/oembed\/"
762 | }
763 | ]
764 | },
765 | {
766 | "provider_name": "Embedly",
767 | "provider_url": "http:\/\/api.embed.ly\/",
768 | "endpoints": [
769 | {
770 | "url": "http:\/\/api.embed.ly\/1\/oembed"
771 | }
772 | ]
773 | },
774 | {
775 | "provider_name": "Ethfiddle",
776 | "provider_url": "https:\/\/www.ethfiddle.com\/",
777 | "endpoints": [
778 | {
779 | "schemes": [
780 | "https:\/\/ethfiddle.com\/*"
781 | ],
782 | "url": "https:\/\/ethfiddle.com\/services\/oembed\/",
783 | "discovery": true
784 | }
785 | ]
786 | },
787 | {
788 | "provider_name": "Eyrie",
789 | "provider_url": "https:\/\/eyrie.io\/",
790 | "endpoints": [
791 | {
792 | "schemes": [
793 | "https:\/\/eyrie.io\/board\/*",
794 | "https:\/\/eyrie.io\/sparkfun\/*"
795 | ],
796 | "url": "https:\/\/eyrie.io\/v1\/oembed",
797 | "discovery": true
798 | }
799 | ]
800 | },
801 | {
802 | "provider_name": "Facebook",
803 | "provider_url": "https:\/\/www.facebook.com\/",
804 | "endpoints": [
805 | {
806 | "schemes": [
807 | "https:\/\/www.facebook.com\/*\/posts\/*",
808 | "https:\/\/www.facebook.com\/photos\/*",
809 | "https:\/\/www.facebook.com\/*\/photos\/*",
810 | "https:\/\/www.facebook.com\/photo.php*",
811 | "https:\/\/www.facebook.com\/photo.php",
812 | "https:\/\/www.facebook.com\/*\/activity\/*",
813 | "https:\/\/www.facebook.com\/permalink.php",
814 | "https:\/\/www.facebook.com\/media\/set?set=*",
815 | "https:\/\/www.facebook.com\/questions\/*",
816 | "https:\/\/www.facebook.com\/notes\/*\/*\/*"
817 | ],
818 | "url": "https:\/\/www.facebook.com\/plugins\/post\/oembed.json",
819 | "discovery": true
820 | },
821 | {
822 | "schemes": [
823 | "https:\/\/www.facebook.com\/*\/videos\/*",
824 | "https:\/\/www.facebook.com\/video.php"
825 | ],
826 | "url": "https:\/\/www.facebook.com\/plugins\/video\/oembed.json",
827 | "discovery": true
828 | }
829 | ]
830 | },
831 | {
832 | "provider_name": "Fader",
833 | "provider_url": "https:\/\/app.getfader.com",
834 | "endpoints": [
835 | {
836 | "schemes": [
837 | "https:\/\/app.getfader.com\/projects\/*\/publish"
838 | ],
839 | "url": "https:\/\/app.getfader.com\/api\/oembed",
840 | "formats": [
841 | "json"
842 | ]
843 | }
844 | ]
845 | },
846 | {
847 | "provider_name": "Faithlife TV",
848 | "provider_url": "https:\/\/faithlifetv.com",
849 | "endpoints": [
850 | {
851 | "schemes": [
852 | "https:\/\/faithlifetv.com\/items\/*",
853 | "https:\/\/faithlifetv.com\/items\/resource\/*\/*",
854 | "https:\/\/faithlifetv.com\/media\/*",
855 | "https:\/\/faithlifetv.com\/media\/assets\/*",
856 | "https:\/\/faithlifetv.com\/media\/resource\/*\/*"
857 | ],
858 | "url": "https:\/\/faithlifetv.com\/api\/oembed",
859 | "discovery": true
860 | }
861 | ]
862 | },
863 | {
864 | "provider_name": "Firework",
865 | "provider_url": "https:\/\/fireworktv.com\/",
866 | "endpoints": [
867 | {
868 | "schemes": [
869 | "https:\/\/*.fireworktv.com\/*",
870 | "https:\/\/*.fireworktv.com\/embed\/*\/v\/*"
871 | ],
872 | "url": "https:\/\/www.fireworktv.com\/oembed",
873 | "discovery": true
874 | }
875 | ]
876 | },
877 | {
878 | "provider_name": "FITE",
879 | "provider_url": "https:\/\/www.fite.tv\/",
880 | "endpoints": [
881 | {
882 | "schemes": [
883 | "https:\/\/www.fite.tv\/watch\/*"
884 | ],
885 | "url": "https:\/\/www.fite.tv\/oembed",
886 | "discovery": true
887 | }
888 | ]
889 | },
890 | {
891 | "provider_name": "Flat",
892 | "provider_url": "https:\/\/flat.io",
893 | "endpoints": [
894 | {
895 | "schemes": [
896 | "https:\/\/flat.io\/score\/*",
897 | "https:\/\/*.flat.io\/score\/*"
898 | ],
899 | "url": "https:\/\/flat.io\/services\/oembed",
900 | "discovery": true
901 | }
902 | ]
903 | },
904 | {
905 | "provider_name": "Flickr",
906 | "provider_url": "https:\/\/www.flickr.com\/",
907 | "endpoints": [
908 | {
909 | "schemes": [
910 | "http:\/\/*.flickr.com\/photos\/*",
911 | "http:\/\/flic.kr\/p\/*",
912 | "https:\/\/*.flickr.com\/photos\/*",
913 | "https:\/\/flic.kr\/p\/*"
914 | ],
915 | "url": "https:\/\/www.flickr.com\/services\/oembed\/",
916 | "discovery": true
917 | }
918 | ]
919 | },
920 | {
921 | "provider_name": "Flourish",
922 | "provider_url": "https:\/\/flourish.studio\/",
923 | "endpoints": [
924 | {
925 | "schemes": [
926 | "https:\/\/public.flourish.studio\/visualisation\/*",
927 | "https:\/\/public.flourish.studio\/story\/*"
928 | ],
929 | "url": "https:\/\/app.flourish.studio\/api\/v1\/oembed",
930 | "discovery": true
931 | }
932 | ]
933 | },
934 | {
935 | "provider_name": "Fontself",
936 | "provider_url": "https:\/\/www.fontself.com",
937 | "endpoints": [
938 | {
939 | "schemes": [
940 | "https:\/\/catapult.fontself.com\/*"
941 | ],
942 | "url": "https:\/\/oembed.fontself.com\/"
943 | }
944 | ]
945 | },
946 | {
947 | "provider_name": "FOX SPORTS Australia",
948 | "provider_url": "http:\/\/www.foxsports.com.au",
949 | "endpoints": [
950 | {
951 | "schemes": [
952 | "http:\/\/fiso.foxsports.com.au\/isomorphic-widget\/*",
953 | "https:\/\/fiso.foxsports.com.au\/isomorphic-widget\/*"
954 | ],
955 | "url": "https:\/\/fiso.foxsports.com.au\/oembed"
956 | }
957 | ]
958 | },
959 | {
960 | "provider_name": "FrameBuzz",
961 | "provider_url": "https:\/\/framebuzz.com\/",
962 | "endpoints": [
963 | {
964 | "schemes": [
965 | "http:\/\/framebuzz.com\/v\/*",
966 | "https:\/\/framebuzz.com\/v\/*"
967 | ],
968 | "url": "https:\/\/framebuzz.com\/oembed\/",
969 | "discovery": true
970 | }
971 | ]
972 | },
973 | {
974 | "provider_name": "FunnyOrDie",
975 | "provider_url": "http:\/\/www.funnyordie.com\/",
976 | "endpoints": [
977 | {
978 | "schemes": [
979 | "http:\/\/www.funnyordie.com\/videos\/*"
980 | ],
981 | "url": "http:\/\/www.funnyordie.com\/oembed.{format}"
982 | }
983 | ]
984 | },
985 | {
986 | "provider_name": "Geograph Britain and Ireland",
987 | "provider_url": "https:\/\/www.geograph.org.uk\/",
988 | "endpoints": [
989 | {
990 | "schemes": [
991 | "http:\/\/*.geograph.org.uk\/*",
992 | "http:\/\/*.geograph.co.uk\/*",
993 | "http:\/\/*.geograph.ie\/*",
994 | "http:\/\/*.wikimedia.org\/*_geograph.org.uk_*"
995 | ],
996 | "url": "http:\/\/api.geograph.org.uk\/api\/oembed"
997 | }
998 | ]
999 | },
1000 | {
1001 | "provider_name": "Geograph Channel Islands",
1002 | "provider_url": "http:\/\/channel-islands.geograph.org\/",
1003 | "endpoints": [
1004 | {
1005 | "schemes": [
1006 | "http:\/\/*.geograph.org.gg\/*",
1007 | "http:\/\/*.geograph.org.je\/*",
1008 | "http:\/\/channel-islands.geograph.org\/*",
1009 | "http:\/\/channel-islands.geographs.org\/*",
1010 | "http:\/\/*.channel.geographs.org\/*"
1011 | ],
1012 | "url": "http:\/\/www.geograph.org.gg\/api\/oembed"
1013 | }
1014 | ]
1015 | },
1016 | {
1017 | "provider_name": "Geograph Germany",
1018 | "provider_url": "http:\/\/geo-en.hlipp.de\/",
1019 | "endpoints": [
1020 | {
1021 | "schemes": [
1022 | "http:\/\/geo-en.hlipp.de\/*",
1023 | "http:\/\/geo.hlipp.de\/*",
1024 | "http:\/\/germany.geograph.org\/*"
1025 | ],
1026 | "url": "http:\/\/geo.hlipp.de\/restapi.php\/api\/oembed"
1027 | }
1028 | ]
1029 | },
1030 | {
1031 | "provider_name": "Getty Images",
1032 | "provider_url": "http:\/\/www.gettyimages.com\/",
1033 | "endpoints": [
1034 | {
1035 | "schemes": [
1036 | "http:\/\/gty.im\/*"
1037 | ],
1038 | "url": "http:\/\/embed.gettyimages.com\/oembed",
1039 | "formats": [
1040 | "json"
1041 | ]
1042 | }
1043 | ]
1044 | },
1045 | {
1046 | "provider_name": "Gfycat",
1047 | "provider_url": "https:\/\/gfycat.com\/",
1048 | "endpoints": [
1049 | {
1050 | "schemes": [
1051 | "http:\/\/gfycat.com\/*",
1052 | "http:\/\/www.gfycat.com\/*",
1053 | "https:\/\/gfycat.com\/*",
1054 | "https:\/\/www.gfycat.com\/*"
1055 | ],
1056 | "url": "https:\/\/api.gfycat.com\/v1\/oembed",
1057 | "discovery": true
1058 | }
1059 | ]
1060 | },
1061 | {
1062 | "provider_name": "Gifnote",
1063 | "provider_url": "https:\/\/www.gifnote.com\/",
1064 | "endpoints": [
1065 | {
1066 | "url": "https:\/\/www.gifnote.com\/services\/oembed",
1067 | "schemes": [
1068 | "https:\/\/www.gifnote.com\/play\/*"
1069 | ],
1070 | "discovery": true
1071 | }
1072 | ]
1073 | },
1074 | {
1075 | "provider_name": "GIPHY",
1076 | "provider_url": "https:\/\/giphy.com",
1077 | "endpoints": [
1078 | {
1079 | "schemes": [
1080 | "https:\/\/giphy.com\/gifs\/*",
1081 | "http:\/\/gph.is\/*",
1082 | "https:\/\/media.giphy.com\/media\/*\/giphy.gif"
1083 | ],
1084 | "url": "https:\/\/giphy.com\/services\/oembed",
1085 | "discovery": true
1086 | }
1087 | ]
1088 | },
1089 | {
1090 | "provider_name": "GloriaTV",
1091 | "provider_url": "https:\/\/gloria.tv\/",
1092 | "endpoints": [
1093 | {
1094 | "url": "https:\/\/gloria.tv\/oembed\/",
1095 | "discovery": true
1096 | }
1097 | ]
1098 | },
1099 | {
1100 | "provider_name": "GT Channel",
1101 | "provider_url": "https:\/\/gtchannel.com",
1102 | "endpoints": [
1103 | {
1104 | "schemes": [
1105 | "https:\/\/gtchannel.com\/watch\/*"
1106 | ],
1107 | "url": "https:\/\/api.luminery.com\/oembed",
1108 | "discovery": true
1109 | }
1110 | ]
1111 | },
1112 | {
1113 | "provider_name": "Gyazo",
1114 | "provider_url": "https:\/\/gyazo.com",
1115 | "endpoints": [
1116 | {
1117 | "schemes": [
1118 | "https:\/\/gyazo.com\/*"
1119 | ],
1120 | "url": "https:\/\/api.gyazo.com\/api\/oembed",
1121 | "formats": [
1122 | "json"
1123 | ]
1124 | }
1125 | ]
1126 | },
1127 | {
1128 | "provider_name": "hearthis.at",
1129 | "provider_url": "https:\/\/hearthis.at\/",
1130 | "endpoints": [
1131 | {
1132 | "schemes": [
1133 | "https:\/\/hearthis.at\/*\/*\/",
1134 | "https:\/\/hearthis.at\/*\/set\/*\/"
1135 | ],
1136 | "url": "https:\/\/hearthis.at\/oembed\/?format=json",
1137 | "discovery": true
1138 | }
1139 | ]
1140 | },
1141 | {
1142 | "provider_name": "Homey",
1143 | "provider_url": "https:\/\/homey.app",
1144 | "endpoints": [
1145 | {
1146 | "schemes": [
1147 | "https:\/\/homey.app\/f\/*",
1148 | "https:\/\/homey.app\/*\/flow\/*"
1149 | ],
1150 | "url": "https:\/\/homey.app\/api\/oembed\/flow",
1151 | "discovery": true
1152 | }
1153 | ]
1154 | },
1155 | {
1156 | "provider_name": "HuffDuffer",
1157 | "provider_url": "http:\/\/huffduffer.com",
1158 | "endpoints": [
1159 | {
1160 | "schemes": [
1161 | "http:\/\/huffduffer.com\/*\/*"
1162 | ],
1163 | "url": "http:\/\/huffduffer.com\/oembed"
1164 | }
1165 | ]
1166 | },
1167 | {
1168 | "provider_name": "Hulu",
1169 | "provider_url": "http:\/\/www.hulu.com\/",
1170 | "endpoints": [
1171 | {
1172 | "schemes": [
1173 | "http:\/\/www.hulu.com\/watch\/*"
1174 | ],
1175 | "url": "http:\/\/www.hulu.com\/api\/oembed.{format}"
1176 | }
1177 | ]
1178 | },
1179 | {
1180 | "provider_name": "iFixit",
1181 | "provider_url": "http:\/\/www.iFixit.com",
1182 | "endpoints": [
1183 | {
1184 | "schemes": [
1185 | "http:\/\/www.ifixit.com\/Guide\/View\/*"
1186 | ],
1187 | "url": "http:\/\/www.ifixit.com\/Embed"
1188 | }
1189 | ]
1190 | },
1191 | {
1192 | "provider_name": "IFTTT",
1193 | "provider_url": "http:\/\/www.ifttt.com\/",
1194 | "endpoints": [
1195 | {
1196 | "schemes": [
1197 | "http:\/\/ifttt.com\/recipes\/*"
1198 | ],
1199 | "url": "http:\/\/www.ifttt.com\/oembed\/",
1200 | "discovery": true
1201 | }
1202 | ]
1203 | },
1204 | {
1205 | "provider_name": "iHeartRadio",
1206 | "provider_url": "https:\/\/www.iheart.com",
1207 | "endpoints": [
1208 | {
1209 | "schemes": [
1210 | "https:\/\/www.iheart.com\/podcast\/*\/*"
1211 | ],
1212 | "url": "https:\/\/www.iheart.com\/oembed",
1213 | "discovery": true
1214 | }
1215 | ]
1216 | },
1217 | {
1218 | "provider_name": "Indaco",
1219 | "provider_url": "https:\/\/player.indacolive.com\/",
1220 | "endpoints": [
1221 | {
1222 | "schemes": [
1223 | "https:\/\/player.indacolive.com\/player\/jwp\/clients\/*"
1224 | ],
1225 | "url": "https:\/\/player.indacolive.com\/services\/oembed",
1226 | "formats": [
1227 | "json"
1228 | ]
1229 | }
1230 | ]
1231 | },
1232 | {
1233 | "provider_name": "Infogram",
1234 | "provider_url": "https:\/\/infogram.com\/",
1235 | "endpoints": [
1236 | {
1237 | "schemes": [
1238 | "https:\/\/infogram.com\/*"
1239 | ],
1240 | "url": "https:\/\/infogram.com\/oembed"
1241 | }
1242 | ]
1243 | },
1244 | {
1245 | "provider_name": "Infoveave",
1246 | "provider_url": "https:\/\/infoveave.net\/",
1247 | "endpoints": [
1248 | {
1249 | "schemes": [
1250 | "https:\/\/*.infoveave.net\/E\/*",
1251 | "https:\/\/*.infoveave.net\/P\/*"
1252 | ],
1253 | "url": "https:\/\/infoveave.net\/services\/oembed\/",
1254 | "discovery": true
1255 | }
1256 | ]
1257 | },
1258 | {
1259 | "provider_name": "Injurymap",
1260 | "provider_url": "https:\/\/www.injurymap.com\/",
1261 | "endpoints": [
1262 | {
1263 | "schemes": [
1264 | "https:\/\/www.injurymap.com\/exercises\/*"
1265 | ],
1266 | "url": "https:\/\/www.injurymap.com\/services\/oembed",
1267 | "discovery": true
1268 | }
1269 | ]
1270 | },
1271 | {
1272 | "provider_name": "Inoreader",
1273 | "provider_url": "https:\/\/www.inoreader.com",
1274 | "endpoints": [
1275 | {
1276 | "schemes": [
1277 | "https:\/\/www.inoreader.com\/oembed\/"
1278 | ],
1279 | "url": "https:\/\/www.inoreader.com\/oembed\/api\/",
1280 | "discovery": true
1281 | }
1282 | ]
1283 | },
1284 | {
1285 | "provider_name": "inphood",
1286 | "provider_url": "http:\/\/inphood.com\/",
1287 | "endpoints": [
1288 | {
1289 | "schemes": [
1290 | "http:\/\/*.inphood.com\/*"
1291 | ],
1292 | "url": "http:\/\/api.inphood.com\/oembed",
1293 | "formats": [
1294 | "json"
1295 | ]
1296 | }
1297 | ]
1298 | },
1299 | {
1300 | "provider_name": "Instagram",
1301 | "provider_url": "https:\/\/instagram.com",
1302 | "endpoints": [
1303 | {
1304 | "schemes": [
1305 | "http:\/\/instagram.com\/*\/p\/*,",
1306 | "http:\/\/www.instagram.com\/*\/p\/*,",
1307 | "https:\/\/instagram.com\/*\/p\/*,",
1308 | "https:\/\/www.instagram.com\/*\/p\/*,",
1309 | "http:\/\/instagram.com\/p\/*",
1310 | "http:\/\/instagr.am\/p\/*",
1311 | "http:\/\/www.instagram.com\/p\/*",
1312 | "http:\/\/www.instagr.am\/p\/*",
1313 | "https:\/\/instagram.com\/p\/*",
1314 | "https:\/\/instagr.am\/p\/*",
1315 | "https:\/\/www.instagram.com\/p\/*",
1316 | "https:\/\/www.instagr.am\/p\/*",
1317 | "http:\/\/instagram.com\/tv\/*",
1318 | "http:\/\/instagr.am\/tv\/*",
1319 | "http:\/\/www.instagram.com\/tv\/*",
1320 | "http:\/\/www.instagr.am\/tv\/*",
1321 | "https:\/\/instagram.com\/tv\/*",
1322 | "https:\/\/instagr.am\/tv\/*",
1323 | "https:\/\/www.instagram.com\/tv\/*",
1324 | "https:\/\/www.instagr.am\/tv\/*"
1325 | ],
1326 | "url": "https:\/\/api.instagram.com\/oembed",
1327 | "formats": [
1328 | "json"
1329 | ]
1330 | }
1331 | ]
1332 | },
1333 | {
1334 | "provider_name": "iSnare Articles",
1335 | "provider_url": "https:\/\/www.isnare.com\/",
1336 | "endpoints": [
1337 | {
1338 | "schemes": [
1339 | "https:\/\/www.isnare.com\/*"
1340 | ],
1341 | "url": "https:\/\/www.isnare.com\/oembed\/"
1342 | }
1343 | ]
1344 | },
1345 | {
1346 | "provider_name": "Issuu",
1347 | "provider_url": "https:\/\/issuu.com\/",
1348 | "endpoints": [
1349 | {
1350 | "schemes": [
1351 | "https:\/\/issuu.com\/*\/docs\/*"
1352 | ],
1353 | "url": "https:\/\/issuu.com\/oembed",
1354 | "discovery": true
1355 | }
1356 | ]
1357 | },
1358 | {
1359 | "provider_name": "ivlismusic",
1360 | "provider_url": "https:\/\/music.ivlis.kr\/",
1361 | "endpoints": [
1362 | {
1363 | "url": "https:\/\/music.ivlis.kr\/oembed",
1364 | "discovery": true
1365 | }
1366 | ]
1367 | },
1368 | {
1369 | "provider_name": "Jovian",
1370 | "provider_url": "https:\/\/jovian.ml\/",
1371 | "endpoints": [
1372 | {
1373 | "schemes": [
1374 | "https:\/\/jovian.ml\/*",
1375 | "https:\/\/jovian.ml\/viewer*",
1376 | "https:\/\/*.jovian.ml\/*"
1377 | ],
1378 | "url": "https:\/\/api.jovian.ai\/oembed.json",
1379 | "discovery": true
1380 | }
1381 | ]
1382 | },
1383 | {
1384 | "provider_name": "KakaoTv",
1385 | "provider_url": "https:\/\/tv.kakao.com\/",
1386 | "endpoints": [
1387 | {
1388 | "schemes": [
1389 | "https:\/\/tv.kakao.com\/channel\/*\/cliplink\/*",
1390 | "https:\/\/tv.kakao.com\/channel\/v\/*",
1391 | "https:\/\/tv.kakao.com\/channel\/*\/livelink\/*",
1392 | "https:\/\/tv.kakao.com\/channel\/l\/*"
1393 | ],
1394 | "url": "https:\/\/tv.kakao.com\/oembed",
1395 | "discovery": true
1396 | }
1397 | ]
1398 | },
1399 | {
1400 | "provider_name": "Kickstarter",
1401 | "provider_url": "http:\/\/www.kickstarter.com",
1402 | "endpoints": [
1403 | {
1404 | "schemes": [
1405 | "http:\/\/www.kickstarter.com\/projects\/*"
1406 | ],
1407 | "url": "http:\/\/www.kickstarter.com\/services\/oembed"
1408 | }
1409 | ]
1410 | },
1411 | {
1412 | "provider_name": "Kidoju",
1413 | "provider_url": "https:\/\/www.kidoju.com\/",
1414 | "endpoints": [
1415 | {
1416 | "schemes": [
1417 | "https:\/\/www.kidoju.com\/en\/x\/*\/*",
1418 | "https:\/\/www.kidoju.com\/fr\/x\/*\/*"
1419 | ],
1420 | "url": "https:\/\/www.kidoju.com\/api\/oembed"
1421 | }
1422 | ]
1423 | },
1424 | {
1425 | "provider_name": "Kirim.Email",
1426 | "provider_url": "https:\/\/kirim.email\/",
1427 | "endpoints": [
1428 | {
1429 | "schemes": [
1430 | "https:\/\/halaman.email\/form\/*",
1431 | "https:\/\/aplikasi.kirim.email\/form\/*"
1432 | ],
1433 | "url": "https:\/\/halaman.email\/service\/oembed",
1434 | "discovery": true
1435 | }
1436 | ]
1437 | },
1438 | {
1439 | "provider_name": "Kit",
1440 | "provider_url": "https:\/\/kit.co\/",
1441 | "endpoints": [
1442 | {
1443 | "schemes": [
1444 | "http:\/\/kit.co\/*\/*",
1445 | "https:\/\/kit.co\/*\/*"
1446 | ],
1447 | "url": "https:\/\/embed.kit.co\/oembed",
1448 | "discovery": true
1449 | }
1450 | ]
1451 | },
1452 | {
1453 | "provider_name": "Kitchenbowl",
1454 | "provider_url": "http:\/\/www.kitchenbowl.com",
1455 | "endpoints": [
1456 | {
1457 | "schemes": [
1458 | "http:\/\/www.kitchenbowl.com\/recipe\/*"
1459 | ],
1460 | "url": "http:\/\/www.kitchenbowl.com\/oembed",
1461 | "discovery": true
1462 | }
1463 | ]
1464 | },
1465 | {
1466 | "provider_name": "Knacki",
1467 | "provider_url": "http:\/\/jdr.knacki.info",
1468 | "endpoints": [
1469 | {
1470 | "schemes": [
1471 | "http:\/\/jdr.knacki.info\/meuh\/*",
1472 | "https:\/\/jdr.knacki.info\/meuh\/*"
1473 | ],
1474 | "url": "https:\/\/jdr.knacki.info\/oembed"
1475 | }
1476 | ]
1477 | },
1478 | {
1479 | "provider_name": "LearningApps.org",
1480 | "provider_url": "http:\/\/learningapps.org\/",
1481 | "endpoints": [
1482 | {
1483 | "schemes": [
1484 | "http:\/\/learningapps.org\/*"
1485 | ],
1486 | "url": "http:\/\/learningapps.org\/oembed.php",
1487 | "discovery": true
1488 | }
1489 | ]
1490 | },
1491 | {
1492 | "provider_name": "Lille.Pod",
1493 | "provider_url": "https:\/\/pod.univ-lille.fr\/",
1494 | "endpoints": [
1495 | {
1496 | "schemes": [
1497 | "https:\/\/pod.univ-lille.fr\/video\/*"
1498 | ],
1499 | "url": "https:\/\/pod.univ-lille.fr\/oembed",
1500 | "discovery": true
1501 | }
1502 | ]
1503 | },
1504 | {
1505 | "provider_name": "Livestream",
1506 | "provider_url": "https:\/\/livestream.com\/",
1507 | "endpoints": [
1508 | {
1509 | "schemes": [
1510 | "https:\/\/livestream.com\/accounts\/*\/events\/*",
1511 | "https:\/\/livestream.com\/accounts\/*\/events\/*\/videos\/*",
1512 | "https:\/\/livestream.com\/*\/events\/*",
1513 | "https:\/\/livestream.com\/*\/events\/*\/videos\/*",
1514 | "https:\/\/livestream.com\/*\/*",
1515 | "https:\/\/livestream.com\/*\/*\/videos\/*"
1516 | ],
1517 | "url": "https:\/\/livestream.com\/oembed",
1518 | "discovery": true
1519 | }
1520 | ]
1521 | },
1522 | {
1523 | "provider_name": "Ludus",
1524 | "provider_url": "https:\/\/ludus.one",
1525 | "endpoints": [
1526 | {
1527 | "schemes": [
1528 | "https:\/\/app.ludus.one\/*"
1529 | ],
1530 | "url": "https:\/\/app.ludus.one\/oembed",
1531 | "discovery": true,
1532 | "formats": [
1533 | "json"
1534 | ]
1535 | }
1536 | ]
1537 | },
1538 | {
1539 | "provider_name": "MathEmbed",
1540 | "provider_url": "http:\/\/mathembed.com",
1541 | "endpoints": [
1542 | {
1543 | "schemes": [
1544 | "http:\/\/mathembed.com\/latex?inputText=*",
1545 | "http:\/\/mathembed.com\/latex?inputText=*"
1546 | ],
1547 | "url": "http:\/\/mathembed.com\/oembed"
1548 | }
1549 | ]
1550 | },
1551 | {
1552 | "provider_name": "Matterport",
1553 | "provider_url": "https:\/\/matterport.com\/",
1554 | "endpoints": [
1555 | {
1556 | "url": "https:\/\/my.matterport.com\/api\/v1\/models\/oembed\/",
1557 | "discovery": true,
1558 | "formats": [
1559 | "json"
1560 | ]
1561 | }
1562 | ]
1563 | },
1564 | {
1565 | "provider_name": "me.me",
1566 | "provider_url": "https:\/\/me.me\/",
1567 | "endpoints": [
1568 | {
1569 | "schemes": [
1570 | "https:\/\/me.me\/i\/*"
1571 | ],
1572 | "url": "https:\/\/me.me\/oembed",
1573 | "discovery": true
1574 | }
1575 | ]
1576 | },
1577 | {
1578 | "provider_name": "MediaLab",
1579 | "provider_url": "https:\/\/www.medialab.co\/",
1580 | "endpoints": [
1581 | {
1582 | "schemes": [
1583 | "https:\/\/*.medialab.app\/share\/watch\/*",
1584 | "https:\/\/*.medialab.co\/share\/watch\/*",
1585 | "https:\/\/*.medialab.app\/share\/social\/*",
1586 | "https:\/\/*.medialab.co\/share\/social\/*",
1587 | "https:\/\/*.medialab.app\/share\/embed\/*",
1588 | "https:\/\/*.medialab.co\/share\/embed\/*"
1589 | ],
1590 | "url": "https:\/\/*.medialab.(co|app)\/api\/oembed\/",
1591 | "discovery": true
1592 | }
1593 | ]
1594 | },
1595 | {
1596 | "provider_name": "Medienarchiv der K\u00fcnste - Z\u00fcrcher Hochschule der K\u00fcnste",
1597 | "provider_url": "https:\/\/medienarchiv.zhdk.ch\/",
1598 | "endpoints": [
1599 | {
1600 | "schemes": [
1601 | "https:\/\/medienarchiv.zhdk.ch\/entries\/*"
1602 | ],
1603 | "url": "https:\/\/medienarchiv.zhdk.ch\/oembed.{format}",
1604 | "discovery": true
1605 | }
1606 | ]
1607 | },
1608 | {
1609 | "provider_name": "Meetup",
1610 | "provider_url": "http:\/\/www.meetup.com",
1611 | "endpoints": [
1612 | {
1613 | "schemes": [
1614 | "http:\/\/meetup.com\/*",
1615 | "https:\/\/www.meetup.com\/*",
1616 | "https:\/\/meetup.com\/*",
1617 | "http:\/\/meetu.ps\/*"
1618 | ],
1619 | "url": "https:\/\/api.meetup.com\/oembed",
1620 | "formats": [
1621 | "json"
1622 | ]
1623 | }
1624 | ]
1625 | },
1626 | {
1627 | "provider_name": "Mermaid Ink",
1628 | "provider_url": "https:\/\/mermaid.ink",
1629 | "endpoints": [
1630 | {
1631 | "schemes": [
1632 | "https:\/\/mermaid.ink\/img\/*",
1633 | "https:\/\/mermaid.ink\/svg\/*"
1634 | ],
1635 | "url": "https:\/\/mermaid.ink\/services\/oembed",
1636 | "discovery": true
1637 | }
1638 | ]
1639 | },
1640 | {
1641 | "provider_name": "Microlink",
1642 | "provider_url": "http:\/\/api.microlink.io",
1643 | "endpoints": [
1644 | {
1645 | "url": "https:\/\/api.microlink.io"
1646 | }
1647 | ]
1648 | },
1649 | {
1650 | "provider_name": "MixCloud",
1651 | "provider_url": "https:\/\/mixcloud.com\/",
1652 | "endpoints": [
1653 | {
1654 | "schemes": [
1655 | "http:\/\/www.mixcloud.com\/*\/*\/",
1656 | "https:\/\/www.mixcloud.com\/*\/*\/"
1657 | ],
1658 | "url": "https:\/\/www.mixcloud.com\/oembed\/"
1659 | }
1660 | ]
1661 | },
1662 | {
1663 | "provider_name": "Moby Picture",
1664 | "provider_url": "http:\/\/www.mobypicture.com",
1665 | "endpoints": [
1666 | {
1667 | "schemes": [
1668 | "http:\/\/www.mobypicture.com\/user\/*\/view\/*",
1669 | "http:\/\/moby.to\/*"
1670 | ],
1671 | "url": "http:\/\/api.mobypicture.com\/oEmbed"
1672 | }
1673 | ]
1674 | },
1675 | {
1676 | "provider_name": "Modelo",
1677 | "provider_url": "http:\/\/modelo.io\/",
1678 | "endpoints": [
1679 | {
1680 | "schemes": [
1681 | "https:\/\/beta.modelo.io\/embedded\/*"
1682 | ],
1683 | "url": "https:\/\/portal.modelo.io\/oembed",
1684 | "discovery": false
1685 | }
1686 | ]
1687 | },
1688 | {
1689 | "provider_name": "MorphCast",
1690 | "provider_url": "https:\/\/www.morphcast.com",
1691 | "endpoints": [
1692 | {
1693 | "schemes": [
1694 | "https:\/\/m-roll.morphcast.com\/mroll\/*"
1695 | ],
1696 | "url": "https:\/\/m-roll.morphcast.com\/service\/oembed",
1697 | "discovery": true,
1698 | "formats": [
1699 | "json"
1700 | ]
1701 | }
1702 | ]
1703 | },
1704 | {
1705 | "provider_name": "Music Box Maniacs",
1706 | "provider_url": "https:\/\/musicboxmaniacs.com\/",
1707 | "endpoints": [
1708 | {
1709 | "schemes": [
1710 | "https:\/\/musicboxmaniacs.com\/explore\/melody\/*"
1711 | ],
1712 | "url": "https:\/\/musicboxmaniacs.com\/embed\/",
1713 | "formats": [
1714 | "json"
1715 | ],
1716 | "discovery": true
1717 | }
1718 | ]
1719 | },
1720 | {
1721 | "provider_name": "myBeweeg",
1722 | "provider_url": "https:\/\/mybeweeg.com",
1723 | "endpoints": [
1724 | {
1725 | "schemes": [
1726 | "https:\/\/mybeweeg.com\/w\/*"
1727 | ],
1728 | "url": "https:\/\/mybeweeg.com\/services\/oembed"
1729 | }
1730 | ]
1731 | },
1732 | {
1733 | "provider_name": "Namchey",
1734 | "provider_url": "https:\/\/namchey.com",
1735 | "endpoints": [
1736 | {
1737 | "schemes": [
1738 | "https:\/\/namchey.com\/embeds\/*"
1739 | ],
1740 | "url": "https:\/\/namchey.com\/api\/oembed",
1741 | "formats": [
1742 | "json",
1743 | "xml"
1744 | ],
1745 | "discovery": true
1746 | }
1747 | ]
1748 | },
1749 | {
1750 | "provider_name": "nanoo.tv",
1751 | "provider_url": "https:\/\/www.nanoo.tv\/",
1752 | "endpoints": [
1753 | {
1754 | "schemes": [
1755 | "http:\/\/*.nanoo.tv\/link\/*",
1756 | "http:\/\/nanoo.tv\/link\/*",
1757 | "http:\/\/*.nanoo.pro\/link\/*",
1758 | "http:\/\/nanoo.pro\/link\/*",
1759 | "https:\/\/*.nanoo.tv\/link\/*",
1760 | "https:\/\/nanoo.tv\/link\/*",
1761 | "https:\/\/*.nanoo.pro\/link\/*",
1762 | "https:\/\/nanoo.pro\/link\/*",
1763 | "http:\/\/media.zhdk.ch\/signatur\/*",
1764 | "http:\/\/new.media.zhdk.ch\/signatur\/*",
1765 | "https:\/\/media.zhdk.ch\/signatur\/*",
1766 | "https:\/\/new.media.zhdk.ch\/signatur\/*"
1767 | ],
1768 | "url": "https:\/\/www.nanoo.tv\/services\/oembed",
1769 | "discovery": true
1770 | }
1771 | ]
1772 | },
1773 | {
1774 | "provider_name": "Nasjonalbiblioteket",
1775 | "provider_url": "https:\/\/www.nb.no\/",
1776 | "endpoints": [
1777 | {
1778 | "schemes": [
1779 | "https:\/\/www.nb.no\/items\/*"
1780 | ],
1781 | "url": "https:\/\/api.nb.no\/catalog\/v1\/oembed",
1782 | "discovery": true
1783 | }
1784 | ]
1785 | },
1786 | {
1787 | "provider_name": "Natural Atlas",
1788 | "provider_url": "https:\/\/naturalatlas.com\/",
1789 | "endpoints": [
1790 | {
1791 | "schemes": [
1792 | "https:\/\/naturalatlas.com\/*",
1793 | "https:\/\/naturalatlas.com\/*\/*",
1794 | "https:\/\/naturalatlas.com\/*\/*\/*",
1795 | "https:\/\/naturalatlas.com\/*\/*\/*\/*"
1796 | ],
1797 | "url": "https:\/\/naturalatlas.com\/oembed.{format}",
1798 | "discovery": true,
1799 | "formats": [
1800 | "json"
1801 | ]
1802 | }
1803 | ]
1804 | },
1805 | {
1806 | "provider_name": "nfb.ca",
1807 | "provider_url": "http:\/\/www.nfb.ca\/",
1808 | "endpoints": [
1809 | {
1810 | "schemes": [
1811 | "http:\/\/*.nfb.ca\/film\/*"
1812 | ],
1813 | "url": "http:\/\/www.nfb.ca\/remote\/services\/oembed\/",
1814 | "discovery": true
1815 | }
1816 | ]
1817 | },
1818 | {
1819 | "provider_name": "Odds.com.au",
1820 | "provider_url": "https:\/\/www.odds.com.au",
1821 | "endpoints": [
1822 | {
1823 | "schemes": [
1824 | "https:\/\/www.odds.com.au\/*",
1825 | "https:\/\/odds.com.au\/*"
1826 | ],
1827 | "url": "https:\/\/www.odds.com.au\/api\/oembed\/"
1828 | }
1829 | ]
1830 | },
1831 | {
1832 | "provider_name": "Official FM",
1833 | "provider_url": "http:\/\/official.fm",
1834 | "endpoints": [
1835 | {
1836 | "schemes": [
1837 | "http:\/\/official.fm\/tracks\/*",
1838 | "http:\/\/official.fm\/playlists\/*"
1839 | ],
1840 | "url": "http:\/\/official.fm\/services\/oembed.{format}"
1841 | }
1842 | ]
1843 | },
1844 | {
1845 | "provider_name": "Omniscope",
1846 | "provider_url": "https:\/\/omniscope.me\/",
1847 | "endpoints": [
1848 | {
1849 | "schemes": [
1850 | "https:\/\/omniscope.me\/*"
1851 | ],
1852 | "url": "https:\/\/omniscope.me\/_global_\/oembed\/json",
1853 | "formats": [
1854 | "json"
1855 | ]
1856 | }
1857 | ]
1858 | },
1859 | {
1860 | "provider_name": "On Aol",
1861 | "provider_url": "http:\/\/on.aol.com\/",
1862 | "endpoints": [
1863 | {
1864 | "schemes": [
1865 | "http:\/\/on.aol.com\/video\/*"
1866 | ],
1867 | "url": "http:\/\/on.aol.com\/api"
1868 | }
1869 | ]
1870 | },
1871 | {
1872 | "provider_name": "Ora TV",
1873 | "provider_url": "http:\/\/www.ora.tv\/",
1874 | "endpoints": [
1875 | {
1876 | "discovery": true,
1877 | "url": "https:\/\/www.ora.tv\/oembed\/*?format={format}"
1878 | }
1879 | ]
1880 | },
1881 | {
1882 | "provider_name": "Orbitvu",
1883 | "provider_url": "https:\/\/orbitvu.co",
1884 | "endpoints": [
1885 | {
1886 | "schemes": [
1887 | "https:\/\/orbitvu.co\/001\/*\/ov3601\/view",
1888 | "https:\/\/orbitvu.co\/001\/*\/ov3601\/*\/view",
1889 | "https:\/\/orbitvu.co\/001\/*\/ov3602\/*\/view",
1890 | "https:\/\/orbitvu.co\/001\/*\/2\/orbittour\/*\/view",
1891 | "https:\/\/orbitvu.co\/001\/*\/1\/2\/orbittour\/*\/view",
1892 | "http:\/\/orbitvu.co\/001\/*\/ov3601\/view",
1893 | "http:\/\/orbitvu.co\/001\/*\/ov3601\/*\/view",
1894 | "http:\/\/orbitvu.co\/001\/*\/ov3602\/*\/view",
1895 | "http:\/\/orbitvu.co\/001\/*\/2\/orbittour\/*\/view",
1896 | "http:\/\/orbitvu.co\/001\/*\/1\/2\/orbittour\/*\/view"
1897 | ],
1898 | "url": "http:\/\/orbitvu.co\/service\/oembed",
1899 | "discovery": true
1900 | }
1901 | ]
1902 | },
1903 | {
1904 | "provider_name": "Oumy",
1905 | "provider_url": "https:\/\/www.oumy.com\/",
1906 | "endpoints": [
1907 | {
1908 | "schemes": [
1909 | "https:\/\/www.oumy.com\/v\/*"
1910 | ],
1911 | "url": "https:\/\/www.oumy.com\/oembed",
1912 | "discovery": true
1913 | }
1914 | ]
1915 | },
1916 | {
1917 | "provider_name": "Outplayed.tv",
1918 | "provider_url": "https:\/\/outplayed.tv\/",
1919 | "endpoints": [
1920 | {
1921 | "schemes": [
1922 | "https:\/\/outplayed.tv\/media\/*"
1923 | ],
1924 | "url": "https:\/\/outplayed.tv\/oembed",
1925 | "discovery": true
1926 | }
1927 | ]
1928 | },
1929 | {
1930 | "provider_name": "Overflow",
1931 | "provider_url": "https:\/\/overflow.io",
1932 | "endpoints": [
1933 | {
1934 | "schemes": [
1935 | "https:\/\/overflow.io\/s\/*",
1936 | "https:\/\/overflow.io\/embed\/*"
1937 | ],
1938 | "url": "https:\/\/overflow.io\/services\/oembed",
1939 | "discovery": true
1940 | }
1941 | ]
1942 | },
1943 | {
1944 | "provider_name": "OZ",
1945 | "provider_url": "https:\/\/www.oz.com\/",
1946 | "endpoints": [
1947 | {
1948 | "schemes": [
1949 | "https:\/\/www.oz.com\/*\/video\/*"
1950 | ],
1951 | "url": "https:\/\/core.oz.com\/oembed",
1952 | "formats": [
1953 | "json",
1954 | "xml"
1955 | ]
1956 | }
1957 | ]
1958 | },
1959 | {
1960 | "provider_name": "Pastery",
1961 | "provider_url": "https:\/\/www.pastery.net",
1962 | "endpoints": [
1963 | {
1964 | "schemes": [
1965 | "http:\/\/pastery.net\/*",
1966 | "https:\/\/pastery.net\/*",
1967 | "http:\/\/www.pastery.net\/*",
1968 | "https:\/\/www.pastery.net\/*"
1969 | ],
1970 | "url": "https:\/\/www.pastery.net\/oembed",
1971 | "discovery": true
1972 | }
1973 | ]
1974 | },
1975 | {
1976 | "provider_name": "PingVP",
1977 | "provider_url": "https:\/\/www.pingvp.com\/",
1978 | "endpoints": [
1979 | {
1980 | "url": "https:\/\/beta.pingvp.com.kpnis.nl\/p\/oembed.php",
1981 | "discovery": true
1982 | }
1983 | ]
1984 | },
1985 | {
1986 | "provider_name": "Pixdor",
1987 | "provider_url": "http:\/\/www.pixdor.com\/",
1988 | "endpoints": [
1989 | {
1990 | "schemes": [
1991 | "https:\/\/store.pixdor.com\/place-marker-widget\/*\/show",
1992 | "https:\/\/store.pixdor.com\/map\/*\/show"
1993 | ],
1994 | "url": "https:\/\/store.pixdor.com\/oembed",
1995 | "formats": [
1996 | "json",
1997 | "xml"
1998 | ],
1999 | "discovery": true
2000 | }
2001 | ]
2002 | },
2003 | {
2004 | "provider_name": "Podbean",
2005 | "provider_url": "http:\/\/podbean.com",
2006 | "endpoints": [
2007 | {
2008 | "schemes": [
2009 | "https:\/\/*.podbean.com\/e\/*",
2010 | "http:\/\/*.podbean.com\/e\/*"
2011 | ],
2012 | "url": "https:\/\/api.podbean.com\/v1\/oembed"
2013 | }
2014 | ]
2015 | },
2016 | {
2017 | "provider_name": "Polaris Share",
2018 | "provider_url": "https:\/\/www.polarishare.com\/",
2019 | "endpoints": [
2020 | {
2021 | "schemes": [
2022 | "https:\/\/www.polarishare.com\/*\/*"
2023 | ],
2024 | "url": "https:\/\/api.polarishare.com\/rest\/api\/oembed",
2025 | "discovery": true
2026 | }
2027 | ]
2028 | },
2029 | {
2030 | "provider_name": "Poll Daddy",
2031 | "provider_url": "http:\/\/polldaddy.com",
2032 | "endpoints": [
2033 | {
2034 | "schemes": [
2035 | "http:\/\/*.polldaddy.com\/s\/*",
2036 | "http:\/\/*.polldaddy.com\/poll\/*",
2037 | "http:\/\/*.polldaddy.com\/ratings\/*"
2038 | ],
2039 | "url": "http:\/\/polldaddy.com\/oembed\/"
2040 | }
2041 | ]
2042 | },
2043 | {
2044 | "provider_name": "Port",
2045 | "provider_url": "http:\/\/www.sellwithport.com\/",
2046 | "endpoints": [
2047 | {
2048 | "schemes": [
2049 | "https:\/\/app.sellwithport.com\/#\/buyer\/*"
2050 | ],
2051 | "url": "https:\/\/api.sellwithport.com\/v1.0\/buyer\/oembed"
2052 | }
2053 | ]
2054 | },
2055 | {
2056 | "provider_name": "Portfolium",
2057 | "provider_url": "https:\/\/portfolium.com",
2058 | "endpoints": [
2059 | {
2060 | "schemes": [
2061 | "https:\/\/portfolium.com\/entry\/*"
2062 | ],
2063 | "url": "https:\/\/api.portfolium.com\/oembed"
2064 | }
2065 | ]
2066 | },
2067 | {
2068 | "provider_name": "posiXion",
2069 | "provider_url": "https:\/\/posixion.com\/",
2070 | "endpoints": [
2071 | {
2072 | "schemes": [
2073 | "https:\/\/posixion.com\/question\/*",
2074 | "https:\/\/posixion.com\/*\/question\/*"
2075 | ],
2076 | "url": "http:\/\/posixion.com\/services\/oembed\/"
2077 | }
2078 | ]
2079 | },
2080 | {
2081 | "provider_name": "Quiz.biz",
2082 | "provider_url": "http:\/\/www.quiz.biz\/",
2083 | "endpoints": [
2084 | {
2085 | "schemes": [
2086 | "http:\/\/www.quiz.biz\/quizz-*.html"
2087 | ],
2088 | "url": "http:\/\/www.quiz.biz\/api\/oembed",
2089 | "discovery": true
2090 | }
2091 | ]
2092 | },
2093 | {
2094 | "provider_name": "Quizz.biz",
2095 | "provider_url": "http:\/\/www.quizz.biz\/",
2096 | "endpoints": [
2097 | {
2098 | "schemes": [
2099 | "http:\/\/www.quizz.biz\/quizz-*.html"
2100 | ],
2101 | "url": "http:\/\/www.quizz.biz\/api\/oembed",
2102 | "discovery": true
2103 | }
2104 | ]
2105 | },
2106 | {
2107 | "provider_name": "RapidEngage",
2108 | "provider_url": "https:\/\/rapidengage.com",
2109 | "endpoints": [
2110 | {
2111 | "schemes": [
2112 | "https:\/\/rapidengage.com\/s\/*"
2113 | ],
2114 | "url": "https:\/\/rapidengage.com\/api\/oembed"
2115 | }
2116 | ]
2117 | },
2118 | {
2119 | "provider_name": "Reddit",
2120 | "provider_url": "https:\/\/reddit.com\/",
2121 | "endpoints": [
2122 | {
2123 | "schemes": [
2124 | "https:\/\/reddit.com\/r\/*\/comments\/*\/*",
2125 | "https:\/\/www.reddit.com\/r\/*\/comments\/*\/*"
2126 | ],
2127 | "url": "https:\/\/www.reddit.com\/oembed"
2128 | }
2129 | ]
2130 | },
2131 | {
2132 | "provider_name": "ReleaseWire",
2133 | "provider_url": "http:\/\/www.releasewire.com\/",
2134 | "endpoints": [
2135 | {
2136 | "schemes": [
2137 | "http:\/\/rwire.com\/*"
2138 | ],
2139 | "url": "http:\/\/publisher.releasewire.com\/oembed\/",
2140 | "discovery": true
2141 | }
2142 | ]
2143 | },
2144 | {
2145 | "provider_name": "Replit",
2146 | "provider_url": "https:\/\/repl.it\/",
2147 | "endpoints": [
2148 | {
2149 | "schemes": [
2150 | "https:\/\/repl.it\/@*\/*"
2151 | ],
2152 | "url": "https:\/\/repl.it\/data\/oembed",
2153 | "discovery": true
2154 | }
2155 | ]
2156 | },
2157 | {
2158 | "provider_name": "RepubHub",
2159 | "provider_url": "http:\/\/repubhub.icopyright.net\/",
2160 | "endpoints": [
2161 | {
2162 | "schemes": [
2163 | "http:\/\/repubhub.icopyright.net\/freePost.act?*"
2164 | ],
2165 | "url": "http:\/\/repubhub.icopyright.net\/oembed.act",
2166 | "discovery": true
2167 | }
2168 | ]
2169 | },
2170 | {
2171 | "provider_name": "ReverbNation",
2172 | "provider_url": "https:\/\/www.reverbnation.com\/",
2173 | "endpoints": [
2174 | {
2175 | "schemes": [
2176 | "https:\/\/www.reverbnation.com\/*",
2177 | "https:\/\/www.reverbnation.com\/*\/songs\/*"
2178 | ],
2179 | "url": "https:\/\/www.reverbnation.com\/oembed",
2180 | "discovery": true
2181 | }
2182 | ]
2183 | },
2184 | {
2185 | "provider_name": "RiffReporter",
2186 | "provider_url": "https:\/\/www.riffreporter.de\/",
2187 | "endpoints": [
2188 | {
2189 | "url": "https:\/\/www.riffreporter.de\/service\/oembed",
2190 | "discovery": true
2191 | }
2192 | ]
2193 | },
2194 | {
2195 | "provider_name": "Roomshare",
2196 | "provider_url": "http:\/\/roomshare.jp",
2197 | "endpoints": [
2198 | {
2199 | "schemes": [
2200 | "http:\/\/roomshare.jp\/post\/*",
2201 | "http:\/\/roomshare.jp\/en\/post\/*"
2202 | ],
2203 | "url": "http:\/\/roomshare.jp\/en\/oembed.{format}"
2204 | }
2205 | ]
2206 | },
2207 | {
2208 | "provider_name": "RoosterTeeth",
2209 | "provider_url": "https:\/\/roosterteeth.com",
2210 | "endpoints": [
2211 | {
2212 | "schemes": [
2213 | "https:\/\/roosterteeth.com\/*"
2214 | ],
2215 | "url": "https:\/\/roosterteeth.com\/oembed",
2216 | "formats": [
2217 | "json"
2218 | ],
2219 | "discovery": true
2220 | }
2221 | ]
2222 | },
2223 | {
2224 | "provider_name": "Rumble",
2225 | "provider_url": "https:\/\/rumble.com\/",
2226 | "endpoints": [
2227 | {
2228 | "url": "https:\/\/rumble.com\/api\/Media\/oembed.{format}",
2229 | "discovery": true
2230 | }
2231 | ]
2232 | },
2233 | {
2234 | "provider_name": "Sapo Videos",
2235 | "provider_url": "http:\/\/videos.sapo.pt",
2236 | "endpoints": [
2237 | {
2238 | "schemes": [
2239 | "http:\/\/videos.sapo.pt\/*"
2240 | ],
2241 | "url": "http:\/\/videos.sapo.pt\/oembed"
2242 | }
2243 | ]
2244 | },
2245 | {
2246 | "provider_name": "Screen9",
2247 | "provider_url": "http:\/\/www.screen9.com\/",
2248 | "endpoints": [
2249 | {
2250 | "schemes": [
2251 | "https:\/\/console.screen9.com\/*",
2252 | "https:\/\/*.screen9.tv\/*"
2253 | ],
2254 | "url": "https:\/\/api.screen9.com\/oembed"
2255 | }
2256 | ]
2257 | },
2258 | {
2259 | "provider_name": "Screencast.com",
2260 | "provider_url": "http:\/\/www.screencast.com\/",
2261 | "endpoints": [
2262 | {
2263 | "url": "https:\/\/api.screencast.com\/external\/oembed",
2264 | "discovery": true
2265 | }
2266 | ]
2267 | },
2268 | {
2269 | "provider_name": "Screenr",
2270 | "provider_url": "http:\/\/www.screenr.com\/",
2271 | "endpoints": [
2272 | {
2273 | "schemes": [
2274 | "http:\/\/www.screenr.com\/*\/"
2275 | ],
2276 | "url": "http:\/\/www.screenr.com\/api\/oembed.{format}"
2277 | }
2278 | ]
2279 | },
2280 | {
2281 | "provider_name": "ScribbleMaps",
2282 | "provider_url": "https:\/\/scribblemaps.com",
2283 | "endpoints": [
2284 | {
2285 | "schemes": [
2286 | "http:\/\/www.scribblemaps.com\/maps\/view\/*",
2287 | "https:\/\/www.scribblemaps.com\/maps\/view\/*",
2288 | "http:\/\/scribblemaps.com\/maps\/view\/*",
2289 | "https:\/\/scribblemaps.com\/maps\/view\/*"
2290 | ],
2291 | "url": "https:\/\/scribblemaps.com\/api\/services\/oembed.{format}",
2292 | "discovery": true
2293 | }
2294 | ]
2295 | },
2296 | {
2297 | "provider_name": "Scribd",
2298 | "provider_url": "http:\/\/www.scribd.com\/",
2299 | "endpoints": [
2300 | {
2301 | "schemes": [
2302 | "http:\/\/www.scribd.com\/doc\/*"
2303 | ],
2304 | "url": "http:\/\/www.scribd.com\/services\/oembed\/"
2305 | }
2306 | ]
2307 | },
2308 | {
2309 | "provider_name": "SendtoNews",
2310 | "provider_url": "http:\/\/www.sendtonews.com\/",
2311 | "endpoints": [
2312 | {
2313 | "schemes": [
2314 | "https:\/\/embed.sendtonews.com\/oembed\/*"
2315 | ],
2316 | "url": "https:\/\/embed.sendtonews.com\/services\/oembed",
2317 | "discovery": true,
2318 | "formats": [
2319 | "json",
2320 | "xml"
2321 | ]
2322 | }
2323 | ]
2324 | },
2325 | {
2326 | "provider_name": "ShortNote",
2327 | "provider_url": "https:\/\/www.shortnote.jp\/",
2328 | "endpoints": [
2329 | {
2330 | "schemes": [
2331 | "https:\/\/www.shortnote.jp\/view\/notes\/*"
2332 | ],
2333 | "url": "https:\/\/www.shortnote.jp\/oembed\/",
2334 | "discovery": true
2335 | }
2336 | ]
2337 | },
2338 | {
2339 | "provider_name": "Shoudio",
2340 | "provider_url": "http:\/\/shoudio.com",
2341 | "endpoints": [
2342 | {
2343 | "schemes": [
2344 | "http:\/\/shoudio.com\/*",
2345 | "http:\/\/shoud.io\/*"
2346 | ],
2347 | "url": "http:\/\/shoudio.com\/api\/oembed"
2348 | }
2349 | ]
2350 | },
2351 | {
2352 | "provider_name": "Show the Way, actionable location info",
2353 | "provider_url": "https:\/\/showtheway.io",
2354 | "endpoints": [
2355 | {
2356 | "schemes": [
2357 | "https:\/\/showtheway.io\/to\/*"
2358 | ],
2359 | "url": "https:\/\/showtheway.io\/oembed",
2360 | "discovery": true
2361 | }
2362 | ]
2363 | },
2364 | {
2365 | "provider_name": "Simplecast",
2366 | "provider_url": "https:\/\/simplecast.com",
2367 | "endpoints": [
2368 | {
2369 | "schemes": [
2370 | "https:\/\/simplecast.com\/s\/*"
2371 | ],
2372 | "url": "https:\/\/simplecast.com\/oembed",
2373 | "formats": [
2374 | "json"
2375 | ]
2376 | }
2377 | ]
2378 | },
2379 | {
2380 | "provider_name": "Sizzle",
2381 | "provider_url": "https:\/\/onsizzle.com\/",
2382 | "endpoints": [
2383 | {
2384 | "schemes": [
2385 | "https:\/\/onsizzle.com\/i\/*"
2386 | ],
2387 | "url": "https:\/\/onsizzle.com\/oembed",
2388 | "discovery": true
2389 | }
2390 | ]
2391 | },
2392 | {
2393 | "provider_name": "Sketchfab",
2394 | "provider_url": "http:\/\/sketchfab.com",
2395 | "endpoints": [
2396 | {
2397 | "schemes": [
2398 | "http:\/\/sketchfab.com\/models\/*",
2399 | "https:\/\/sketchfab.com\/models\/*",
2400 | "https:\/\/sketchfab.com\/*\/folders\/*"
2401 | ],
2402 | "url": "http:\/\/sketchfab.com\/oembed",
2403 | "formats": [
2404 | "json"
2405 | ]
2406 | }
2407 | ]
2408 | },
2409 | {
2410 | "provider_name": "SlideShare",
2411 | "provider_url": "http:\/\/www.slideshare.net\/",
2412 | "endpoints": [
2413 | {
2414 | "schemes": [
2415 | "http:\/\/www.slideshare.net\/*\/*",
2416 | "http:\/\/fr.slideshare.net\/*\/*",
2417 | "http:\/\/de.slideshare.net\/*\/*",
2418 | "http:\/\/es.slideshare.net\/*\/*",
2419 | "http:\/\/pt.slideshare.net\/*\/*"
2420 | ],
2421 | "url": "http:\/\/www.slideshare.net\/api\/oembed\/2",
2422 | "discovery": true
2423 | }
2424 | ]
2425 | },
2426 | {
2427 | "provider_name": "SmashNotes",
2428 | "provider_url": "https:\/\/smashnotes.com",
2429 | "endpoints": [
2430 | {
2431 | "schemes": [
2432 | "https:\/\/smashnotes.com\/p\/*",
2433 | "https:\/\/smashnotes.com\/p\/*\/e\/* - https:\/\/smashnotes.com\/p\/*\/e\/*\/s\/*"
2434 | ],
2435 | "url": "https:\/\/smashnotes.com\/services\/oembed",
2436 | "discovery": true
2437 | }
2438 | ]
2439 | },
2440 | {
2441 | "provider_name": "SmugMug",
2442 | "provider_url": "https:\/\/www.smugmug.com\/",
2443 | "endpoints": [
2444 | {
2445 | "schemes": [
2446 | "http:\/\/*.smugmug.com\/*",
2447 | "https:\/\/*.smugmug.com\/*"
2448 | ],
2449 | "url": "https:\/\/api.smugmug.com\/services\/oembed\/",
2450 | "discovery": true
2451 | }
2452 | ]
2453 | },
2454 | {
2455 | "provider_name": "SocialExplorer",
2456 | "provider_url": "https:\/\/www.socialexplorer.com\/",
2457 | "endpoints": [
2458 | {
2459 | "schemes": [
2460 | "https:\/\/www.socialexplorer.com\/*\/explore",
2461 | "https:\/\/www.socialexplorer.com\/*\/view",
2462 | "https:\/\/www.socialexplorer.com\/*\/edit",
2463 | "https:\/\/www.socialexplorer.com\/*\/embed"
2464 | ],
2465 | "url": "https:\/\/www.socialexplorer.com\/services\/oembed\/",
2466 | "discovery": true
2467 | }
2468 | ]
2469 | },
2470 | {
2471 | "provider_name": "Songlink",
2472 | "provider_url": "https:\/\/song.link",
2473 | "endpoints": [
2474 | {
2475 | "schemes": [
2476 | "https:\/\/song.link\/*"
2477 | ],
2478 | "url": "https:\/\/song.link\/oembed",
2479 | "formats": [
2480 | "json"
2481 | ],
2482 | "discovery": true
2483 | }
2484 | ]
2485 | },
2486 | {
2487 | "provider_name": "SoundCloud",
2488 | "provider_url": "http:\/\/soundcloud.com\/",
2489 | "endpoints": [
2490 | {
2491 | "schemes": [
2492 | "http:\/\/soundcloud.com\/*",
2493 | "https:\/\/soundcloud.com\/*"
2494 | ],
2495 | "url": "https:\/\/soundcloud.com\/oembed"
2496 | }
2497 | ]
2498 | },
2499 | {
2500 | "provider_name": "Soundsgood",
2501 | "provider_url": "https:\/\/soundsgood.co",
2502 | "endpoints": [
2503 | {
2504 | "schemes": [
2505 | "https:\/\/play.soundsgood.co\/playlist\/*",
2506 | "https:\/\/soundsgood.co\/playlist\/*"
2507 | ],
2508 | "url": "https:\/\/play.soundsgood.co\/oembed",
2509 | "discovery": true,
2510 | "formats": [
2511 | "json",
2512 | "xml"
2513 | ]
2514 | }
2515 | ]
2516 | },
2517 | {
2518 | "provider_name": "SpeakerDeck",
2519 | "provider_url": "https:\/\/speakerdeck.com",
2520 | "endpoints": [
2521 | {
2522 | "schemes": [
2523 | "http:\/\/speakerdeck.com\/*\/*",
2524 | "https:\/\/speakerdeck.com\/*\/*"
2525 | ],
2526 | "url": "https:\/\/speakerdeck.com\/oembed.json",
2527 | "discovery": true,
2528 | "formats": [
2529 | "json"
2530 | ]
2531 | }
2532 | ]
2533 | },
2534 | {
2535 | "provider_name": "Spotful",
2536 | "provider_url": "https:\/\/bespotful.com",
2537 | "endpoints": [
2538 | {
2539 | "schemes": [
2540 | "http:\/\/play.bespotful.com\/*"
2541 | ],
2542 | "url": "https:\/\/api.bespotful.com\/oembed",
2543 | "discovery": true
2544 | }
2545 | ]
2546 | },
2547 | {
2548 | "provider_name": "Spotify",
2549 | "provider_url": "https:\/\/spotify.com\/",
2550 | "endpoints": [
2551 | {
2552 | "schemes": [
2553 | "https:\/\/*.spotify.com\/*",
2554 | "spotify:*"
2555 | ],
2556 | "url": "https:\/\/embed.spotify.com\/oembed\/"
2557 | }
2558 | ]
2559 | },
2560 | {
2561 | "provider_name": "Spreaker",
2562 | "provider_url": "https:\/\/www.spreaker.com\/",
2563 | "endpoints": [
2564 | {
2565 | "schemes": [
2566 | "http:\/\/*.spreaker.com\/*",
2567 | "https:\/\/*.spreaker.com\/*"
2568 | ],
2569 | "url": "https:\/\/api.spreaker.com\/oembed",
2570 | "discovery": true
2571 | }
2572 | ]
2573 | },
2574 | {
2575 | "provider_name": "Stanford Digital Repository",
2576 | "provider_url": "https:\/\/purl.stanford.edu\/",
2577 | "endpoints": [
2578 | {
2579 | "schemes": [
2580 | "https:\/\/purl.stanford.edu\/*"
2581 | ],
2582 | "url": "https:\/\/purl.stanford.edu\/embed.{format}",
2583 | "discovery": true
2584 | }
2585 | ]
2586 | },
2587 | {
2588 | "provider_name": "Streamable",
2589 | "provider_url": "https:\/\/streamable.com\/",
2590 | "endpoints": [
2591 | {
2592 | "schemes": [
2593 | "http:\/\/streamable.com\/*",
2594 | "https:\/\/streamable.com\/*"
2595 | ],
2596 | "url": "https:\/\/api.streamable.com\/oembed.json",
2597 | "discovery": true
2598 | }
2599 | ]
2600 | },
2601 | {
2602 | "provider_name": "StreamOneCloud",
2603 | "provider_url": "https:\/\/www.streamone.nl",
2604 | "endpoints": [
2605 | {
2606 | "schemes": [
2607 | "https:\/\/content.streamonecloud.net\/embed\/*"
2608 | ],
2609 | "url": "https:\/\/content.streamonecloud.net\/oembed",
2610 | "discovery": true
2611 | }
2612 | ]
2613 | },
2614 | {
2615 | "provider_name": "Sutori",
2616 | "provider_url": "https:\/\/www.sutori.com\/",
2617 | "endpoints": [
2618 | {
2619 | "schemes": [
2620 | "https:\/\/www.sutori.com\/story\/*"
2621 | ],
2622 | "url": "https:\/\/www.sutori.com\/api\/oembed",
2623 | "discovery": true,
2624 | "formats": [
2625 | "json"
2626 | ]
2627 | }
2628 | ]
2629 | },
2630 | {
2631 | "provider_name": "Sway",
2632 | "provider_url": "https:\/\/www.sway.com",
2633 | "endpoints": [
2634 | {
2635 | "schemes": [
2636 | "https:\/\/sway.com\/*",
2637 | "https:\/\/www.sway.com\/*"
2638 | ],
2639 | "url": "https:\/\/sway.com\/api\/v1.0\/oembed",
2640 | "discovery": true
2641 | }
2642 | ]
2643 | },
2644 | {
2645 | "provider_name": "TED",
2646 | "provider_url": "https:\/\/www.ted.com",
2647 | "endpoints": [
2648 | {
2649 | "schemes": [
2650 | "http:\/\/ted.com\/talks\/*",
2651 | "https:\/\/ted.com\/talks\/*",
2652 | "https:\/\/www.ted.com\/talks\/*"
2653 | ],
2654 | "url": "https:\/\/www.ted.com\/services\/v1\/oembed.{format}",
2655 | "discovery": true
2656 | }
2657 | ]
2658 | },
2659 | {
2660 | "provider_name": "The New York Times",
2661 | "provider_url": "https:\/\/www.nytimes.com",
2662 | "endpoints": [
2663 | {
2664 | "schemes": [
2665 | "https:\/\/www.nytimes.com\/svc\/oembed",
2666 | "https:\/\/nytimes.com\/*",
2667 | "https:\/\/*.nytimes.com\/*"
2668 | ],
2669 | "url": "https:\/\/www.nytimes.com\/svc\/oembed\/json\/",
2670 | "discovery": true
2671 | }
2672 | ]
2673 | },
2674 | {
2675 | "provider_name": "They Said So",
2676 | "provider_url": "https:\/\/theysaidso.com\/",
2677 | "endpoints": [
2678 | {
2679 | "schemes": [
2680 | "https:\/\/theysaidso.com\/image\/*"
2681 | ],
2682 | "url": "https:\/\/theysaidso.com\/extensions\/oembed\/",
2683 | "discovery": true
2684 | }
2685 | ]
2686 | },
2687 | {
2688 | "provider_name": "TickCounter",
2689 | "provider_url": "https:\/\/www.tickcounter.com",
2690 | "endpoints": [
2691 | {
2692 | "schemes": [
2693 | "http:\/\/www.tickcounter.com\/countdown\/*",
2694 | "http:\/\/www.tickcounter.com\/countup\/*",
2695 | "http:\/\/www.tickcounter.com\/ticker\/*",
2696 | "http:\/\/www.tickcounter.com\/worldclock\/*",
2697 | "https:\/\/www.tickcounter.com\/countdown\/*",
2698 | "https:\/\/www.tickcounter.com\/countup\/*",
2699 | "https:\/\/www.tickcounter.com\/ticker\/*",
2700 | "https:\/\/www.tickcounter.com\/worldclock\/*"
2701 | ],
2702 | "url": "https:\/\/www.tickcounter.com\/oembed",
2703 | "discovery": true
2704 | }
2705 | ]
2706 | },
2707 | {
2708 | "provider_name": "Toornament",
2709 | "provider_url": "https:\/\/www.toornament.com\/",
2710 | "endpoints": [
2711 | {
2712 | "schemes": [
2713 | "https:\/\/www.toornament.com\/tournaments\/*\/information",
2714 | "https:\/\/www.toornament.com\/tournaments\/*\/registration\/",
2715 | "https:\/\/www.toornament.com\/tournaments\/*\/matches\/schedule",
2716 | "https:\/\/www.toornament.com\/tournaments\/*\/stages\/*\/"
2717 | ],
2718 | "url": "https:\/\/widget.toornament.com\/oembed",
2719 | "discovery": true,
2720 | "formats": [
2721 | "json",
2722 | "xml"
2723 | ]
2724 | }
2725 | ]
2726 | },
2727 | {
2728 | "provider_name": "Topy",
2729 | "provider_url": "http:\/\/www.topy.se\/",
2730 | "endpoints": [
2731 | {
2732 | "schemes": [
2733 | "http:\/\/www.topy.se\/image\/*"
2734 | ],
2735 | "url": "http:\/\/www.topy.se\/oembed\/",
2736 | "discovery": true
2737 | }
2738 | ]
2739 | },
2740 | {
2741 | "provider_name": "Tuxx",
2742 | "provider_url": "https:\/\/www.tuxx.be\/",
2743 | "endpoints": [
2744 | {
2745 | "schemes": [
2746 | "https:\/\/www.tuxx.be\/*"
2747 | ],
2748 | "url": "https:\/\/www.tuxx.be\/services\/oembed",
2749 | "discovery": true
2750 | }
2751 | ]
2752 | },
2753 | {
2754 | "provider_name": "tvcf",
2755 | "provider_url": "http:\/\/tvcf.co.kr",
2756 | "endpoints": [
2757 | {
2758 | "schemes": [
2759 | "https:\/\/play.tvcf.co.kr\/*",
2760 | "https:\/\/*.tvcf.co.kr\/*"
2761 | ],
2762 | "url": "https:\/\/play.tvcf.co.kr\/rest\/oembed"
2763 | }
2764 | ]
2765 | },
2766 | {
2767 | "provider_name": "Twitch",
2768 | "provider_url": "https:\/\/www.twitch.tv",
2769 | "endpoints": [
2770 | {
2771 | "schemes": [
2772 | "http:\/\/clips.twitch.tv\/*",
2773 | "https:\/\/clips.twitch.tv\/*",
2774 | "http:\/\/www.twitch.tv\/*",
2775 | "https:\/\/www.twitch.tv\/*",
2776 | "http:\/\/twitch.tv\/*",
2777 | "https:\/\/twitch.tv\/*"
2778 | ],
2779 | "url": "https:\/\/api.twitch.tv\/v5\/oembed",
2780 | "formats": [
2781 | "json"
2782 | ]
2783 | }
2784 | ]
2785 | },
2786 | {
2787 | "provider_name": "Twitter",
2788 | "provider_url": "http:\/\/www.twitter.com\/",
2789 | "endpoints": [
2790 | {
2791 | "schemes": [
2792 | "https:\/\/twitter.com\/*\/status\/*",
2793 | "https:\/\/*.twitter.com\/*\/status\/*"
2794 | ],
2795 | "url": "https:\/\/publish.twitter.com\/oembed"
2796 | }
2797 | ]
2798 | },
2799 | {
2800 | "provider_name": "TypeCast",
2801 | "provider_url": "https:\/\/typecast.ai",
2802 | "endpoints": [
2803 | {
2804 | "schemes": [
2805 | "https:\/\/play.typecast.ai\/s\/*",
2806 | "https:\/\/play.typecast.ai\/e\/*",
2807 | "https:\/\/play.typecast.ai\/*"
2808 | ],
2809 | "url": "https:\/\/play.typecast.ai\/oembed"
2810 | }
2811 | ]
2812 | },
2813 | {
2814 | "provider_name": "Typlog",
2815 | "provider_url": "https:\/\/typlog.com",
2816 | "endpoints": [
2817 | {
2818 | "url": "https:\/\/typlog.com\/oembed",
2819 | "discovery": true
2820 | }
2821 | ]
2822 | },
2823 | {
2824 | "provider_name": "Ubideo",
2825 | "provider_url": "https:\/\/player.ubideo.com\/",
2826 | "endpoints": [
2827 | {
2828 | "schemes": [
2829 | "https:\/\/player.ubideo.com\/*"
2830 | ],
2831 | "url": "https:\/\/player.ubideo.com\/api\/oembed.json",
2832 | "formats": [
2833 | "json"
2834 | ]
2835 | }
2836 | ]
2837 | },
2838 | {
2839 | "provider_name": "University of Cambridge Map",
2840 | "provider_url": "https:\/\/map.cam.ac.uk",
2841 | "endpoints": [
2842 | {
2843 | "schemes": [
2844 | "https:\/\/map.cam.ac.uk\/*"
2845 | ],
2846 | "url": "https:\/\/map.cam.ac.uk\/oembed\/"
2847 | }
2848 | ]
2849 | },
2850 | {
2851 | "provider_name": "UnivParis1.Pod",
2852 | "provider_url": "https:\/\/mediatheque.univ-paris1.fr\/",
2853 | "endpoints": [
2854 | {
2855 | "schemes": [
2856 | "https:\/\/mediatheque.univ-paris1.fr\/video\/*"
2857 | ],
2858 | "url": "https:\/\/mediatheque.univ-paris1.fr\/oembed",
2859 | "discovery": true
2860 | }
2861 | ]
2862 | },
2863 | {
2864 | "provider_name": "UOL",
2865 | "provider_url": "https:\/\/mais.uol.com.br\/",
2866 | "endpoints": [
2867 | {
2868 | "schemes": [
2869 | "https:\/\/*.uol.com.br\/view\/*",
2870 | "https:\/\/*.uol.com.br\/video\/*"
2871 | ],
2872 | "url": "https:\/\/mais.uol.com.br\/apiuol\/v3\/oembed\/view",
2873 | "discovery": true
2874 | }
2875 | ]
2876 | },
2877 | {
2878 | "provider_name": "Ustream",
2879 | "provider_url": "http:\/\/www.ustream.tv",
2880 | "endpoints": [
2881 | {
2882 | "schemes": [
2883 | "http:\/\/*.ustream.tv\/*",
2884 | "http:\/\/*.ustream.com\/*"
2885 | ],
2886 | "url": "http:\/\/www.ustream.tv\/oembed",
2887 | "formats": [
2888 | "json"
2889 | ]
2890 | }
2891 | ]
2892 | },
2893 | {
2894 | "provider_name": "uStudio, Inc.",
2895 | "provider_url": "https:\/\/www.ustudio.com",
2896 | "endpoints": [
2897 | {
2898 | "schemes": [
2899 | "https:\/\/*.ustudio.com\/embed\/*",
2900 | "https:\/\/*.ustudio.com\/embed\/*\/*"
2901 | ],
2902 | "url": "https:\/\/app.ustudio.com\/api\/v2\/oembed",
2903 | "discovery": true,
2904 | "formats": [
2905 | "json"
2906 | ]
2907 | }
2908 | ]
2909 | },
2910 | {
2911 | "provider_name": "Utposts",
2912 | "provider_url": "https:\/\/www.utposts.com\/",
2913 | "endpoints": [
2914 | {
2915 | "schemes": [
2916 | "https:\/\/www.utposts.com\/products\/*",
2917 | "http:\/\/www.utposts.com\/products\/*",
2918 | "https:\/\/utposts.com\/products\/*",
2919 | "http:\/\/utposts.com\/products\/*"
2920 | ],
2921 | "url": "https:\/\/www.utposts.com\/api\/oembed",
2922 | "formats": [
2923 | "json"
2924 | ]
2925 | }
2926 | ]
2927 | },
2928 | {
2929 | "provider_name": "Uttles",
2930 | "provider_url": "http:\/\/uttles.com",
2931 | "endpoints": [
2932 | {
2933 | "schemes": [
2934 | "http:\/\/uttles.com\/uttle\/*"
2935 | ],
2936 | "url": "http:\/\/uttles.com\/api\/reply\/oembed",
2937 | "discovery": true
2938 | }
2939 | ]
2940 | },
2941 | {
2942 | "provider_name": "VeeR VR",
2943 | "provider_url": "http:\/\/veer.tv\/",
2944 | "endpoints": [
2945 | {
2946 | "schemes": [
2947 | "http:\/\/veer.tv\/videos\/*"
2948 | ],
2949 | "url": "https:\/\/api.veer.tv\/oembed",
2950 | "discovery": true
2951 | },
2952 | {
2953 | "schemes": [
2954 | "http:\/\/veervr.tv\/videos\/*"
2955 | ],
2956 | "url": "https:\/\/api.veervr.tv\/oembed",
2957 | "discovery": true
2958 | }
2959 | ]
2960 | },
2961 | {
2962 | "provider_name": "Verse",
2963 | "provider_url": "http:\/\/verse.com\/",
2964 | "endpoints": [
2965 | {
2966 | "url": "http:\/\/verse.com\/services\/oembed\/"
2967 | }
2968 | ]
2969 | },
2970 | {
2971 | "provider_name": "VEVO",
2972 | "provider_url": "http:\/\/www.vevo.com\/",
2973 | "endpoints": [
2974 | {
2975 | "schemes": [
2976 | "http:\/\/www.vevo.com\/*",
2977 | "https:\/\/www.vevo.com\/*"
2978 | ],
2979 | "url": "https:\/\/www.vevo.com\/oembed",
2980 | "discovery": false
2981 | }
2982 | ]
2983 | },
2984 | {
2985 | "provider_name": "VideoJug",
2986 | "provider_url": "http:\/\/www.videojug.com",
2987 | "endpoints": [
2988 | {
2989 | "schemes": [
2990 | "http:\/\/www.videojug.com\/film\/*",
2991 | "http:\/\/www.videojug.com\/interview\/*"
2992 | ],
2993 | "url": "http:\/\/www.videojug.com\/oembed.{format}"
2994 | }
2995 | ]
2996 | },
2997 | {
2998 | "provider_name": "Vidlit",
2999 | "provider_url": "https:\/\/vidl.it\/",
3000 | "endpoints": [
3001 | {
3002 | "schemes": [
3003 | "https:\/\/vidl.it\/*"
3004 | ],
3005 | "url": "https:\/\/api.vidl.it\/oembed",
3006 | "discovery": true
3007 | }
3008 | ]
3009 | },
3010 | {
3011 | "provider_name": "Vidmizer",
3012 | "provider_url": "https:\/\/www.vidmizer.com\/",
3013 | "endpoints": [
3014 | {
3015 | "schemes": [
3016 | "https:\/\/players.vidmizer.com\/*"
3017 | ],
3018 | "url": "https:\/\/app-v2.vidmizer.com\/api\/oembed",
3019 | "discovery": true
3020 | }
3021 | ]
3022 | },
3023 | {
3024 | "provider_name": "Vidyard",
3025 | "provider_url": "http:\/\/www.vidyard.com",
3026 | "endpoints": [
3027 | {
3028 | "schemes": [
3029 | "http:\/\/embed.vidyard.com\/*",
3030 | "http:\/\/play.vidyard.com\/*",
3031 | "http:\/\/share.vidyard.com\/*",
3032 | "http:\/\/*.hubs.vidyard.com\/*"
3033 | ],
3034 | "url": "https:\/\/api.vidyard.com\/dashboard\/v1.1\/oembed",
3035 | "discovery": true
3036 | }
3037 | ]
3038 | },
3039 | {
3040 | "provider_name": "Vimeo",
3041 | "provider_url": "https:\/\/vimeo.com\/",
3042 | "endpoints": [
3043 | {
3044 | "schemes": [
3045 | "https:\/\/vimeo.com\/*",
3046 | "https:\/\/vimeo.com\/album\/*\/video\/*",
3047 | "https:\/\/vimeo.com\/channels\/*\/*",
3048 | "https:\/\/vimeo.com\/groups\/*\/videos\/*",
3049 | "https:\/\/vimeo.com\/ondemand\/*\/*",
3050 | "https:\/\/player.vimeo.com\/video\/*"
3051 | ],
3052 | "url": "https:\/\/vimeo.com\/api\/oembed.{format}",
3053 | "discovery": true
3054 | }
3055 | ]
3056 | },
3057 | {
3058 | "provider_name": "Viously",
3059 | "provider_url": "https:\/\/www.viously.com",
3060 | "endpoints": [
3061 | {
3062 | "schemes": [
3063 | "https:\/\/www.viously.com\/*\/*"
3064 | ],
3065 | "url": "https:\/\/www.viously.com\/oembed",
3066 | "discovery": true,
3067 | "formats": [
3068 | "json",
3069 | "xml"
3070 | ]
3071 | }
3072 | ]
3073 | },
3074 | {
3075 | "provider_name": "Viziosphere",
3076 | "provider_url": "http:\/\/www.viziosphere.com",
3077 | "endpoints": [
3078 | {
3079 | "schemes": [
3080 | "http:\/\/viziosphere.com\/3dphoto*"
3081 | ],
3082 | "url": "http:\/\/viziosphere.com\/services\/oembed\/",
3083 | "discovery": true
3084 | }
3085 | ]
3086 | },
3087 | {
3088 | "provider_name": "Vizydrop",
3089 | "provider_url": "https:\/\/vizydrop.com",
3090 | "endpoints": [
3091 | {
3092 | "schemes": [
3093 | "https:\/\/vizydrop.com\/shared\/*"
3094 | ],
3095 | "url": "https:\/\/vizydrop.com\/oembed"
3096 | }
3097 | ]
3098 | },
3099 | {
3100 | "provider_name": "Vlipsy",
3101 | "provider_url": "https:\/\/vlipsy.com\/",
3102 | "endpoints": [
3103 | {
3104 | "schemes": [
3105 | "https:\/\/vlipsy.com\/*"
3106 | ],
3107 | "url": "https:\/\/vlipsy.com\/oembed",
3108 | "discovery": true
3109 | }
3110 | ]
3111 | },
3112 | {
3113 | "provider_name": "VLIVE",
3114 | "provider_url": "https:\/\/www.vlive.tv",
3115 | "endpoints": [
3116 | {
3117 | "url": "https:\/\/www.vlive.tv\/oembed",
3118 | "schemes": [
3119 | "https:\/\/www.vlive.tv\/video\/*"
3120 | ],
3121 | "formats": [
3122 | "json"
3123 | ]
3124 | }
3125 | ]
3126 | },
3127 | {
3128 | "provider_name": "Vlurb",
3129 | "provider_url": "https:\/\/www.vlurb.co\/",
3130 | "endpoints": [
3131 | {
3132 | "schemes": [
3133 | "http:\/\/vlurb.co\/video\/*",
3134 | "https:\/\/vlurb.co\/video\/*"
3135 | ],
3136 | "url": "https:\/\/vlurb.co\/oembed.json",
3137 | "discovery": true
3138 | }
3139 | ]
3140 | },
3141 | {
3142 | "provider_name": "VoxSnap",
3143 | "provider_url": "https:\/\/voxsnap.com\/",
3144 | "endpoints": [
3145 | {
3146 | "schemes": [
3147 | "https:\/\/article.voxsnap.com\/*\/*"
3148 | ],
3149 | "url": "https:\/\/data.voxsnap.com\/oembed",
3150 | "discovery": true,
3151 | "formats": [
3152 | "json"
3153 | ]
3154 | }
3155 | ]
3156 | },
3157 | {
3158 | "provider_name": "wecandeo",
3159 | "provider_url": "http:\/\/www.wecandeo.com\/",
3160 | "endpoints": [
3161 | {
3162 | "url": "http:\/\/play.wecandeo.com\/oembed",
3163 | "discovery": true
3164 | }
3165 | ]
3166 | },
3167 | {
3168 | "provider_name": "Wiredrive",
3169 | "provider_url": "https:\/\/www.wiredrive.com\/",
3170 | "endpoints": [
3171 | {
3172 | "schemes": [
3173 | "https:\/\/*.wiredrive.com\/*"
3174 | ],
3175 | "url": "http:\/\/*.wiredrive.com\/present-oembed\/",
3176 | "formats": [
3177 | "json"
3178 | ],
3179 | "discovery": true
3180 | }
3181 | ]
3182 | },
3183 | {
3184 | "provider_name": "Wistia, Inc.",
3185 | "provider_url": "https:\/\/wistia.com\/",
3186 | "endpoints": [
3187 | {
3188 | "schemes": [
3189 | "https:\/\/fast.wistia.com\/embed\/iframe\/*",
3190 | "https:\/\/fast.wistia.com\/embed\/playlists\/*",
3191 | "https:\/\/*.wistia.com\/medias\/*"
3192 | ],
3193 | "url": "https:\/\/fast.wistia.com\/oembed.{format}",
3194 | "discovery": true
3195 | }
3196 | ]
3197 | },
3198 | {
3199 | "provider_name": "wizer.me",
3200 | "provider_url": "http:\/\/www.wizer.me\/",
3201 | "endpoints": [
3202 | {
3203 | "schemes": [
3204 | "http:\/\/*.wizer.me\/learn\/*",
3205 | "https:\/\/*.wizer.me\/learn\/*",
3206 | "http:\/\/*.wizer.me\/preview\/*",
3207 | "https:\/\/*.wizer.me\/preview\/*"
3208 | ],
3209 | "url": "http:\/\/app.wizer.me\/api\/oembed.{format}",
3210 | "discovery": true
3211 | }
3212 | ]
3213 | },
3214 | {
3215 | "provider_name": "Wootled",
3216 | "provider_url": "http:\/\/www.wootled.com\/",
3217 | "endpoints": [
3218 | {
3219 | "url": "http:\/\/www.wootled.com\/oembed"
3220 | }
3221 | ]
3222 | },
3223 | {
3224 | "provider_name": "WordPress.com",
3225 | "provider_url": "http:\/\/wordpress.com\/",
3226 | "endpoints": [
3227 | {
3228 | "url": "http:\/\/public-api.wordpress.com\/oembed\/",
3229 | "discovery": true
3230 | }
3231 | ]
3232 | },
3233 | {
3234 | "provider_name": "Xpression",
3235 | "provider_url": "https:\/\/web.xpression.jp",
3236 | "endpoints": [
3237 | {
3238 | "schemes": [
3239 | "https:\/\/web.xpression.jp\/video\/*"
3240 | ],
3241 | "url": "https:\/\/web.xpression.jp\/api\/oembed",
3242 | "formats": [
3243 | "json",
3244 | "xml"
3245 | ]
3246 | }
3247 | ]
3248 | },
3249 | {
3250 | "provider_name": "Yes, I Know IT!",
3251 | "provider_url": "http:\/\/yesik.it",
3252 | "endpoints": [
3253 | {
3254 | "schemes": [
3255 | "http:\/\/yesik.it\/*",
3256 | "http:\/\/www.yesik.it\/*"
3257 | ],
3258 | "url": "http:\/\/yesik.it\/s\/oembed",
3259 | "formats": [
3260 | "json"
3261 | ],
3262 | "discovery": true
3263 | }
3264 | ]
3265 | },
3266 | {
3267 | "provider_name": "YFrog",
3268 | "provider_url": "http:\/\/yfrog.com\/",
3269 | "endpoints": [
3270 | {
3271 | "schemes": [
3272 | "http:\/\/*.yfrog.com\/*",
3273 | "http:\/\/yfrog.us\/*"
3274 | ],
3275 | "url": "http:\/\/www.yfrog.com\/api\/oembed",
3276 | "formats": [
3277 | "json"
3278 | ]
3279 | }
3280 | ]
3281 | },
3282 | {
3283 | "provider_name": "YouTube",
3284 | "provider_url": "https:\/\/www.youtube.com\/",
3285 | "endpoints": [
3286 | {
3287 | "schemes": [
3288 | "https:\/\/*.youtube.com\/watch*",
3289 | "https:\/\/*.youtube.com\/v\/*",
3290 | "https:\/\/youtu.be\/*"
3291 | ],
3292 | "url": "https:\/\/www.youtube.com\/oembed",
3293 | "discovery": true
3294 | }
3295 | ]
3296 | },
3297 | {
3298 | "provider_name": "ZingSoft",
3299 | "provider_url": "https:\/\/app.zingsoft.com",
3300 | "endpoints": [
3301 | {
3302 | "schemes": [
3303 | "https:\/\/app.zingsoft.com\/embed\/*",
3304 | "https:\/\/app.zingsoft.com\/view\/*"
3305 | ],
3306 | "url": "https:\/\/app.zingsoft.com\/oembed",
3307 | "discovery": true
3308 | }
3309 | ]
3310 | },
3311 | {
3312 | "provider_name": "ZnipeTV",
3313 | "provider_url": "https:\/\/www.znipe.tv\/",
3314 | "endpoints": [
3315 | {
3316 | "schemes": [
3317 | "https:\/\/*.znipe.tv\/*"
3318 | ],
3319 | "url": "https:\/\/api.znipe.tv\/v3\/oembed\/",
3320 | "discovery": true
3321 | }
3322 | ]
3323 | },
3324 | {
3325 | "provider_name": "Zoomable",
3326 | "provider_url": "https:\/\/zoomable.ca\/",
3327 | "endpoints": [
3328 | {
3329 | "schemes": [
3330 | "https:\/\/srv2.zoomable.ca\/viewer.php*"
3331 | ],
3332 | "url": "https:\/\/srv2.zoomable.ca\/oembed",
3333 | "discovery": true
3334 | }
3335 | ]
3336 | }
3337 | ]
--------------------------------------------------------------------------------
/favicon.go:
--------------------------------------------------------------------------------
1 | package unfurlist
2 |
3 | import (
4 | "bytes"
5 |
6 | "golang.org/x/net/html"
7 | "golang.org/x/net/html/atom"
8 | "golang.org/x/net/html/charset"
9 | )
10 |
11 | // extractFaviconLink parses html data in search of the first element and returns value of its href attribute.
13 | func extractFaviconLink(htmlBody []byte, ct string) string {
14 | bodyReader, err := charset.NewReader(bytes.NewReader(htmlBody), ct)
15 | if err != nil {
16 | return ""
17 | }
18 | z := html.NewTokenizer(bodyReader)
19 | tokenize:
20 | for {
21 | tt := z.Next()
22 | switch tt {
23 | case html.ErrorToken:
24 | return ""
25 | case html.StartTagToken:
26 | name, hasAttr := z.TagName()
27 | switch atom.Lookup(name) {
28 | case atom.Body:
29 | return ""
30 | case atom.Link:
31 | var href string
32 | var isIconLink bool
33 | for hasAttr {
34 | var k, v []byte
35 | k, v, hasAttr = z.TagAttr()
36 | switch string(k) {
37 | case "rel":
38 | if !bytes.EqualFold(v, []byte("icon")) {
39 | continue tokenize
40 | }
41 | isIconLink = true
42 | case "href":
43 | href = string(v)
44 | }
45 | }
46 | if isIconLink && href != "" {
47 | return href
48 | }
49 | }
50 | }
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/favicon_test.go:
--------------------------------------------------------------------------------
1 | package unfurlist
2 |
3 | import "testing"
4 |
5 | func Test_extractFaviconLink(t *testing.T) {
6 | table := []struct{ input, want string }{
7 | {`foo`, ""},
8 | {`foo`,
9 | "https://example.com/favicon.ico"},
10 | }
11 | for i, tt := range table {
12 | got := extractFaviconLink([]byte(tt.input), "text/html")
13 | if got != tt.want {
14 | t.Errorf("case %d failed:\n got: %q,\nwant: %q,\ninput is:\n%s", i, got, tt.want, tt.input)
15 | }
16 | }
17 | }
18 |
--------------------------------------------------------------------------------
/fetcher.go:
--------------------------------------------------------------------------------
1 | package unfurlist
2 |
3 | import (
4 | "context"
5 | "net/http"
6 | "net/url"
7 | )
8 |
9 | // FetchFunc defines custom metadata fetchers that can be attached to unfurl
10 | // handler
11 | type FetchFunc func(context.Context, *http.Client, *url.URL) (*Metadata, bool)
12 |
13 | // Metadata represents metadata retrieved by FetchFunc. At least one of Title,
14 | // Description or Image attributes are expected to be non-empty.
15 | type Metadata struct {
16 | Title string
17 | Type string // TODO: make this int8 w/enum constants
18 | Description string
19 | Image string // image/thumbnail url
20 | ImageWidth int
21 | ImageHeight int
22 | }
23 |
24 | // Valid check that at least one of the mandatory attributes is non-empty
25 | func (m *Metadata) Valid() bool {
26 | return m != nil && (m.Title != "" || m.Description != "" || m.Image != "")
27 | }
28 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/Doist/unfurlist
2 |
3 | require (
4 | github.com/artyom/autoflags v1.1.1
5 | github.com/artyom/httpflags v1.2.0
6 | github.com/artyom/oembed v1.0.1
7 | github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d
8 | github.com/dyatlov/go-opengraph v0.0.0-20210112100619-dae8665a5b09
9 | github.com/golang/snappy v0.0.4
10 | golang.org/x/net v0.39.0
11 | golang.org/x/sync v0.13.0
12 | rsc.io/markdown v0.0.0-20241212154241-6bf72452917f
13 | )
14 |
15 | require (
16 | github.com/gobwas/glob v0.2.3 // indirect
17 | golang.org/x/text v0.24.0 // indirect
18 | )
19 |
20 | go 1.24.2
21 |
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | github.com/artyom/autoflags v1.1.1 h1:8flRmpb7xpjLHFVcM+HN+cEEKLw+H5a2hABDbRvfG9A=
2 | github.com/artyom/autoflags v1.1.1/go.mod h1:Th9KgAVvFcYp7t8b//Pu21xHjExLpzr4SXCbwVbHL7Y=
3 | github.com/artyom/httpflags v1.2.0 h1:Eyx41cC1Z3dR/j68aHBEzpGT49cwzOOJix3hd/lk5ms=
4 | github.com/artyom/httpflags v1.2.0/go.mod h1:f85rgrdm0w+DDDVbSKJXRTQ9rBho16YUt2mwEHLb8OA=
5 | github.com/artyom/oembed v1.0.1 h1:YxOT7S6mtJd/AiRxhgR205u3KMaXqOPdvjFHWOhYrVs=
6 | github.com/artyom/oembed v1.0.1/go.mod h1:7ga4NCmPl7HCnjjNeUqCQiqyrYM3vhF/f6d5pXkpNNo=
7 | github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d h1:pVrfxiGfwelyab6n21ZBkbkmbevaf+WvMIiR7sr97hw=
8 | github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA=
9 | github.com/dyatlov/go-opengraph v0.0.0-20210112100619-dae8665a5b09 h1:AQLr//nh20BzN3hIWj2+/Gt3FwSs8Nwo/nz4hMIcLPg=
10 | github.com/dyatlov/go-opengraph v0.0.0-20210112100619-dae8665a5b09/go.mod h1:nYia/MIs9OyvXXYboPmNOj0gVWo97Wx0sde+ZuKkoM4=
11 | github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
12 | github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
13 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
14 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
15 | github.com/yuin/goldmark v1.6.0 h1:boZcn2GTjpsynOsC0iJHnBWa4Bi0qzfJjthwauItG68=
16 | github.com/yuin/goldmark v1.6.0/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
17 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
18 | golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
19 | golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
20 | golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
21 | golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
22 | golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
23 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
24 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
25 | golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
26 | golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
27 | golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
28 | golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
29 | rsc.io/markdown v0.0.0-20241212154241-6bf72452917f h1:zQHn9vNRGvg+k5NdSMZ5jdSQcuz7k5niNMO0f0XkwKc=
30 | rsc.io/markdown v0.0.0-20241212154241-6bf72452917f/go.mod h1:dTYI7HoCsVAs6SKPMgkC2TV2xRFJB9WqcVydnnZby2Y=
31 |
--------------------------------------------------------------------------------
/googlemaps.go:
--------------------------------------------------------------------------------
1 | package unfurlist
2 |
3 | import (
4 | "context"
5 | "net/http"
6 | "net/url"
7 | "regexp"
8 | "strings"
9 | )
10 |
11 | // GoogleMapsFetcher returns FetchFunc that recognizes some Google Maps urls and
12 | // constructs metadata for them containing preview image from Google Static Maps
13 | // API. The only argument is the API key to create image links with.
14 | func GoogleMapsFetcher(key string) FetchFunc {
15 | if key == "" {
16 | return func(context.Context, *http.Client, *url.URL) (*Metadata, bool) { return nil, false }
17 | }
18 | return func(_ context.Context, _ *http.Client, u *url.URL) (*Metadata, bool) {
19 | if u == nil {
20 | return nil, false
21 | }
22 | if idx := strings.LastIndexByte(u.Host, '.'); idx == -1 ||
23 | !(strings.HasSuffix(u.Host[:idx], ".google") &&
24 | strings.HasPrefix(u.Path, "/maps")) {
25 | return nil, false
26 | }
27 | if u.Path == "/maps/api/staticmap" {
28 | return &Metadata{Image: u.String(), Type: "image"}, true
29 | }
30 | g := &url.URL{
31 | Scheme: "https",
32 | Host: "maps.googleapis.com",
33 | Path: "/maps/api/staticmap",
34 | }
35 | vals := make(url.Values)
36 | vals.Set("key", key)
37 | vals.Set("zoom", "16")
38 | vals.Set("size", "640x480")
39 | vals.Set("scale", "2")
40 | if q := u.Query().Get("q"); u.Path == "/maps" && q != "" {
41 | if zoom := u.Query().Get("z"); zoom != "" {
42 | vals.Set("zoom", zoom)
43 | }
44 | vals.Set("markers", "color:red|"+q)
45 | g.RawQuery = vals.Encode()
46 | return &Metadata{
47 | Type: "website",
48 | Image: g.String(),
49 | ImageWidth: 640 * 2,
50 | ImageHeight: 480 * 2,
51 | }, true
52 | }
53 | name, coords, zoom, ok := coordsFromPath(u.Path)
54 | if !ok {
55 | return &Metadata{Title: "Google Maps", Type: "website"}, true
56 | }
57 | vals.Set("zoom", zoom)
58 | vals.Set("markers", "color:red|"+coords)
59 | g.RawQuery = vals.Encode()
60 | return &Metadata{
61 | Title: name,
62 | Type: "website",
63 | Image: g.String(),
64 | ImageWidth: 640 * 2,
65 | ImageHeight: 480 * 2,
66 | }, true
67 | }
68 | }
69 |
70 | var googlePlace = regexp.MustCompile(`^/maps/place/(?P[^/]+)/@(?P[0-9.-]+,[0-9.-]+),(?P[0-9.]+)z`)
71 |
72 | // coordsFromPath extracts name, coordinates and zoom level from urls of the
73 | // following format:
74 | // https://www.google.com/maps/place/Passeig+de+Gràcia,+Barcelona,+Spain/@41.3931702,2.1617715,17z
75 | func coordsFromPath(p string) (name, coords, zoom string, ok bool) {
76 | ix := googlePlace.FindStringSubmatchIndex(p)
77 | // 4*2 is len(googlePlace.SubexpNames())*2
78 | if ix == nil || len(ix) != 4*2 {
79 | return "", "", "", false
80 | }
81 | name = p[ix[1*2]:ix[1*2+1]]
82 | coords = p[ix[2*2]:ix[2*2+1]]
83 | zoom = p[ix[3*2]:ix[3*2+1]]
84 | // normally p is already unescaped URL.Path, but it still has spaces
85 | // presented as +, this unescapes them
86 | if name, err := url.QueryUnescape(name); err == nil {
87 | return name, coords, zoom, true
88 | }
89 | return name, coords, zoom, true
90 | }
91 |
--------------------------------------------------------------------------------
/googlemaps_test.go:
--------------------------------------------------------------------------------
1 | package unfurlist
2 |
3 | import (
4 | "net/url"
5 | "testing"
6 | )
7 |
8 | func TestCoordsFromPath(t *testing.T) {
9 | testCases := []struct {
10 | input string
11 | name string
12 | coords string
13 | zoom string
14 | ok bool
15 | }{
16 | {"https://maps.google.com/maps/place/The+Manufacturing+Technology+Centre+(MTC)/@52.430763,-1.403385,16z/data=foo+bar",
17 | "The Manufacturing Technology Centre (MTC)",
18 | "52.430763,-1.403385", "16", true},
19 | {"https://www.google.com/maps/place/36%C2%B005'06.7%22N+5%C2%B030'49.6%22W/@36.0856728,-5.5169964,16z/data=",
20 | `36°05'06.7"N 5°30'49.6"W`, "36.0856728,-5.5169964", "16", true},
21 | }
22 | for _, tc := range testCases {
23 | u, err := url.Parse(tc.input)
24 | if err != nil {
25 | t.Fatal(err)
26 | }
27 | name, coords, zoom, ok := coordsFromPath(u.Path)
28 | if ok != tc.ok || name != tc.name || coords != tc.coords || zoom != tc.zoom {
29 | t.Fatalf("wrong result for input %q: got name:%q, coords:%q, zoom:%q, ok:%v", tc.input, name, coords, zoom, ok)
30 | }
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/html_meta_parser.go:
--------------------------------------------------------------------------------
1 | // Implements a basic HTML parser that just checks
2 | // It also annotates mime Type if possible
3 |
4 | package unfurlist
5 |
6 | import (
7 | "bytes"
8 | "errors"
9 | "io"
10 | "net/http"
11 | "strings"
12 |
13 | "golang.org/x/net/html"
14 | "golang.org/x/net/html/atom"
15 | "golang.org/x/net/html/charset"
16 | )
17 |
18 | func basicParseHTML(chunk *pageChunk) *unfurlResult {
19 | result := new(unfurlResult)
20 | sniffedContentType := http.DetectContentType(chunk.data)
21 | result.Type = sniffedContentType
22 | switch {
23 | case strings.HasPrefix(result.Type, "image/"):
24 | result.Type = "image"
25 | result.Image = chunk.url.String()
26 | case strings.HasPrefix(result.Type, "text/"):
27 | result.Type = "website"
28 | // pass Content-Type from response headers as it may have
29 | // charset definition like "text/html; charset=windows-1251"
30 | ct := chunk.ct
31 | // There are cases where Content-Type header is "text/html", but http.DetectContentType
32 | // narrows it down to a more specific "text/html; charset=utf-8". In such a case use
33 | // the latter.
34 | if !strings.Contains(ct, "charset=") && strings.Contains(sniffedContentType, "charset=") {
35 | ct = sniffedContentType
36 | }
37 | if title, desc, err := extractData(chunk.data, ct); err == nil {
38 | result.Title = title
39 | result.Description = desc
40 | }
41 | case strings.HasPrefix(result.Type, "video/"):
42 | result.Type = "video"
43 | }
44 | return result
45 | }
46 |
47 | func extractData(htmlBody []byte, ct string) (title, description string, err error) {
48 | bodyReader, err := charset.NewReader(bytes.NewReader(htmlBody), ct)
49 | if err != nil {
50 | return "", "", err
51 | }
52 | z := html.NewTokenizer(bodyReader)
53 | tokenize:
54 | for {
55 | tt := z.Next()
56 | switch tt {
57 | case html.ErrorToken:
58 | if z.Err() == io.EOF {
59 | goto finish
60 | }
61 | return "", "", z.Err()
62 | case html.StartTagToken:
63 | name, hasAttr := z.TagName()
64 | switch atom.Lookup(name) {
65 | case atom.Body:
66 | goto finish // title/meta should preceed body tag
67 | case atom.Title:
68 | if title != "" {
69 | continue
70 | }
71 | if tt := z.Next(); tt == html.TextToken {
72 | title = string(z.Text())
73 | if description != "" {
74 | goto finish
75 | }
76 | }
77 | case atom.Meta:
78 | if description != "" {
79 | continue
80 | }
81 | var content []byte
82 | var isDescription bool
83 | for hasAttr {
84 | var k, v []byte
85 | k, v, hasAttr = z.TagAttr()
86 | switch string(k) {
87 | case "name":
88 | if !bytes.Equal(v, []byte("description")) {
89 | continue tokenize
90 | }
91 | isDescription = true
92 | case "content":
93 | content = v
94 | }
95 | }
96 | if isDescription && len(content) > 0 {
97 | description = string(content)
98 | if title != "" {
99 | goto finish
100 | }
101 | }
102 | }
103 | }
104 | }
105 | finish:
106 | if title != "" || description != "" {
107 | return title, description, nil
108 | }
109 | return "", "", errNoMetadataFound
110 | }
111 |
112 | var (
113 | errNoMetadataFound = errors.New("no metadata found")
114 | )
115 |
--------------------------------------------------------------------------------
/html_meta_parser_test.go:
--------------------------------------------------------------------------------
1 | package unfurlist
2 |
3 | import (
4 | "os"
5 | "testing"
6 | )
7 |
8 | func TestExtractData_explicitCharset(t *testing.T) {
9 | // this file has its charset defined at around ~1600 bytes, WHATWG
10 | // charset detection algorithm [1] fails here as it only scans first
11 | // 1024 bytes, so we also need to rely on server-provided charset
12 | // parameter from Content-Type header
13 | //
14 | // [1]: https://html.spec.whatwg.org/multipage/syntax.html#determining-the-character-encoding
15 | data, err := os.ReadFile("testdata/no-charset-in-first-1024bytes")
16 | if err != nil {
17 | t.Fatal(err)
18 | }
19 | title, _, err := extractData(data, "text/html; charset=windows-1251")
20 | if err != nil {
21 | t.Fatal(err)
22 | }
23 | want := `Кубань и Адыгея объединят усилия по созданию курорта "Лагонаки"`
24 | if title != want {
25 | t.Fatalf("unexpected title: got %q, want %q", title, want)
26 | }
27 | }
28 |
29 | func TestExtractData_multibyte1(t *testing.T) {
30 | data, err := os.ReadFile("testdata/korean")
31 | if err != nil {
32 | t.Fatal(err)
33 | }
34 | title, _, err := extractData(data, "text/html")
35 | if err != nil {
36 | t.Fatal(err)
37 | }
38 | want := `심장정지 환자 못살리는 119 구급차 - 1등 인터넷뉴스 조선닷컴 - 의료ㆍ보건`
39 | if title != want {
40 | t.Fatalf("unexpected title: got %q, want %q", title, want)
41 | }
42 | }
43 |
44 | func TestExtractData_multibyte2(t *testing.T) {
45 | data, err := os.ReadFile("testdata/japanese")
46 | if err != nil {
47 | t.Fatal(err)
48 | }
49 | title, _, err := extractData(data, "text/html")
50 | if err != nil {
51 | t.Fatal(err)
52 | }
53 | want := `【楽天市場】テレビ台【ALTER／アルター】コーナータイプ【ＴＶ台】薄型ＴＶ３７型対応 ＡＶ収納【ＡＶボード】【コーナーボード】【幅１００】◆代引不可★一部組立【駅伝_中_四】：インテリア雑貨通販 H-collection`
54 | if title != want {
55 | t.Fatalf("unexpected title: got %q, want %q", title, want)
56 | }
57 | }
58 |
59 | func TestExtractData(t *testing.T) {
60 | for i, c := range titleTestCases {
61 | title, _, err := extractData([]byte(c.body), "text/html")
62 | if err != nil {
63 | t.Errorf("case %d failed: %v", i, err)
64 | continue
65 | }
66 | if title != c.want {
67 | t.Errorf("case %d mismatch: %q != %q", i, title, c.want)
68 | }
69 | }
70 | }
71 |
72 | func TestExtractData_full(t *testing.T) {
73 | body := `<html>
74 | <meta name="keywords" content="test">
75 | <meta name="description" content="hello page">
76 | <meta name="description" content="ignored">
77 | <title>Hello
78 |
79 | `
80 | title, desc, err := extractData([]byte(body), "text/html")
81 | if err != nil {
82 | t.Fatal(err)
83 | }
84 | if want := "Hello"; title != want {
85 | t.Errorf("got title %q, want %q", title, want)
86 | }
87 | if want := "hello page"; desc != want {
88 | t.Errorf("got description %q, want %q", desc, want)
89 | }
90 | }
91 |
92 | func BenchmarkExtractData(b *testing.B) {
93 | for b.Loop() {
94 | for i, c := range titleTestCases {
95 | title, _, err := extractData([]byte(c.body), "text/html")
96 | if err != nil {
97 | b.Fatalf("case %d failed: %v", i, err)
98 | }
99 | if title != c.want {
100 | b.Fatalf("case %d mismatch: %q != %q", i, title, c.want)
101 | }
102 | }
103 | }
104 | }
105 |
106 | var titleTestCases = []struct {
107 | body string
108 | want string
109 | }{
110 | {"Hello", "Hello"},
111 | {"Hello", "Hello"},
112 | {"Hello\n", "Hello\n"},
113 | }
114 |
--------------------------------------------------------------------------------
/image.go:
--------------------------------------------------------------------------------
1 | package unfurlist
2 |
3 | import (
4 | "context"
5 | "errors"
6 | "fmt"
7 | "image"
8 | _ "image/gif" // register supported image types
9 | _ "image/jpeg"
10 | _ "image/png"
11 | "net/http"
12 | "net/url"
13 | "strings"
14 | )
15 |
16 | var errEmptyImageURL = errors.New("empty image url")
17 |
18 | // absoluteImageUrl makes imageUrl absolute if it's not. Image url can either be
19 | // relative or schemaless url.
20 | func absoluteImageURL(originURL, imageURL string) (string, error) {
21 | if imageURL == "" {
22 | return "", errEmptyImageURL
23 | }
24 | if strings.HasPrefix(imageURL, "https") {
25 | return imageURL, nil
26 | }
27 | iu, err := url.Parse(imageURL)
28 | if err != nil {
29 | return "", err
30 | }
31 | switch iu.Scheme {
32 | case "https", "":
33 | default:
34 | return "", fmt.Errorf("unsupported url scheme %q", iu.Scheme)
35 | }
36 | base, err := url.Parse(originURL)
37 | if err != nil {
38 | return "", err
39 | }
40 | return base.ResolveReference(iu).String(), nil
41 | }
42 |
43 | // imageDimensions tries to retrieve enough of image to get its dimensions. If
44 | // provided client is nil, http.DefaultClient is used.
45 | func imageDimensions(ctx context.Context, client *http.Client, imageURL string) (width, height int, err error) {
46 | cl := client
47 | if cl == nil {
48 | cl = http.DefaultClient
49 | }
50 | req, err := http.NewRequest(http.MethodGet, imageURL, nil)
51 | if err != nil {
52 | return 0, 0, err
53 | }
54 | req = req.WithContext(ctx)
55 | resp, err := cl.Do(req)
56 | if err != nil {
57 | return 0, 0, err
58 | }
59 | defer resp.Body.Close()
60 |
61 | if resp.StatusCode >= http.StatusBadRequest {
62 | return 0, 0, errors.New(resp.Status)
63 | }
64 | switch ct := strings.ToLower(resp.Header.Get("Content-Type")); ct {
65 | case "image/jpeg", "image/png", "image/gif":
66 | default:
67 | // for broken servers responding with image/png;charset=UTF-8
68 | // (i.e. www.evernote.com)
69 | if strings.HasPrefix(ct, "image/jpeg") ||
70 | strings.HasPrefix(ct, "image/png") ||
71 | strings.HasPrefix(ct, "image/gif") {
72 | break
73 | }
74 | return 0, 0, fmt.Errorf("unsupported content-type %q", ct)
75 | }
76 | cfg, _, err := image.DecodeConfig(resp.Body)
77 | if err != nil {
78 | return 0, 0, err
79 | }
80 | return cfg.Width, cfg.Height, nil
81 | }
82 |
--------------------------------------------------------------------------------
/internal/useragent/LICENSE.txt:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2017 Artyom Pervukhin
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/internal/useragent/useragent.go:
--------------------------------------------------------------------------------
1 | // This is a vendored copy of https://github.com/artyom/useragent
2 | //
3 | // Package useragent provides http.RoundTripper wrapper to set User-Agent header
4 | // on each http request made.
5 | //
6 | // Basic usage:
7 | //
8 | // client := &http.Client{
9 | // Transport: useragent.Set(http.DefaultTransport, "MyRobot/1.0"),
10 | // }
11 | // resp, err := client.Get("https://...")
12 | package useragent
13 |
14 | import (
15 | "net/http"
16 | "strings"
17 | )
18 |
19 | // Set wraps provided http.RoundTripper returning a new one that adds given
20 | // agent as User-Agent header for requests without such header or with empty
21 | // User-Agent header.
22 | //
23 | // If rt is a *http.Transport, the returned RoundTripper would have Transport's
24 | // methods visible so they can be accessed after type assertion to required
25 | // interface.
26 | func Set(rt http.RoundTripper, agent string) http.RoundTripper {
27 | if agent == "" {
28 | return rt
29 | }
30 | if t, ok := rt.(*http.Transport); ok {
31 | return uaT{t, agent}
32 | }
33 | return uaRT{rt, agent}
34 | }
35 |
36 | type uaT struct {
37 | *http.Transport
38 | userAgent string
39 | }
40 |
41 | func (t uaT) RoundTrip(r *http.Request) (*http.Response, error) {
42 | if _, ok := r.Header["User-Agent"]; ok {
43 | return t.Transport.RoundTrip(r)
44 | }
45 | r2 := new(http.Request)
46 | *r2 = *r
47 | r2.Header = make(http.Header, len(r.Header)+1)
48 | for k, v := range r.Header {
49 | r2.Header[k] = v
50 | }
51 | r2.Header.Set("User-Agent", t.userAgent)
52 | if r.URL.Host == "twitter.com" || strings.HasSuffix(r.URL.Host, ".twitter.com") ||
53 | r.URL.Host == "x.com" || strings.HasSuffix(r.URL.Host, ".x.com") {
54 | r2.Header.Set("User-Agent", "DiscourseBot/1.0")
55 | }
56 | return t.Transport.RoundTrip(r2)
57 | }
58 |
59 | type uaRT struct {
60 | http.RoundTripper
61 | userAgent string
62 | }
63 |
64 | func (t uaRT) RoundTrip(r *http.Request) (*http.Response, error) {
65 | if _, ok := r.Header["User-Agent"]; ok {
66 | return t.RoundTripper.RoundTrip(r)
67 | }
68 | r2 := new(http.Request)
69 | *r2 = *r
70 | r2.Header = make(http.Header, len(r.Header)+1)
71 | for k, v := range r.Header {
72 | r2.Header[k] = v
73 | }
74 | r2.Header.Set("User-Agent", t.userAgent)
75 | return t.RoundTripper.RoundTrip(r2)
76 | }
77 |
--------------------------------------------------------------------------------
/oembed_parser.go:
--------------------------------------------------------------------------------
1 | package unfurlist
2 |
3 | import (
4 | "context"
5 | "net/http"
6 |
7 | "github.com/artyom/oembed"
8 | )
9 |
10 | func fetchOembed(ctx context.Context, url string, fn func(context.Context, string) (*http.Response, error)) (*unfurlResult, error) {
11 | resp, err := fn(ctx, url)
12 | if err != nil {
13 | return nil, err
14 | }
15 | defer resp.Body.Close()
16 | meta, err := oembed.FromResponse(resp)
17 | if err != nil {
18 | return nil, err
19 | }
20 | res := &unfurlResult{
21 | Title: meta.Title,
22 | SiteName: meta.Provider,
23 | Type: string(meta.Type),
24 | HTML: meta.HTML,
25 | Image: meta.Thumbnail,
26 | }
27 | if meta.Type == oembed.TypePhoto && meta.URL != "" {
28 | res.Image = meta.URL
29 | }
30 | return res, nil
31 | }
32 |
--------------------------------------------------------------------------------
/opengraph_parser.go:
--------------------------------------------------------------------------------
1 | // Implements the basic Open Graph parser ( http://ogp.me/ )
2 | // Currently we only parse Title, Description, Type and the first Image
3 |
4 | package unfurlist
5 |
6 | import (
7 | "bytes"
8 | "net/http"
9 | "strings"
10 |
11 | "golang.org/x/net/html/charset"
12 |
13 | "github.com/dyatlov/go-opengraph/opengraph"
14 | )
15 |
16 | func openGraphParseHTML(chunk *pageChunk) *unfurlResult {
17 | if !strings.HasPrefix(http.DetectContentType(chunk.data), "text/html") {
18 | return nil
19 | }
20 | // use explicit content type received from headers here but not the one returned by
21 | // http.DetectContentType because this function scans only first 512
22 | // bytes and can report content as "text/html; charset=utf-8" even for
23 | // bodies having characters outside utf8 range later; use
24 | // charset.NewReader that relies on charset.DetermineEncoding which
25 | // implements more elaborate encoding detection specific to html content
26 | bodyReader, err := charset.NewReader(bytes.NewReader(chunk.data), chunk.ct)
27 | if err != nil {
28 | return nil
29 | }
30 | og := opengraph.NewOpenGraph()
31 | err = og.ProcessHTML(bodyReader)
32 | if err != nil || og.Title == "" {
33 | return nil
34 | }
35 | res := &unfurlResult{
36 | Type: og.Type,
37 | Title: og.Title,
38 | Description: og.Description,
39 | SiteName: og.SiteName,
40 | }
41 | if len(og.Images) > 0 {
42 | res.Image = og.Images[0].URL
43 | }
44 | if chunk.url.Host == "twitter.com" &&
45 | strings.Contains(chunk.url.Path, "/status/") &&
46 | !bytes.Contains(chunk.data, []byte(`property="og:image:user_generated" content="true"`)) {
47 | res.Image = ""
48 | }
49 | return res
50 | }
51 |
--------------------------------------------------------------------------------
/prefixmap.go:
--------------------------------------------------------------------------------
1 | package unfurlist
2 |
3 | import "sort"
4 |
5 | // prefixMap allows fast checks against predefined set of prefixes.
6 | // Uninitialized/empty prefixMap considers any string as matching.
7 | type prefixMap struct {
8 | prefixes map[string]struct{}
9 | lengths []int // sorted, smaller first
10 | }
11 |
12 | // newPrefixMap initializes new prefixMap from given slice of prefixes
13 | func newPrefixMap(prefixes []string) *prefixMap {
14 | if len(prefixes) == 0 {
15 | return nil
16 | }
17 | m := make(map[string]struct{}, len(prefixes))
18 | l1 := make([]int, 0, len(prefixes))
19 | for _, p := range prefixes {
20 | if p == "" {
21 | continue
22 | }
23 | m[p] = struct{}{}
24 | l1 = append(l1, len(p))
25 | }
26 | if len(l1) == 0 {
27 | return nil
28 | }
29 | sort.Ints(l1)
30 | // remove duplicates
31 | l2 := l1[:1]
32 | for i := 1; i < len(l1); i++ {
33 | if l1[i] != l1[i-1] {
34 | l2 = append(l2, l1[i])
35 | }
36 | }
37 | return &prefixMap{
38 | prefixes: m,
39 | lengths: l2,
40 | }
41 | }
42 |
43 | // Match validates string against set of prefixes. It returns true only if
44 | // prefixMap is non-empty and string matches at least one prefix.
45 | func (m *prefixMap) Match(url string) bool {
46 | if m == nil || m.prefixes == nil {
47 | return false
48 | }
49 | sLen := len(url)
50 | if sLen < m.lengths[0] {
51 | return false
52 | }
53 | for _, x := range m.lengths {
54 | if sLen < x {
55 | continue
56 | }
57 | if _, ok := m.prefixes[url[:x]]; ok {
58 | return true
59 | }
60 | }
61 | return false
62 | }
63 |
--------------------------------------------------------------------------------
/prefixmap_test.go:
--------------------------------------------------------------------------------
1 | package unfurlist
2 |
3 | import "fmt"
4 |
5 | func ExampleprefixMap() {
6 | pm := newPrefixMap([]string{"https://mail.google.com/mail/", "https://trello.com/c/"})
7 |
8 | urls := []string{
9 | "http://example.com/index.html",
10 | "https://mail.google.com/mail/u/0/#inbox",
11 | "https://trello.com/c/a12def34",
12 | }
13 | for _, u := range urls {
14 | fmt.Printf("%q\t%v\n", u, pm.Match(u))
15 | }
16 | // Output:
17 | // "http://example.com/index.html" false
18 | // "https://mail.google.com/mail/u/0/#inbox" true
19 | // "https://trello.com/c/a12def34" true
20 | }
21 |
--------------------------------------------------------------------------------
/remote-data-update.go:
--------------------------------------------------------------------------------
1 | //go:build ignore
2 | // +build ignore
3 |
4 | package main
5 |
6 | import (
7 | "encoding/json"
8 | "flag"
9 | "io"
10 | "log"
11 | "net"
12 | "net/http"
13 | "net/url"
14 | "os"
15 | "path/filepath"
16 | "time"
17 |
18 | "github.com/Doist/unfurlist/internal/useragent"
19 | )
20 |
21 | var urls = []string{
22 | "http://techcrunch.com/2015/11/09/basic-income-createathon/",
23 | "https://news.ycombinator.com/",
24 | "https://twitter.com/amix3k/status/1399300280206909440",
25 | "http://news.chosun.com/site/data/html_dir/2009/09/24/2009092401755.html",
26 | }
27 |
28 | func main() {
29 | outfile := filepath.FromSlash("testdata/remote-dump.json")
30 | flag.StringVar(&outfile, "f", outfile, "`file` to save data to (will be overwritten)")
31 | flag.Parse()
32 | if outfile == "" {
33 | log.Fatal("-f is empty")
34 | }
35 | data := make(map[string]string)
36 |
37 | httpClient := &http.Client{
38 | Timeout: 10 * time.Second,
39 | Transport: useragent.Set(&http.Transport{
40 | Proxy: http.ProxyFromEnvironment,
41 | DialContext: (&net.Dialer{
42 | Timeout: 10 * time.Second,
43 | KeepAlive: 30 * time.Second,
44 | DualStack: true,
45 | }).DialContext,
46 | MaxIdleConns: 100,
47 | IdleConnTimeout: 90 * time.Second,
48 | TLSHandshakeTimeout: 10 * time.Second,
49 | ExpectContinueTimeout: 1 * time.Second,
50 | }, "unfurlist (https://github.com/Doist/unfurlist)"),
51 | }
52 |
53 | for _, v := range urls {
54 | u, err := url.Parse(v)
55 | if err != nil {
56 | log.Fatal(v, err)
57 | }
58 | req, err := http.NewRequest(http.MethodGet, v, nil)
59 | if err != nil {
60 | log.Fatal(v, err)
61 | }
62 | r, err := httpClient.Do(req)
63 | if err != nil {
64 | log.Fatal(v, err)
65 | }
66 | if r.StatusCode >= 400 {
67 | log.Fatal(v, r.Status)
68 | }
69 | b, err := io.ReadAll(r.Body)
70 | if err != nil {
71 | log.Fatal(v, err)
72 | }
73 | r.Body.Close()
74 | // store key without scheme
75 | data[u.Host+u.RequestURI()] = string(b)
76 | }
77 | if err := dump(data, outfile); err != nil {
78 | log.Fatal(err)
79 | }
80 | }
81 |
82 | func dump(data map[string]string, name string) error {
83 | f, err := os.Create(name)
84 | if err != nil {
85 | return err
86 | }
87 | defer f.Close()
88 | enc := json.NewEncoder(f)
89 | enc.SetIndent("", " ")
90 | enc.SetEscapeHTML(false)
91 | if err := enc.Encode(data); err != nil {
92 | return err
93 | }
94 | return f.Close()
95 | }
96 |
--------------------------------------------------------------------------------
/testdata/japanese:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Doist/unfurlist/515f2735f8e5b1f0cc9694bee80da2e86d25b7ca/testdata/japanese
--------------------------------------------------------------------------------
/testdata/korean:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Doist/unfurlist/515f2735f8e5b1f0cc9694bee80da2e86d25b7ca/testdata/korean
--------------------------------------------------------------------------------
/testdata/no-charset-in-first-1024bytes:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Doist/unfurlist/515f2735f8e5b1f0cc9694bee80da2e86d25b7ca/testdata/no-charset-in-first-1024bytes
--------------------------------------------------------------------------------
/unfurlist.go:
--------------------------------------------------------------------------------
1 | // Package unfurlist implements a service that unfurls URLs and provides more information about them.
2 | //
3 | // The current version supports Open Graph and oEmbed formats, Twitter card format is also planned.
4 | // If the URL does not support common formats, unfurlist falls back to looking at common HTML tags
5 | // such as and <meta name="description">.
6 | //
7 | // The endpoint accepts GET and POST requests with `content` as the main argument.
8 | // It then returns a JSON encoded list of URLs that were parsed.
9 | //
10 | // If an URL lacks an attribute (e.g. `image`) then this attribute will be omitted from the result.
11 | //
12 | // Example:
13 | //
14 | // ?content=Check+this+out+https://www.youtube.com/watch?v=dQw4w9WgXcQ
15 | //
16 | // Will return:
17 | //
18 | // Type: "application/json"
19 | //
20 | // [
21 | // {
22 | // "url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
23 | // "title": "Rick Astley - Never Gonna Give You Up (Video)",
24 | // "url_type": "video.other",
25 | // "description": "Rick Astley - Never Gonna Give You Up...",
26 | // "site_name": "YouTube",
27 | // "favicon": "https://www.youtube.com/yts/img/favicon_32-vflOogEID.png",
28 | // "image": "https://i.ytimg.com/vi/dQw4w9WgXcQ/maxresdefault.jpg"
29 | // }
30 | // ]
31 | //
32 | // If handler was configured with FetchImageSize=true in its config, each hash
33 | // may have additional fields `image_width` and `image_height` specifying
34 | // dimensions of image provided by `image` attribute.
35 | //
36 | // Additionally you can supply `callback` to wrap the result in a JavaScript callback (JSONP),
37 | // the type of this response would be "application/x-javascript"
38 | //
39 | // If an optional `markdown` boolean argument is set (markdown=true), then
40 | // provided content is parsed as markdown formatted text and links are extracted
41 | // in context-aware mode — i.e. preformatted text blocks are skipped.
42 | //
43 | // # Security
44 | //
45 | // Care should be taken when running this inside internal network since it may
46 | // disclose internal endpoints. It is a good idea to run the service on
47 | // a separate host in an isolated subnet.
48 | //
49 | // Alternatively access to internal resources may be limited with firewall
50 | // rules, i.e. if service is running as 'unfurlist' user on linux box, the
51 | // following iptables rules can reduce chances of it connecting to internal
52 | // endpoints (note this example is for ipv4 only!):
53 | //
54 | // iptables -A OUTPUT -m owner --uid-owner unfurlist -p tcp --syn \
55 | // -d 127/8,10/8,169.254/16,172.16/12,192.168/16 \
56 | // -j REJECT --reject-with icmp-net-prohibited
57 | // ip6tables -A OUTPUT -m owner --uid-owner unfurlist -p tcp --syn \
58 | // -d ::1/128,fe80::/10 \
59 | // -j REJECT --reject-with adm-prohibited
60 | package unfurlist
61 |
62 | import (
63 | "bytes"
64 | "cmp"
65 | "compress/zlib"
66 | "context"
67 | "crypto/sha1"
68 | _ "embed"
69 | "encoding/hex"
70 | "encoding/json"
71 | "errors"
72 | "io"
73 | "log"
74 | "net/http"
75 | "net/url"
76 | "slices"
77 | "strings"
78 | "time"
79 |
80 | "golang.org/x/net/html/charset"
81 | "golang.org/x/sync/singleflight"
82 |
83 | "github.com/artyom/httpflags"
84 | "github.com/artyom/oembed"
85 | "github.com/bradfitz/gomemcache/memcache"
86 | "github.com/golang/snappy"
87 | )
88 |
89 | const defaultMaxBodyChunkSize = 1024 * 64 //64KB
90 |
91 | // DefaultMaxResults is maximum number of urls to process if not configured by
92 | // WithMaxResults function
93 | const DefaultMaxResults = 20
94 |
95 | type unfurlHandler struct {
96 | HTTPClient *http.Client
97 | Log Logger
98 | oembedLookupFunc oembed.LookupFunc
99 | Cache *memcache.Client
100 | MaxBodyChunkSize int64
101 | FetchImageSize bool
102 |
103 | // Headers specify key-value pairs of extra headers to add to each
104 | // outgoing request made by Handler. Headers length must be even,
105 | // otherwise Headers are ignored.
106 | Headers []string
107 |
108 | titleBlocklist []string
109 |
110 | pmap *prefixMap // built from BlocklistPrefix
111 |
112 | maxResults int // max number of urls to process
113 |
114 | fetchers []FetchFunc
115 | inFlight singleflight.Group // in-flight urls processed
116 | }
117 |
118 | // Result that's returned back to the client
119 | type unfurlResult struct {
120 | URL string `json:"url"`
121 | Title string `json:"title,omitempty"`
122 | Type string `json:"url_type,omitempty"`
123 | Description string `json:"description,omitempty"`
124 | HTML string `json:"html,omitempty"`
125 | SiteName string `json:"site_name,omitempty"`
126 | Favicon string `json:"favicon,omitempty"`
127 | Image string `json:"image,omitempty"`
128 | ImageWidth int `json:"image_width,omitempty"`
129 | ImageHeight int `json:"image_height,omitempty"`
130 |
131 | idx int
132 | }
133 |
134 | func (u *unfurlResult) Empty() bool {
135 | return u.URL == "" && u.Title == "" && u.Type == "" &&
136 | u.Description == "" && u.Image == ""
137 | }
138 |
139 | func (u *unfurlResult) normalize() {
140 | b := bytes.Join(bytes.Fields([]byte(u.Title)), []byte{' '})
141 | u.Title = string(b)
142 | }
143 |
144 | func (u *unfurlResult) Merge(u2 *unfurlResult) {
145 | if u2 == nil {
146 | return
147 | }
148 | if u.URL == "" {
149 | u.URL = u2.URL
150 | }
151 | if u.Title == "" {
152 | u.Title = u2.Title
153 | }
154 | if u.Type == "" {
155 | u.Type = u2.Type
156 | }
157 | if u.Description == "" {
158 | u.Description = u2.Description
159 | }
160 | if u.HTML == "" {
161 | u.HTML = u2.HTML
162 | }
163 | if u.SiteName == "" {
164 | u.SiteName = u2.SiteName
165 | }
166 | if u.Image == "" {
167 | u.Image = u2.Image
168 | }
169 | if u.ImageWidth == 0 {
170 | u.ImageWidth = u2.ImageWidth
171 | }
172 | if u.ImageHeight == 0 {
173 | u.ImageHeight = u2.ImageHeight
174 | }
175 | }
176 |
177 | // ConfFunc is used to configure new unfurl handler; such functions should be
178 | // used as arguments to New function
179 | type ConfFunc func(*unfurlHandler) *unfurlHandler
180 |
181 | // New returns new initialized unfurl handler. If no configuration functions
182 | // provided, sane defaults would be used.
183 | func New(conf ...ConfFunc) http.Handler {
184 | h := &unfurlHandler{
185 | maxResults: DefaultMaxResults,
186 | }
187 | for _, f := range conf {
188 | h = f(h)
189 | }
190 | if h.HTTPClient == nil {
191 | h.HTTPClient = http.DefaultClient
192 | }
193 | if len(h.Headers)%2 != 0 {
194 | h.Headers = nil
195 | }
196 | if h.MaxBodyChunkSize == 0 {
197 | h.MaxBodyChunkSize = defaultMaxBodyChunkSize
198 | }
199 | if h.Log == nil {
200 | h.Log = log.New(io.Discard, "", 0)
201 | }
202 | if h.oembedLookupFunc == nil {
203 | fn, err := oembed.Providers(bytes.NewReader(providersData))
204 | if err != nil {
205 | panic(err)
206 | }
207 | h.oembedLookupFunc = fn
208 | }
209 | return h
210 | }
211 |
212 | func (h *unfurlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
213 | switch r.Method {
214 | case http.MethodGet, http.MethodPost:
215 | default:
216 | w.Header().Set("Allow", "GET, POST")
217 | http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
218 | return
219 | }
220 | args := struct {
221 | Content string `flag:"content"`
222 | Callback string `flag:"callback"`
223 | Markdown bool `flag:"markdown"`
224 | }{}
225 | if err := httpflags.Parse(&args, r); err != nil || args.Content == "" {
226 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
227 | return
228 | }
229 |
230 | var urls []string
231 | switch {
232 | case args.Markdown:
233 | urls = parseMarkdownURLs(args.Content, h.maxResults)
234 | default:
235 | urls = parseURLsMax(args.Content, h.maxResults)
236 | }
237 |
238 | jobResults := make(chan *unfurlResult, 1)
239 | results := make([]*unfurlResult, 0, len(urls))
240 | ctx := r.Context()
241 |
242 | for i, r := range urls {
243 | go func(ctx context.Context, i int, link string, jobResults chan *unfurlResult) {
244 | select {
245 | case jobResults <- h.processURLidx(ctx, i, link):
246 | case <-ctx.Done():
247 | }
248 | }(ctx, i, r, jobResults)
249 | }
250 | for range urls {
251 | select {
252 | case <-ctx.Done():
253 | return
254 | case res := <-jobResults:
255 | results = append(results, res)
256 | }
257 | }
258 |
259 | slices.SortFunc(results, func(a, b *unfurlResult) int { return cmp.Compare(a.idx, b.idx) })
260 | for _, r := range results {
261 | r.normalize()
262 | }
263 |
264 | if args.Callback != "" {
265 | w.Header().Set("Content-Type", "application/x-javascript")
266 | w.Header().Set("Access-Control-Allow-Origin", "*")
267 | } else {
268 | w.Header().Set("Content-Type", "application/json")
269 | }
270 |
271 | if args.Callback != "" {
272 | io.WriteString(w, args.Callback+"(")
273 | json.NewEncoder(w).Encode(results)
274 | w.Write([]byte(")"))
275 | return
276 | }
277 | json.NewEncoder(w).Encode(results)
278 | }
279 |
280 | // processURLidx wraps processURL and adds provided index i to the result. It
281 | // also collapses multiple in-flight requests for the same url to a single
282 | // processURL call
283 | func (h *unfurlHandler) processURLidx(ctx context.Context, i int, link string) *unfurlResult {
284 | defer h.inFlight.Forget(link)
285 | v, _, shared := h.inFlight.Do(link, func() (any, error) { return h.processURL(ctx, link), nil })
286 | res, ok := v.(*unfurlResult)
287 | if !ok {
288 | panic("got unexpected type from singleflight.Do")
289 | }
290 | if shared && (*res == unfurlResult{URL: link}) && ctx.Err() == nil {
291 | // an *incomplete* shared result, e.g. if context in another goroutine
292 | // that called processURL was canceled early, need to refetch
293 | res = h.processURL(ctx, link)
294 | }
295 | res2 := *res // make a copy because we're going to modify it
296 | res2.idx = i
297 | return &res2
298 | }
299 |
300 | // Processes the URL by first looking in cache, then trying oEmbed, OpenGraph
301 | // If no match is found the result will be an object that just contains the URL
302 | func (h *unfurlHandler) processURL(ctx context.Context, link string) *unfurlResult {
303 | result := &unfurlResult{URL: link}
304 | if h.pmap != nil && h.pmap.Match(link) { // blocklisted
305 | h.Log.Printf("Blocklisted %q", link)
306 | return result
307 | }
308 |
309 | if mc := h.Cache; mc != nil {
310 | if it, err := mc.Get(mcKey(link)); err == nil {
311 | if b, err := snappy.Decode(nil, it.Value); err == nil {
312 | var cached unfurlResult
313 | if err = json.Unmarshal(b, &cached); err == nil {
314 | h.Log.Printf("Cache hit for %q", link)
315 | return &cached
316 | }
317 | }
318 | }
319 | }
320 | var chunk *pageChunk
321 | var err error
322 | // Optimistically apply oembed logic to url we have, which can only work
323 | // for non-minimized urls; however if it works, it'll let us skip fetching
324 | // url altogether. This can also somewhat help against sites redirecting to
325 | // captchas/login pages when they see requests from non "home ISP"
326 | // networks.
327 | if endpoint, ok := h.oembedLookupFunc(result.URL); ok {
328 | if res, err := fetchOembed(ctx, endpoint, h.httpGet); err == nil {
329 | result.Merge(res)
330 | goto hasMatch
331 | }
332 | }
333 | chunk, err = h.fetchData(ctx, result.URL)
334 | if err != nil {
335 | if chunk != nil && strings.Contains(chunk.url.Host, "youtube.com") {
336 | if meta, ok := youtubeFetcher(ctx, h.HTTPClient, chunk.url); ok && meta.Valid() {
337 | result.Title = meta.Title
338 | result.Type = meta.Type
339 | result.Description = meta.Description
340 | result.Image = meta.Image
341 | result.ImageWidth = meta.ImageWidth
342 | result.ImageHeight = meta.ImageHeight
343 | goto hasMatch
344 | }
345 | }
346 | return result
347 | }
348 | if s, err := h.faviconLookup(ctx, chunk); err == nil && s != "" {
349 | result.Favicon = s
350 | }
351 | for _, f := range h.fetchers {
352 | meta, ok := f(ctx, h.HTTPClient, chunk.url)
353 | if !ok || !meta.Valid() {
354 | continue
355 | }
356 | result.Title = meta.Title
357 | result.Type = meta.Type
358 | result.Description = meta.Description
359 | result.Image = meta.Image
360 | result.ImageWidth = meta.ImageWidth
361 | result.ImageHeight = meta.ImageHeight
362 | goto hasMatch
363 | }
364 |
365 | if res := openGraphParseHTML(chunk); res != nil {
366 | if !blocklisted(h.titleBlocklist, res.Title) {
367 | result.Merge(res)
368 | goto hasMatch
369 | }
370 | }
371 | if endpoint, found := chunk.oembedEndpoint(h.oembedLookupFunc); found {
372 | if res, err := fetchOembed(ctx, endpoint, h.httpGet); err == nil {
373 | result.Merge(res)
374 | goto hasMatch
375 | }
376 | }
377 | if res := basicParseHTML(chunk); res != nil {
378 | if !blocklisted(h.titleBlocklist, res.Title) {
379 | result.Merge(res)
380 | }
381 | }
382 |
383 | hasMatch:
384 | switch absURL, err := absoluteImageURL(result.URL, result.Image); err {
385 | case errEmptyImageURL:
386 | case nil:
387 | switch {
388 | case validURL(absURL):
389 | result.Image = absURL
390 | default:
391 | result.Image = ""
392 | }
393 | if result.Image != "" && h.FetchImageSize && (result.ImageWidth == 0 || result.ImageHeight == 0) {
394 | if width, height, err := imageDimensions(ctx, h.HTTPClient, result.Image); err != nil {
395 | h.Log.Printf("dimensions detect for image %q: %v", result.Image, err)
396 | } else {
397 | result.ImageWidth, result.ImageHeight = width, height
398 | }
399 | }
400 | default:
401 | h.Log.Printf("cannot get absolute image url for %q: %v", result.Image, err)
402 | result.Image, result.ImageWidth, result.ImageHeight = "", 0, 0
403 | }
404 |
405 | if mc := h.Cache; mc != nil && !result.Empty() {
406 | if cdata, err := json.Marshal(result); err == nil {
407 | h.Log.Printf("Cache update for %q", link)
408 | mc.Set(&memcache.Item{Key: mcKey(link), Value: snappy.Encode(nil, cdata)})
409 | }
410 | }
411 | return result
412 | }
413 |
414 | // pageChunk describes first chunk of resource
415 | type pageChunk struct {
416 | data []byte // first chunk of resource data
417 | url *url.URL // final url resource was fetched from (after all redirects)
418 | ct string // Content-Type as reported by server
419 | }
420 |
421 | func (p *pageChunk) oembedEndpoint(fn oembed.LookupFunc) (url string, found bool) {
422 | if p == nil || fn == nil {
423 | return "", false
424 | }
425 | if u, ok := fn(p.url.String()); ok {
426 | return u, true
427 | }
428 | r, err := charset.NewReader(bytes.NewReader(p.data), p.ct)
429 | if err != nil {
430 | return "", false
431 | }
432 | if u, ok, err := oembed.Discover(r); err == nil && ok {
433 | return u, true
434 | }
435 | return "", false
436 | }
437 |
438 | func (h *unfurlHandler) httpGet(ctx context.Context, URL string) (*http.Response, error) {
439 | client := h.HTTPClient
440 | if client == nil {
441 | client = http.DefaultClient
442 | }
443 | req, err := http.NewRequest(http.MethodGet, URL, nil)
444 | if err != nil {
445 | return nil, err
446 | }
447 | for i := 0; i < len(h.Headers); i += 2 {
448 | req.Header.Set(h.Headers[i], h.Headers[i+1])
449 | }
450 | req = req.WithContext(ctx)
451 | return client.Do(req)
452 | }
453 |
454 | // fetchData fetches the first chunk of the resource. The chunk size is
455 | // determined by h.MaxBodyChunkSize.
456 | func (h *unfurlHandler) fetchData(ctx context.Context, URL string) (*pageChunk, error) {
457 | resp, err := h.httpGet(ctx, URL)
458 | if err != nil {
459 | return nil, err
460 | }
461 | defer resp.Body.Close()
462 |
463 | if resp.StatusCode >= http.StatusBadRequest {
464 | // returning pageChunk with the final url (after all redirects) so that
465 | // special cases like youtube returning 429 can be handled by
466 | // specialized fetchers like youtubeFetcher
467 | return &pageChunk{url: resp.Request.URL}, errors.New("bad status: " + resp.Status)
468 | }
469 | if resp.Header.Get("Content-Encoding") == "deflate" &&
470 | (strings.HasSuffix(resp.Request.Host, "twitter.com") ||
471 | strings.HasSuffix(resp.Request.Host, "x.com")) {
472 | // twitter/X sends unsolicited deflate-encoded responses
473 | // violating RFC; workaround this.
474 | // See https://golang.org/issues/18779 for background
475 | var err error
476 | if resp.Body, err = zlib.NewReader(resp.Body); err != nil {
477 | return nil, err
478 | }
479 | }
480 | head, err := io.ReadAll(io.LimitReader(resp.Body, h.MaxBodyChunkSize))
481 | if err != nil {
482 | return nil, err
483 | }
484 | return &pageChunk{
485 | data: head,
486 | url: resp.Request.URL,
487 | ct: resp.Header.Get("Content-Type"),
488 | }, nil
489 | }
490 |
491 | func (h *unfurlHandler) faviconLookup(ctx context.Context, chunk *pageChunk) (string, error) {
492 | if strings.HasPrefix(chunk.ct, "text/html") {
493 | href := extractFaviconLink(chunk.data, chunk.ct)
494 | if href == "" {
495 | goto probeDefaultIcon
496 | }
497 | u, err := url.Parse(href)
498 | if err != nil {
499 | return "", err
500 | }
501 | return chunk.url.ResolveReference(u).String(), nil
502 | }
503 | probeDefaultIcon:
504 | u := &url.URL{Scheme: chunk.url.Scheme, Host: chunk.url.Host, Path: "/favicon.ico"}
505 | client := h.HTTPClient
506 | if client == nil {
507 | client = http.DefaultClient
508 | }
509 | req, err := http.NewRequest(http.MethodHead, u.String(), nil)
510 | if err != nil {
511 | return "", err
512 | }
513 | for i := 0; i < len(h.Headers); i += 2 {
514 | req.Header.Set(h.Headers[i], h.Headers[i+1])
515 | }
516 | ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
517 | defer cancel()
518 | req = req.WithContext(ctx)
519 | r, err := client.Do(req)
520 | if err != nil {
521 | return "", err
522 | }
523 | defer r.Body.Close()
524 | if r.StatusCode == http.StatusOK {
525 | return u.String(), nil
526 | }
527 | return "", nil
528 | }
529 |
530 | // mcKey returns string of hex representation of sha1 sum of string provided.
531 | // Used to get safe keys to use with memcached
532 | func mcKey(s string) string {
533 | sum := sha1.Sum([]byte(s))
534 | return hex.EncodeToString(sum[:])
535 | }
536 |
537 | func blocklisted(blocklilst []string, title string) bool {
538 | if title == "" || len(blocklilst) == 0 {
539 | return false
540 | }
541 | lt := strings.ToLower(title)
542 | for _, s := range blocklilst {
543 | if strings.Contains(lt, s) {
544 | return true
545 | }
546 | }
547 | return false
548 | }
549 |
550 | //go:embed data/providers.json
551 | var providersData []byte
552 |
--------------------------------------------------------------------------------
/unfurlist_test.go:
--------------------------------------------------------------------------------
1 | package unfurlist
2 |
3 | import (
4 | _ "embed"
5 | "encoding/json"
6 | "errors"
7 | "net"
8 | "net/http"
9 | "net/http/httptest"
10 | "strings"
11 | "sync"
12 | "testing"
13 | "time"
14 | )
15 |
16 | func TestOpenGraph(t *testing.T) {
17 | result := doRequest("/?content=Test+http://techcrunch.com/2015/11/09/basic-income-createathon/", t)
18 | if len(result) != 1 {
19 | t.Fatalf("invalid result length: %v", result)
20 | }
21 |
22 | want := "Robots To Eat All The Jobs? Hackers, Policy Wonks Collaborate On A Basic Income Createathon This Weekend – TechCrunch"
23 | if result[0].Title != want {
24 | t.Errorf("unexpected Title, want %q, got %q", want, result[0].Title)
25 | }
26 |
27 | want = "https://techcrunch.com/wp-content/uploads/2015/11/basic-income-createathon.jpg?w=602"
28 | if result[0].Image != want {
29 | t.Errorf("unexpected Image, want %q, got %q", want, result[0].Image)
30 | }
31 | }
32 |
33 | func TestOpenGraphTwitter(t *testing.T) {
34 | result := doRequest("/?content=Test+https://twitter.com/amix3k/status/1399300280206909440", t)
35 | if len(result) != 1 {
36 | t.Fatalf("invalid result length: %v", result)
37 | }
38 |
39 | want := "My current meeting schedule this week"
40 | if !strings.Contains(result[0].Description, want) {
41 | t.Errorf("unexpected Description, want %q, got %q", want, result[0].Description)
42 | }
43 | }
44 |
45 | func TestHtml(t *testing.T) {
46 | result := doRequest("/?content=https://news.ycombinator.com/", t)
47 | if len(result) != 1 {
48 | t.Fatalf("invalid result length: %v", result)
49 | }
50 |
51 | want := "Hacker News"
52 | if result[0].Title != want {
53 | t.Errorf("unexpected Title, want %q, got %q", want, result[0].Title)
54 | }
55 |
56 | want = ""
57 | if result[0].Image != want {
58 | t.Errorf("unexpected Image, want %q, got %q", want, result[0].Image)
59 | }
60 |
61 | want = "website"
62 | if result[0].Type != want {
63 | t.Errorf("unexpected Type, want %q, got %q", want, result[0].Type)
64 | }
65 | }
66 |
67 | func TestUnfurlist__multibyteHTML(t *testing.T) {
68 | res := doRequest("/?content=http://news.chosun.com/site/data/html_dir/2009/09/24/2009092401755.html", t)
69 | want := `심장정지 환자 못살리는 119 구급차`
70 | if len(res) != 1 {
71 | t.Fatalf("invalid result length: %v", res)
72 | }
73 | if res[0].Title != want {
74 | t.Errorf("unexpected Title, want %q, got %q", want, res[0].Title)
75 | }
76 | }
77 |
78 | func doRequest(url string, t *testing.T) []unfurlResult {
79 | pp := newPipePool()
80 | defer pp.Close()
81 | go http.Serve(pp, http.HandlerFunc(replayHandler))
82 | handler := New(WithHTTPClient(&http.Client{
83 | Transport: &http.Transport{
84 | Dial: pp.Dial,
85 | DialTLS: pp.Dial,
86 | }}))
87 |
88 | w := httptest.NewRecorder()
89 | req, _ := http.NewRequest("GET", url, nil)
90 |
91 | handler.ServeHTTP(w, req)
92 |
93 | if w.Code != http.StatusOK {
94 | t.Fatalf("invalid status code: %v", w.Code)
95 | return nil
96 | }
97 |
98 | var result []unfurlResult
99 | err := json.Unmarshal(w.Body.Bytes(), &result)
100 | if err != nil {
101 | t.Fatalf("Result isn't JSON %v", w.Body.String())
102 | return nil
103 | }
104 |
105 | return result
106 | }
107 |
108 | func TestUnfurlist__singleInFlightRequest(t *testing.T) {
109 | pp := newPipePool()
110 | defer pp.Close()
111 | go http.Serve(pp, http.HandlerFunc(replayHandlerSerial(t)))
112 | handler := New(WithHTTPClient(&http.Client{
113 | Transport: &http.Transport{
114 | Dial: pp.Dial,
115 | DialTLS: pp.Dial,
116 | },
117 | }))
118 |
119 | var wg sync.WaitGroup
120 | barrier := make(chan struct{})
121 | for range 3 {
122 | wg.Add(1)
123 | go func() {
124 | w := httptest.NewRecorder()
125 | <-barrier
126 | req := httptest.NewRequest("GET", "/?content=https://news.ycombinator.com/", nil)
127 | handler.ServeHTTP(w, req)
128 | wg.Done()
129 | }()
130 | }
131 | // ensure multiple calls of unfurlistHandler.processURL() would be done
132 | // as close to each other as possible
133 | close(barrier)
134 | wg.Wait()
135 | }
136 |
137 | // replayHandlerSerial returns http.Handler responding with pre-recorded data
138 | // while ensuring that it doesn't process two simultaneous requests for the same
139 | // url
140 | func replayHandlerSerial(t *testing.T) func(w http.ResponseWriter, r *http.Request) {
141 | inFlight := struct {
142 | mu sync.Mutex
143 | reqs map[string]struct{}
144 | }{
145 | reqs: make(map[string]struct{}),
146 | }
147 | return func(w http.ResponseWriter, r *http.Request) {
148 | key := r.Host + r.URL.RequestURI()
149 | inFlight.mu.Lock()
150 | _, ok := inFlight.reqs[key]
151 | if ok {
152 | inFlight.mu.Unlock()
153 | t.Errorf("request for %q is already in flight", key)
154 | return
155 | }
156 | inFlight.reqs[key] = struct{}{}
157 | inFlight.mu.Unlock()
158 | defer func() {
159 | inFlight.mu.Lock()
160 | delete(inFlight.reqs, key)
161 | inFlight.mu.Unlock()
162 | }()
163 |
164 | d, ok := remoteData[r.Host+r.URL.RequestURI()]
165 | if !ok {
166 | http.Error(w, "not found", http.StatusNotFound)
167 | return
168 | }
169 | time.Sleep(10 * time.Millisecond) // increasing chances that multiple goroutines will call handler concurrently
170 | w.Write([]byte(d))
171 | }
172 | }
173 |
174 | // replayHandler is a http.Handler responding with pre-recorded data
175 | func replayHandler(w http.ResponseWriter, r *http.Request) {
176 | d, ok := remoteData[r.Host+r.URL.RequestURI()]
177 | if !ok {
178 | http.Error(w, "not found", http.StatusNotFound)
179 | return
180 | }
181 | // avoid type auto-detecting of saved pages
182 | w.Header().Set("Content-Type", "text/html")
183 | w.Write([]byte(d))
184 | }
185 |
186 | // pipePool implements net.Listener interface and provides a Dial() func to dial
187 | // to this listener
188 | type pipePool struct {
189 | m sync.RWMutex
190 | closed bool
191 | serverConns chan net.Conn
192 | }
193 |
194 | func newPipePool() *pipePool { return &pipePool{serverConns: make(chan net.Conn)} }
195 |
196 | func (p *pipePool) Accept() (net.Conn, error) {
197 | c, ok := <-p.serverConns
198 | if !ok {
199 | return nil, errors.New("listener is closed")
200 | }
201 | return c, nil
202 | }
203 |
204 | func (p *pipePool) Close() error {
205 | p.m.Lock()
206 | defer p.m.Unlock()
207 | if !p.closed {
208 | close(p.serverConns)
209 | p.closed = true
210 | }
211 | return nil
212 | }
213 | func (p *pipePool) Addr() net.Addr { return phonyAddr{} }
214 |
215 | func (p *pipePool) Dial(network, addr string) (net.Conn, error) {
216 | p.m.RLock()
217 | defer p.m.RUnlock()
218 | if p.closed {
219 | return nil, errors.New("listener is closed")
220 | }
221 | c1, c2 := net.Pipe()
222 | p.serverConns <- c1
223 | return c2, nil
224 | }
225 |
226 | type phonyAddr struct{}
227 |
228 | func (a phonyAddr) Network() string { return "pipe" }
229 | func (a phonyAddr) String() string { return "pipe" }
230 |
231 | //go:generate go run remote-data-update.go
232 |
233 | //go:embed testdata/remote-dump.json
234 | var remoteDataJson []byte
235 |
236 | var remoteData map[string]string
237 |
238 | func init() {
239 | var err error
240 | if err = json.Unmarshal(remoteDataJson, &remoteData); err != nil {
241 | panic(err)
242 | }
243 | }
244 |
--------------------------------------------------------------------------------
/url_parser.go:
--------------------------------------------------------------------------------
1 | package unfurlist
2 |
3 | import (
4 | "iter"
5 | "net/url"
6 | "regexp"
7 | "strings"
8 |
9 | "rsc.io/markdown"
10 | )
11 |
12 | // reUrls matches sequence of characters described by RFC 3986 having http:// or
13 | // https:// prefix. It actually allows superset of characters from RFC 3986,
14 | // allowing some most commonly used characters like {}, etc.
15 | var reUrls = regexp.MustCompile(`(?i:https?)://[%:/?#\[\]@!$&'\(\){}*+,;=\pL\pN._~-]+`)
16 |
17 | // ParseURLs tries to extract unique url-like (http/https scheme only) substrings from
18 | // given text. Results may not be proper urls, since only sequence of matched
19 | // characters are searched for. This function is optimized for extraction of
20 | // urls from plain text where it can be mixed with punctuation symbols: trailing
21 | // symbols []()<>,;. are removed, but // trailing >]) are left if any opening
22 | // <[( is found inside url.
23 | func ParseURLs(content string) []string { return parseURLsMax(content, -1) }
24 |
25 | func parseURLsMax(content string, maxItems int) []string {
26 | const punct = `[]()<>{},;.*_`
27 | res := reUrls.FindAllString(content, maxItems)
28 | for i, s := range res {
29 | // remove all combinations of trailing >)],. characters only if
30 | // no similar characters were found somewhere in the middle
31 | if idx := strings.IndexAny(s, punct); idx < 0 {
32 | continue
33 | }
34 | cleanLoop:
35 | for {
36 | idx2 := strings.LastIndexAny(s, punct)
37 | if idx2 != len(s)-1 {
38 | break
39 | }
40 | switch s[idx2] {
41 | case ')':
42 | if strings.Index(s, `(`) > 0 {
43 | break cleanLoop
44 | }
45 | case ']':
46 | if strings.Index(s, `[`) > 0 {
47 | break cleanLoop
48 | }
49 | case '>':
50 | if strings.Index(s, `<`) > 0 {
51 | break cleanLoop
52 | }
53 | case '}':
54 | if strings.Index(s, `{`) > 0 {
55 | break cleanLoop
56 | }
57 | }
58 | s = s[:idx2]
59 | }
60 | res[i] = s
61 | }
62 | out := res[:0]
63 | seen := make(map[string]struct{})
64 | for _, v := range res {
65 | if _, ok := seen[v]; ok {
66 | continue
67 | }
68 | out = append(out, v)
69 | seen[v] = struct{}{}
70 | }
71 | return out
72 | }
73 |
74 | // validURL returns true if s is a valid absolute url with http/https scheme.
75 | // In addition to verification that s is not empty and url.Parse(s) returns nil
76 | // error, validURL also ensures that query part only contains characters allowed
77 | // by RFC 3986 3.4.
78 | //
79 | // This is required because url.Parse doesn't verify query part of the URI.
80 | func validURL(s string) bool {
81 | if s == "" {
82 | return false
83 | }
84 | u, err := url.Parse(s)
85 | if err != nil {
86 | return false
87 | }
88 | if u.Host == "" {
89 | return false
90 | }
91 | switch u.Scheme {
92 | case "http", "https":
93 | default:
94 | return false
95 | }
96 | for _, r := range u.RawQuery {
97 | // https://tools.ietf.org/html/rfc3986#section-3.4 defines:
98 | //
99 | // query = *( pchar / "/" / "?" )
100 | // pchar = unreserved / pct-encoded / sub-delims / ":" / "@"
101 | // unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
102 | // pct-encoded = "%" HEXDIG HEXDIG
103 | // sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
104 | // / "*" / "+" / "," / ";" / "="
105 | //
106 | // check for these
107 | switch {
108 | case r >= '0' && r <= '9':
109 | case r >= 'A' && r <= 'Z':
110 | case r >= 'a' && r <= 'z':
111 | default:
112 | switch r {
113 | case '/', '?',
114 | ':', '@',
115 | '-', '.', '_', '~',
116 | '%', '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=':
117 | default:
118 | return false
119 | }
120 | }
121 | }
122 | return true
123 | }
124 |
125 | var parser = markdown.Parser{AutoLinkText: true}
126 |
127 | func parseMarkdownURLs(content string, maxItems int) []string {
128 | var out []string
129 | for link := range docLinks(parser.Parse(content)) {
130 | if !validURL(link.URL) {
131 | continue
132 | }
133 | out = append(out, link.URL)
134 | if maxItems != 0 && len(out) == maxItems {
135 | return out
136 | }
137 | }
138 | return out
139 | }
140 |
141 | func docLinks(doc *markdown.Document) iter.Seq[*markdown.Link] {
142 | var walkLinks func(markdown.Inlines, func(*markdown.Link) bool) bool
143 | walkLinks = func(inlines markdown.Inlines, yield func(*markdown.Link) bool) bool {
144 | for _, inl := range inlines {
145 | switch ent := inl.(type) {
146 | case *markdown.Strong:
147 | if !walkLinks(ent.Inner, yield) {
148 | return false
149 | }
150 | case *markdown.Emph:
151 | if !walkLinks(ent.Inner, yield) {
152 | return false
153 | }
154 | case *markdown.Link:
155 | if !yield(ent) {
156 | return false
157 | }
158 | }
159 | }
160 | return true
161 | }
162 | var walkBlocks func(markdown.Block, func(*markdown.Link) bool) bool
163 | walkBlocks = func(block markdown.Block, yield func(*markdown.Link) bool) bool {
164 | switch bl := block.(type) {
165 | case *markdown.Item:
166 | for _, b := range bl.Blocks {
167 | if !walkBlocks(b, yield) {
168 | return false
169 | }
170 | }
171 | case *markdown.List:
172 | for _, b := range bl.Items {
173 | if !walkBlocks(b, yield) {
174 | return false
175 | }
176 | }
177 | case *markdown.Paragraph:
178 | if !walkLinks(bl.Text.Inline, yield) {
179 | return false
180 | }
181 | case *markdown.Quote:
182 | for _, b := range bl.Blocks {
183 | if !walkBlocks(b, yield) {
184 | return false
185 | }
186 | }
187 | case *markdown.Text:
188 | if !walkLinks(bl.Inline, yield) {
189 | return false
190 | }
191 | }
192 | return true
193 | }
194 |
195 | return func(yield func(*markdown.Link) bool) {
196 | for _, b := range doc.Blocks {
197 | if !walkBlocks(b, yield) {
198 | return
199 | }
200 | }
201 | }
202 | }
203 |
--------------------------------------------------------------------------------
/url_parser_test.go:
--------------------------------------------------------------------------------
1 | package unfurlist
2 |
3 | import (
4 | "fmt"
5 | "testing"
6 | )
7 |
8 | func ExampleParseURLs() {
9 | text := `This text contains various urls mixed with different reserved per rfc3986 characters:
10 | http://google.com, https://doist.com/#about (also see https://todoist.com), <http://example.com/foo>,
11 | **[markdown](http://daringfireball.net/projects/markdown/)**,
12 | http://marvel-movies.wikia.com/wiki/The_Avengers_(film), https://pt.wikipedia.org/wiki/Mamão.
13 | https://docs.live.net/foo/?section-id={D7CEDACE-AEFB-4B61-9C63-BDE05EEBD80A},
14 | http://example.com/?param=foo;bar
15 | HTTPS://EXAMPLE.COM/UPPERCASE
16 | hTtP://example.com/mixedCase
17 | `
18 | for _, u := range ParseURLs(text) {
19 | fmt.Println(u)
20 | }
21 | // Output:
22 | // http://google.com
23 | // https://doist.com/#about
24 | // https://todoist.com
25 | // http://example.com/foo
26 | // http://daringfireball.net/projects/markdown/
27 | // http://marvel-movies.wikia.com/wiki/The_Avengers_(film)
28 | // https://pt.wikipedia.org/wiki/Mamão
29 | // https://docs.live.net/foo/?section-id={D7CEDACE-AEFB-4B61-9C63-BDE05EEBD80A}
30 | // http://example.com/?param=foo;bar
31 | // HTTPS://EXAMPLE.COM/UPPERCASE
32 | // hTtP://example.com/mixedCase
33 | }
34 |
35 | func TestParseURLs__unique(t *testing.T) {
36 | got := ParseURLs("Only two unique urls should be extracted from this text: http://google.com, http://twitter.com, http://google.com")
37 | want := []string{"http://google.com", "http://twitter.com"}
38 | if len(got) != len(want) {
39 | t.Fatalf("want %v, got %v", want, got)
40 | }
41 | for i, v := range got {
42 | if v != want[i] {
43 | t.Fatalf("want %v, got %v", want, got)
44 | }
45 | }
46 | }
47 |
48 | func TestBasicURLs(t *testing.T) {
49 | got := ParseURLs("Testing this out http://doist.com/#about https://todoist.com/chrome")
50 | want := []string{"http://doist.com/#about", "https://todoist.com/chrome"}
51 |
52 | if len(got) != len(want) {
53 | t.Errorf("Length not the same got: %d != want: %d", len(got), len(want))
54 | } else {
55 | for i := range want {
56 | if got[i] != want[i] {
57 | t.Errorf("%q != %s", got, want)
58 | }
59 | }
60 | }
61 | }
62 |
63 | func TestBugURL(t *testing.T) {
64 | got := ParseURLs("Testing this out Bug report http://f.cl.ly/items/000V0N1B31283s3O350q/Screen%20Shot%202015-12-22%20at%2014.49.28.png")
65 | want := []string{"http://f.cl.ly/items/000V0N1B31283s3O350q/Screen%20Shot%202015-12-22%20at%2014.49.28.png"}
66 |
67 | if len(got) != len(want) {
68 | t.Errorf("Length not the same got: %d != want: %d", len(got), len(want))
69 | } else {
70 | for i := range want {
71 | if got[i] != want[i] {
72 | t.Errorf("%q != %s", got, want)
73 | }
74 | }
75 | }
76 | }
77 |
78 | func TestValidURL(t *testing.T) {
79 | testCases := []struct {
80 | u string
81 | res bool
82 | }{
83 | {"https://example.com/path?multi+word+escaped+query", true},
84 | {"https://example.com/path?unescaped query", false},
85 | {"ftp://example.com/unsupported/scheme", false},
86 | {"", false},
87 | {"https://example.com/path", true},
88 | {"https:///path", false},
89 | }
90 | for _, tc := range testCases {
91 | if validURL(tc.u) != tc.res {
92 | t.Fatalf("validURL(%q)==%t, want %t", tc.u, !tc.res, tc.res)
93 | }
94 | }
95 | }
96 |
97 | func TestParseMarkdownURLs(t *testing.T) {
98 | text := `Implicit url: http://example.com/1, [explicit url](http://example.com/2).
99 |
100 | This url should be skipped ` + "`http://example.com/3`" + `, as well as the one inside code block:
101 |
102 | preformatted text block with url: http://example.com/4
103 |
104 | Another paragraph with implicit link http://example.com/5.
105 | `
106 | got := parseMarkdownURLs(text, 10)
107 | want := []string{"http://example.com/1", "http://example.com/2", "http://example.com/5"}
108 | if len(got) != len(want) {
109 | t.Fatalf("want: %v, got: %v", want, got)
110 | }
111 | for i := range got {
112 | if got[i] != want[i] {
113 | t.Fatalf("%d: want %q, got %q", i, want[i], got[i])
114 | }
115 | }
116 | }
117 |
118 | var escape []string
119 |
120 | func BenchmarkMarkdownURLs(b *testing.B) {
121 | text := `Implicit url: http://example.com/1, [explicit url](http://example.com/2).
122 |
123 | This url should be skipped ` + "`http://example.com/3`" + `, as well as the one inside code block:
124 |
125 | preformatted text block with url: http://example.com/4
126 |
127 | Another paragraph with implicit link http://example.com/5.
128 | `
129 | b.ReportAllocs()
130 | b.SetBytes(int64(len(text)))
131 | for b.Loop() {
132 | escape = parseMarkdownURLs(text, 10)
133 | }
134 | }
135 |
--------------------------------------------------------------------------------
/youtube.go:
--------------------------------------------------------------------------------
1 | package unfurlist
2 |
3 | import (
4 | "context"
5 | "net/http"
6 | "net/url"
7 | "strings"
8 | "time"
9 |
10 | "github.com/artyom/oembed"
11 | )
12 |
13 | // youtubeFetcher that retrieves metadata directly from
14 | // https://www.youtube.com/oembed endpoint.
15 | //
16 | // This is only needed because sometimes youtube may return captcha-walled
17 | // response that does not include oembed endpoint address as part of such html
18 | // page.
19 | func youtubeFetcher(ctx context.Context, client *http.Client, u *url.URL) (*Metadata, bool) {
20 | switch {
21 | case u.Host == "youtu.be" && len(u.Path) > 2:
22 | case u.Host == "www.youtube.com" && u.Path == "/watch" && strings.HasPrefix(u.RawQuery, "v="):
23 | default:
24 | return nil, false
25 | }
26 | ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
27 | defer cancel()
28 | const endpointPrefix = `https://www.youtube.com/oembed?format=json&url=`
29 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpointPrefix+url.QueryEscape(u.String()), nil)
30 | if err != nil {
31 | return nil, false
32 | }
33 | resp, err := client.Do(req)
34 | if err != nil {
35 | return nil, false
36 | }
37 | defer resp.Body.Close()
38 | meta, err := oembed.FromResponse(resp)
39 | if err != nil {
40 | return nil, false
41 | }
42 | return &Metadata{
43 | Title: meta.Title,
44 | Type: string(meta.Type),
45 | Image: meta.Thumbnail,
46 | ImageWidth: meta.ThumbnailWidth,
47 | ImageHeight: meta.ThumbnailHeight,
48 | }, true
49 | }
50 |
--------------------------------------------------------------------------------