├── .github
    └── workflows
    │   └── main.yml
├── .gitignore
├── LICENSE
├── README.md
├── flake.lock
├── flake.nix
├── pyproject.toml
├── requirements.txt
├── urlscan.1
└── urlscan
    ├── __init__.py
    ├── __main__.py
    ├── assets
        └── tlds-alpha-by-domain.txt
    ├── urlchoose.py
    └── urlscan.py


/.github/workflows/main.yml:
--------------------------------------------------------------------------------
  1 | name: main
  2 | 
  3 | on:
  4 |   push:
  5 |     branches:
  6 |       - main
  7 |     tags:
  8 |       - '*'
  9 |   pull_request:
 10 |     branches:
 11 |       - main
 12 | 
 13 | jobs:
 14 |   build:
 15 |     runs-on: ubuntu-latest
 16 |     steps:
 17 |       - uses: actions/checkout@v4
 18 |         with:
 19 |           fetch-depth: 0  # Ensures version gets set correctly
 20 | 
 21 |       - name: Set up Python
 22 |         uses: actions/setup-python@v5
 23 |         with:
 24 |           python-version: "3.x"
 25 | 
 26 |       - name: Install Hatch
 27 |         run: |
 28 |           python -m pip install --upgrade pip
 29 |           pip install hatch hatchling hatch-vcs
 30 | 
 31 |       - name: Build package
 32 |         run: hatch build
 33 | 
 34 |       - name: Store the distribution packages
 35 |         uses: actions/upload-artifact@v4
 36 |         with:
 37 |           name: urlscan
 38 |           path: dist/
 39 | 
 40 |   publish-to-pypi:
 41 |     name: Publish to PyPI
 42 |     if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
 43 |     needs:
 44 |       - build
 45 |     runs-on: ubuntu-latest
 46 |     environment:
 47 |       name: pypi
 48 |       url: https://pypi.org/p/urlscan  # Replace <package-name> with your PyPI project name
 49 |     permissions:
 50 |       id-token: write  # IMPORTANT: mandatory for trusted publishing
 51 | 
 52 |     steps:
 53 |       - name: Download all the dists
 54 |         uses: actions/download-artifact@v4
 55 |         with:
 56 |           name: urlscan
 57 |           path: dist/
 58 |       - name: Publish to PyPI
 59 |         uses: pypa/gh-action-pypi-publish@release/v1
 60 | 
 61 |   github-release:
 62 |     name: >-
 63 |       Sign the with Sigstore and upload to GitHub Release
 64 |     needs:
 65 |       - publish-to-pypi
 66 |     runs-on: ubuntu-latest
 67 | 
 68 |     permissions:
 69 |       contents: write  # IMPORTANT: mandatory for making GitHub Releases
 70 |       id-token: write  # IMPORTANT: mandatory for sigstore
 71 | 
 72 |     steps:
 73 |       - name: Check out the repository
 74 |         uses: actions/checkout@v4
 75 |         with:
 76 |           fetch-depth: 0  # Fetch all tags and history
 77 |       - name: Download all the dists
 78 |         uses: actions/download-artifact@v4
 79 |         with:
 80 |           name: urlscan
 81 |           path: dist/
 82 |       - name: Sign the dists with Sigstore
 83 |         uses: sigstore/gh-action-sigstore-python@v3.0.0
 84 |         with:
 85 |           inputs: >-
 86 |             ./dist/*.tar.gz
 87 |             ./dist/*.whl
 88 |       - name: Get tag annotation
 89 |         id: tag_annotation
 90 |         run: |
 91 |           # Extract the tag annotation for release notes
 92 |           tag_annotation=$(git for-each-ref --format '%(contents:body)' refs/tags/${{ github.ref_name }})
 93 |           echo "annotation=$tag_annotation" >> $GITHUB_ENV
 94 |       - name: Create GitHub Release
 95 |         env:
 96 |           GITHUB_TOKEN: ${{ github.token }}
 97 |         run: >-
 98 |           gh release create
 99 |           '${{ github.ref_name }}'
100 |           --repo '${{ github.repository }}'
101 |           --title '${{ github.ref_name }}'
102 |           --notes "${{ env.annotation }}"
103 |       - name: Upload artifact signatures to GitHub Release
104 |         env:
105 |           GITHUB_TOKEN: ${{ github.token }}
106 |         # Upload to GitHub Release using the `gh` CLI.
107 |         # `dist/` contains the built packages, and the
108 |         # sigstore-produced signatures and certificates.
109 |         run: >-
110 |           gh release upload
111 |           '${{ github.ref_name }}' dist/**
112 |           --repo '${{ github.repository }}'
113 | 
114 |   publish-to-testpypi:
115 |     name: Publish to TestPyPI
116 |     needs:
117 |       - build
118 |     runs-on: ubuntu-latest
119 | 
120 |     environment:
121 |       name: testpypi
122 |       url: https://test.pypi.org/p/urlscan
123 | 
124 |     permissions:
125 |       id-token: write  # IMPORTANT: mandatory for trusted publishing
126 | 
127 |     steps:
128 |       - name: Download all the dists
129 |         uses: actions/download-artifact@v4
130 |         with:
131 |           name: urlscan
132 |           path: dist/
133 |       - name: Publish to TestPyPI
134 |         uses: pypa/gh-action-pypi-publish@release/v1
135 |         with:
136 |           repository-url: https://test.pypi.org/legacy/
137 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | *.pyc
 2 | *.egg
 3 | *.egg-info
 4 | *.gz
 5 | *.xz
 6 | .envrc
 7 | .direnv/
 8 | build
 9 | dist
10 | test_emails/
11 | urlscan/_version.py
12 | MANIFEST
13 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 | 		    GNU GENERAL PUBLIC LICENSE
  2 | 		       Version 2, June 1991
  3 | 
  4 |  Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
  5 |  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  6 |  Everyone is permitted to copy and distribute verbatim copies
  7 |  of this license document, but changing it is not allowed.
  8 | 
  9 | 			    Preamble
 10 | 
 11 |   The licenses for most software are designed to take away your
 12 | freedom to share and change it.  By contrast, the GNU General Public
 13 | License is intended to guarantee your freedom to share and change free
 14 | software--to make sure the software is free for all its users.  This
 15 | General Public License applies to most of the Free Software
 16 | Foundation's software and to any other program whose authors commit to
 17 | using it.  (Some other Free Software Foundation software is covered by
 18 | the GNU Lesser General Public License instead.)  You can apply it to
 19 | your programs, too.
 20 | 
 21 |   When we speak of free software, we are referring to freedom, not
 22 | price.  Our General Public Licenses are designed to make sure that you
 23 | have the freedom to distribute copies of free software (and charge for
 24 | this service if you wish), that you receive source code or can get it
 25 | if you want it, that you can change the software or use pieces of it
 26 | in new free programs; and that you know you can do these things.
 27 | 
 28 |   To protect your rights, we need to make restrictions that forbid
 29 | anyone to deny you these rights or to ask you to surrender the rights.
 30 | These restrictions translate to certain responsibilities for you if you
 31 | distribute copies of the software, or if you modify it.
 32 | 
 33 |   For example, if you distribute copies of such a program, whether
 34 | gratis or for a fee, you must give the recipients all the rights that
 35 | you have.  You must make sure that they, too, receive or can get the
 36 | source code.  And you must show them these terms so they know their
 37 | rights.
 38 | 
 39 |   We protect your rights with two steps: (1) copyright the software, and
 40 | (2) offer you this license which gives you legal permission to copy,
 41 | distribute and/or modify the software.
 42 | 
 43 |   Also, for each author's protection and ours, we want to make certain
 44 | that everyone understands that there is no warranty for this free
 45 | software.  If the software is modified by someone else and passed on, we
 46 | want its recipients to know that what they have is not the original, so
 47 | that any problems introduced by others will not reflect on the original
 48 | authors' reputations.
 49 | 
 50 |   Finally, any free program is threatened constantly by software
 51 | patents.  We wish to avoid the danger that redistributors of a free
 52 | program will individually obtain patent licenses, in effect making the
 53 | program proprietary.  To prevent this, we have made it clear that any
 54 | patent must be licensed for everyone's free use or not licensed at all.
 55 | 
 56 |   The precise terms and conditions for copying, distribution and
 57 | modification follow.
 58 | 
 59 | 		    GNU GENERAL PUBLIC LICENSE
 60 |    TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
 61 | 
 62 |   0. This License applies to any program or other work which contains
 63 | a notice placed by the copyright holder saying it may be distributed
 64 | under the terms of this General Public License.  The "Program", below,
 65 | refers to any such program or work, and a "work based on the Program"
 66 | means either the Program or any derivative work under copyright law:
 67 | that is to say, a work containing the Program or a portion of it,
 68 | either verbatim or with modifications and/or translated into another
 69 | language.  (Hereinafter, translation is included without limitation in
 70 | the term "modification".)  Each licensee is addressed as "you".
 71 | 
 72 | Activities other than copying, distribution and modification are not
 73 | covered by this License; they are outside its scope.  The act of
 74 | running the Program is not restricted, and the output from the Program
 75 | is covered only if its contents constitute a work based on the
 76 | Program (independent of having been made by running the Program).
 77 | Whether that is true depends on what the Program does.
 78 | 
 79 |   1. You may copy and distribute verbatim copies of the Program's
 80 | source code as you receive it, in any medium, provided that you
 81 | conspicuously and appropriately publish on each copy an appropriate
 82 | copyright notice and disclaimer of warranty; keep intact all the
 83 | notices that refer to this License and to the absence of any warranty;
 84 | and give any other recipients of the Program a copy of this License
 85 | along with the Program.
 86 | 
 87 | You may charge a fee for the physical act of transferring a copy, and
 88 | you may at your option offer warranty protection in exchange for a fee.
 89 | 
 90 |   2. You may modify your copy or copies of the Program or any portion
 91 | of it, thus forming a work based on the Program, and copy and
 92 | distribute such modifications or work under the terms of Section 1
 93 | above, provided that you also meet all of these conditions:
 94 | 
 95 |     a) You must cause the modified files to carry prominent notices
 96 |     stating that you changed the files and the date of any change.
 97 | 
 98 |     b) You must cause any work that you distribute or publish, that in
 99 |     whole or in part contains or is derived from the Program or any
100 |     part thereof, to be licensed as a whole at no charge to all third
101 |     parties under the terms of this License.
102 | 
103 |     c) If the modified program normally reads commands interactively
104 |     when run, you must cause it, when started running for such
105 |     interactive use in the most ordinary way, to print or display an
106 |     announcement including an appropriate copyright notice and a
107 |     notice that there is no warranty (or else, saying that you provide
108 |     a warranty) and that users may redistribute the program under
109 |     these conditions, and telling the user how to view a copy of this
110 |     License.  (Exception: if the Program itself is interactive but
111 |     does not normally print such an announcement, your work based on
112 |     the Program is not required to print an announcement.)
113 | 
114 | These requirements apply to the modified work as a whole.  If
115 | identifiable sections of that work are not derived from the Program,
116 | and can be reasonably considered independent and separate works in
117 | themselves, then this License, and its terms, do not apply to those
118 | sections when you distribute them as separate works.  But when you
119 | distribute the same sections as part of a whole which is a work based
120 | on the Program, the distribution of the whole must be on the terms of
121 | this License, whose permissions for other licensees extend to the
122 | entire whole, and thus to each and every part regardless of who wrote it.
123 | 
124 | Thus, it is not the intent of this section to claim rights or contest
125 | your rights to work written entirely by you; rather, the intent is to
126 | exercise the right to control the distribution of derivative or
127 | collective works based on the Program.
128 | 
129 | In addition, mere aggregation of another work not based on the Program
130 | with the Program (or with a work based on the Program) on a volume of
131 | a storage or distribution medium does not bring the other work under
132 | the scope of this License.
133 | 
134 |   3. You may copy and distribute the Program (or a work based on it,
135 | under Section 2) in object code or executable form under the terms of
136 | Sections 1 and 2 above provided that you also do one of the following:
137 | 
138 |     a) Accompany it with the complete corresponding machine-readable
139 |     source code, which must be distributed under the terms of Sections
140 |     1 and 2 above on a medium customarily used for software interchange; or,
141 | 
142 |     b) Accompany it with a written offer, valid for at least three
143 |     years, to give any third party, for a charge no more than your
144 |     cost of physically performing source distribution, a complete
145 |     machine-readable copy of the corresponding source code, to be
146 |     distributed under the terms of Sections 1 and 2 above on a medium
147 |     customarily used for software interchange; or,
148 | 
149 |     c) Accompany it with the information you received as to the offer
150 |     to distribute corresponding source code.  (This alternative is
151 |     allowed only for noncommercial distribution and only if you
152 |     received the program in object code or executable form with such
153 |     an offer, in accord with Subsection b above.)
154 | 
155 | The source code for a work means the preferred form of the work for
156 | making modifications to it.  For an executable work, complete source
157 | code means all the source code for all modules it contains, plus any
158 | associated interface definition files, plus the scripts used to
159 | control compilation and installation of the executable.  However, as a
160 | special exception, the source code distributed need not include
161 | anything that is normally distributed (in either source or binary
162 | form) with the major components (compiler, kernel, and so on) of the
163 | operating system on which the executable runs, unless that component
164 | itself accompanies the executable.
165 | 
166 | If distribution of executable or object code is made by offering
167 | access to copy from a designated place, then offering equivalent
168 | access to copy the source code from the same place counts as
169 | distribution of the source code, even though third parties are not
170 | compelled to copy the source along with the object code.
171 | 
172 |   4. You may not copy, modify, sublicense, or distribute the Program
173 | except as expressly provided under this License.  Any attempt
174 | otherwise to copy, modify, sublicense or distribute the Program is
175 | void, and will automatically terminate your rights under this License.
176 | However, parties who have received copies, or rights, from you under
177 | this License will not have their licenses terminated so long as such
178 | parties remain in full compliance.
179 | 
180 |   5. You are not required to accept this License, since you have not
181 | signed it.  However, nothing else grants you permission to modify or
182 | distribute the Program or its derivative works.  These actions are
183 | prohibited by law if you do not accept this License.  Therefore, by
184 | modifying or distributing the Program (or any work based on the
185 | Program), you indicate your acceptance of this License to do so, and
186 | all its terms and conditions for copying, distributing or modifying
187 | the Program or works based on it.
188 | 
189 |   6. Each time you redistribute the Program (or any work based on the
190 | Program), the recipient automatically receives a license from the
191 | original licensor to copy, distribute or modify the Program subject to
192 | these terms and conditions.  You may not impose any further
193 | restrictions on the recipients' exercise of the rights granted herein.
194 | You are not responsible for enforcing compliance by third parties to
195 | this License.
196 | 
197 |   7. If, as a consequence of a court judgment or allegation of patent
198 | infringement or for any other reason (not limited to patent issues),
199 | conditions are imposed on you (whether by court order, agreement or
200 | otherwise) that contradict the conditions of this License, they do not
201 | excuse you from the conditions of this License.  If you cannot
202 | distribute so as to satisfy simultaneously your obligations under this
203 | License and any other pertinent obligations, then as a consequence you
204 | may not distribute the Program at all.  For example, if a patent
205 | license would not permit royalty-free redistribution of the Program by
206 | all those who receive copies directly or indirectly through you, then
207 | the only way you could satisfy both it and this License would be to
208 | refrain entirely from distribution of the Program.
209 | 
210 | If any portion of this section is held invalid or unenforceable under
211 | any particular circumstance, the balance of the section is intended to
212 | apply and the section as a whole is intended to apply in other
213 | circumstances.
214 | 
215 | It is not the purpose of this section to induce you to infringe any
216 | patents or other property right claims or to contest validity of any
217 | such claims; this section has the sole purpose of protecting the
218 | integrity of the free software distribution system, which is
219 | implemented by public license practices.  Many people have made
220 | generous contributions to the wide range of software distributed
221 | through that system in reliance on consistent application of that
222 | system; it is up to the author/donor to decide if he or she is willing
223 | to distribute software through any other system and a licensee cannot
224 | impose that choice.
225 | 
226 | This section is intended to make thoroughly clear what is believed to
227 | be a consequence of the rest of this License.
228 | 
229 |   8. If the distribution and/or use of the Program is restricted in
230 | certain countries either by patents or by copyrighted interfaces, the
231 | original copyright holder who places the Program under this License
232 | may add an explicit geographical distribution limitation excluding
233 | those countries, so that distribution is permitted only in or among
234 | countries not thus excluded.  In such case, this License incorporates
235 | the limitation as if written in the body of this License.
236 | 
237 |   9. The Free Software Foundation may publish revised and/or new versions
238 | of the General Public License from time to time.  Such new versions will
239 | be similar in spirit to the present version, but may differ in detail to
240 | address new problems or concerns.
241 | 
242 | Each version is given a distinguishing version number.  If the Program
243 | specifies a version number of this License which applies to it and "any
244 | later version", you have the option of following the terms and conditions
245 | either of that version or of any later version published by the Free
246 | Software Foundation.  If the Program does not specify a version number of
247 | this License, you may choose any version ever published by the Free Software
248 | Foundation.
249 | 
250 |   10. If you wish to incorporate parts of the Program into other free
251 | programs whose distribution conditions are different, write to the author
252 | to ask for permission.  For software which is copyrighted by the Free
253 | Software Foundation, write to the Free Software Foundation; we sometimes
254 | make exceptions for this.  Our decision will be guided by the two goals
255 | of preserving the free status of all derivatives of our free software and
256 | of promoting the sharing and reuse of software generally.
257 | 
258 | 			    NO WARRANTY
259 | 
260 |   11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
268 | REPAIR OR CORRECTION.
269 | 
270 |   12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
278 | POSSIBILITY OF SUCH DAMAGES.
279 | 
280 | 		     END OF TERMS AND CONDITIONS
281 | 
282 | 	    How to Apply These Terms to Your New Programs
283 | 
284 |   If you develop a new program, and you want it to be of the greatest
285 | possible use to the public, the best way to achieve this is to make it
286 | free software which everyone can redistribute and change under these terms.
287 | 
288 |   To do so, attach the following notices to the program.  It is safest
289 | to attach them to the start of each source file to most effectively
290 | convey the exclusion of warranty; and each file should have at least
291 | the "copyright" line and a pointer to where the full notice is found.
292 | 
293 |     <one line to give the program's name and a brief idea of what it does.>
294 |     Copyright (C) <year>  <name of author>
295 | 
296 |     This program is free software; you can redistribute it and/or modify
297 |     it under the terms of the GNU General Public License as published by
298 |     the Free Software Foundation; either version 2 of the License, or
299 |     (at your option) any later version.
300 | 
301 |     This program is distributed in the hope that it will be useful,
302 |     but WITHOUT ANY WARRANTY; without even the implied warranty of
303 |     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
304 |     GNU General Public License for more details.
305 | 
306 |     You should have received a copy of the GNU General Public License along
307 |     with this program; if not, write to the Free Software Foundation, Inc.,
308 |     51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
309 | 
310 | Also add information on how to contact you by electronic and paper mail.
311 | 
312 | If the program is interactive, make it output a short notice like this
313 | when it starts in an interactive mode:
314 | 
315 |     Gnomovision version 69, Copyright (C) year name of author
316 |     Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
317 |     This is free software, and you are welcome to redistribute it
318 |     under certain conditions; type `show c' for details.
319 | 
320 | The hypothetical commands `show w' and `show c' should show the appropriate
321 | parts of the General Public License.  Of course, the commands you use may
322 | be called something other than `show w' and `show c'; they could even be
323 | mouse-clicks or menu items--whatever suits your program.
324 | 
325 | You should also get your employer (if you work as a programmer) or your
326 | school, if any, to sign a "copyright disclaimer" for the program, if
327 | necessary.  Here is a sample; alter the names:
328 | 
329 |   Yoyodyne, Inc., hereby disclaims all copyright interest in the program
330 |   `Gnomovision' (which makes passes at compilers) written by James Hacker.
331 | 
332 |   <signature of Ty Coon>, 1 April 1989
333 |   Ty Coon, President of Vice
334 | 
335 | This General Public License does not permit incorporating your program into
336 | proprietary programs.  If your program is a subroutine library, you may
337 | consider it more useful to permit linking proprietary applications with the
338 | library.  If this is what you want to do, use the GNU Lesser General
339 | Public License instead of this License.
340 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Urlscan
  2 | 
  3 | [![main](https://github.com/firecat53/urlscan/actions/workflows/main.yml/badge.svg)](https://github.com/firecat53/urlscan/actions/workflows/main.yml)
  4 | 
  5 | ## Contributors
  6 | 
  7 | Scott Hansen \<tech@firecat53.net\> (Author and Maintainer)
  8 | 
  9 | Maxime Chatelle \<xakz@rxsoft.eu\> (Debian Maintainer)
 10 | 
 11 | Daniel Burrows \<dburrows@debian.org\> (Original Author)
 12 | 
 13 | ## Purpose and Requirements
 14 | 
 15 | Urlscan is a small program that is designed to integrate with the "mutt"
 16 | mailreader to allow you to easily launch a Web browser for URLs contained in
 17 | email messages. It is a replacement for the "urlview" program.
 18 | 
 19 | Requires: Python 3.7+ and the python-urwid library
 20 | 
 21 | ## Features
 22 | 
 23 | Urlscan parses an email message or file and scans it for URLs and email
 24 | addresses. It then displays the URLs and their context within the message, and
 25 | allows you to choose one or more URLs to send to your Web browser.
 26 | Alternatively, it send a list of all URLs to stdout.
 27 | 
 28 | Relative to urlview, urlscan has the following additional features:
 29 | 
 30 | - Support for emails in quoted-printable and base64 encodings. No more stripping
 31 |   out =40D from URLs by hand!
 32 | 
 33 | - The context of each URL is provided along with the URL. For HTML mails, a
 34 |   crude parser is used to render the HTML into text. Context view can be toggled
 35 |   on/off with `c`.
 36 | 
 37 | - URLs are shortened by default to fit on one line. Viewing full URL (for one or
 38 |   all) is toggled with `s` or `S`.
 39 | 
 40 | - Jump to a URL by typing the number.
 41 | 
 42 | - Incremental case-insensitive search with `/`.
 43 | 
 44 | - Execute an arbitrary function (for example, copy URL to clipboard) instead of
 45 |   opening URL in a browser.
 46 | 
 47 | - Use `l` to cycle through whether URLs are opened using the Python webbrowser
 48 |   module (default), xdg-open (if installed) or opened by a function passed on
 49 |   the command line with `--run` or `--run-safe`.
 50 | 
 51 | - Configure colors and keybindings via ~/.config/urlscan/config.json. Generate
 52 |   default config file for editing by running `urlscan -g`. Cycle through
 53 |   available palettes with `p`. Set display width with `--width`.
 54 | 
 55 | - Set the urwid color mode with `--colors`. Options include 'true' (2**24
 56 |   colors), '256', '88', '16', '8', or 'mono'. Default is '16'.
 57 | 
 58 | - Copy URL to clipboard with `C` or to primary selection with `P`.  Requires
 59 |   xsel or xclip.
 60 | 
 61 | - Run a command with the selected URL as the argument or pipe the selected
 62 |   URL to a command.
 63 | 
 64 | - Show complete help menu with `F1`. Hide header on startup with `--nohelp`.
 65 | 
 66 | - Use a custom regular expression with `-E` for matching urls or any
 67 |   other pattern. In junction with `-r`, this effectively turns urlscan
 68 |   into a general purpose CLI selector-type utility.
 69 | 
 70 | - Scan certain email headers for URLs. Currently `Link`, `Archived-At` and
 71 |   `List-*` are scanned when `--headers` is passed.
 72 | 
 73 | - Queue multiple URLs for opening and open them all at once with `a` and `o`.
 74 | 
 75 | ## Installation and setup
 76 | 
 77 | To install urlscan, install from your distribution repositories, from Pypi, or do
 78 | a local development install with pip -e:
 79 | 
 80 |     pipx install urlscan
 81 | 
 82 |     OR
 83 | 
 84 |     pip install --user urlscan
 85 | 
 86 |     OR
 87 | 
 88 |     cd <path/to/urlscan> && pip install --user -e .
 89 | 
 90 | **NOTE**
 91 | 
 92 |     The minimum required version of urwid is 1.2.1.
 93 | 
 94 | Once urlscan is installed, add the following lines to your .muttrc:
 95 | 
 96 |     macro index,pager \cb "<pipe-message> urlscan<Enter>" "call urlscan to
 97 |     extract URLs out of a message"
 98 | 
 99 |     macro attach,compose \cb "<pipe-entry> urlscan<Enter>" "call urlscan to
100 |     extract URLs out of a message"
101 | 
102 | Once this is done, Control-b while reading mail in mutt will automatically
103 | invoke urlscan on the message.
104 | 
105 | > Note for Neomutt users: [As of version
106 | > `2023-05-17`](https://github.com/neomutt/neomutt/releases/tag/20230517) true
107 | > color support was implemented. If you are using true color support with Neomutt,
108 | > or are encountering the error `setupterm: could not find terminfo database`,
109 | > then you should also add `TERM=xterm-256color` to your macro in `.muttrc`.
110 | > See more here [#135](https://github.com/firecat53/urlscan/issues/135). For example:
111 | > `macro index,pager  \cb "<pipe-message>  TERM=xterm-256color urlscan<Enter>" "call urlscan to
112 | extract URLs out of a message"`
113 | 
114 | To choose a particular browser, set the environment variable BROWSER. If BROWSER
115 | is not set, xdg-open will control which browser is used, if it's available.:
116 | 
117 |     export BROWSER=/usr/bin/epiphany
118 | 
119 | 
120 | ## Command Line usage
121 | 
122 |     urlscan OPTIONS <file>
123 | 
124 |     OPTIONS [-c, --compact]
125 |             [-d, --dedupe]
126 |             [-E, --regex <expression>]
127 |             [-f, --run-safe <expression>]
128 |             [-g, --genconf]
129 |             [-H, --nohelp]
130 |             [    --headers]
131 |             [-n, --no-browser]
132 |             [-p, --pipe]
133 |             [-r, --run <expression>]
134 |             [-R, --reverse]
135 |             [-s, --single]
136 |             [-w, --width]
137 |             [-W  --whitespace-off]
138 |             [-C, --colors {true,256,88,16,8,mono}]
139 | 
140 | Urlscan can extract URLs and email addresses from emails or any text file.
141 | Calling with no flags will start the curses browser. Calling with '-n' will just
142 | output a list of URLs/email addressess to stdout. The '-c' flag removes the
143 | context from around the URLs in the curses browser, and the '-d' flag removes
144 | duplicate URLs. The '-R' flag reverses the displayed order of URLs and context.
145 | Files can also be piped to urlscan using normal shell pipe mechanisms: `cat
146 | <something> | urlscan` or `urlscan < <something>`. The '-W' flag condenses the
147 | display output by suppressing blank lines and ellipses lines.
148 | 
149 | Instead of opening a web browser, the selected URL can be passed as the argument
150 | to a command using `--run-safe "<command> {}"` or `--run "<command> {}"`. Note
151 | the use of `{}` in the command string to denote the selected URL. Alternatively,
152 | the URL can be piped to the command using `--run-safe <command> --pipe` (or
153 | `--run`). Using --run-safe with --pipe is preferred if the command supports it,
154 | as it is marginally more secure and tolerant of special characters in the URL.
155 | 
156 | ## Theming
157 | 
158 | Run `urlscan -g` to generate ~/.config/urlscan/config.json with the default
159 | color and black & white palettes. This can be edited or added to, as desired.
160 | The first palette in the list will be the default. Configure the palettes
161 | according to the [Urwid display attributes][1].
162 | 
163 | Partial example from default palette:
164 | ```json
165 |         "default": [
166 |             [
167 |                 "header",   # Urwid widget name
168 |                 "white",    # Standard foreground color
169 |                 "dark blue",# Standard background color
170 |                 "standout", # Monochrome settings
171 |                 "#ffffff",  # Extended foreground color (88 through True colors)
172 |                 "#0000aa"   # Extended foreground color (88 through True colors)
173 |             ],
174 |             [
175 |                 "footer",
176 |                 ...
177 | ```
178 | 
179 | Display width can be set with `--width`.
180 | 
181 | ## Keybindings
182 | 
183 | Run `urlscan -g` to generate ~/.config/urlscan/config.json. All of the keys will
184 | be listed. You can either leave in place or delete any that will not be altered.
185 | 
186 | To unset a binding, set it equal to "".  For example: `"P": ""`
187 | 
188 | The follow actions are supported:
189 | 
190 | - `add_url` -- add a URL to the queue (default: `a`)
191 | - `all_escape` -- toggle unescape all URLs (default: `u`)
192 | - `all_shorten` -- toggle shorten all URLs (default: `S`)
193 | - `bottom` -- move cursor to last item (default: `G`)
194 | - `clear_screen` -- redraw screen (default: `Ctrl-l`)
195 | - `clipboard` -- copy highlighted URL to clipboard using xsel/xclip (default: `C`)
196 | - `clipboard_pri` -- copy highlighted URL to primary selection using xsel/xclip (default: `P`)
197 | - `context` -- show/hide context (default: `c`)
198 | - `del_url` -- delete URL from the queue (default: `d`)
199 | - `down` -- cursor down (default: `j`)
200 | - `help_menu` -- show/hide help menu (default: `F1`)
201 | - `link_handler` -- cycle link handling (webbrowser, xdg-open, --run-safe or --run) (default: `l`)
202 | - `next` -- jump to next URL (default: `J`)
203 | - `open_queue` -- open all URLs in queue (default: `o`)
204 | - `open_queue_win` -- open all URLs in queue in new window (default: `O`)
205 | - `open_url` -- open selected URL (default: `space` or `enter`)
206 | - `palette` -- cycle through palettes (default: `p`)
207 | - `previous` -- jump to previous URL (default: `K`)
208 | - `quit` -- quit (default: `q` or `Q`)
209 | - `reverse` -- reverse display order (default: `R`)
210 | - `shorten` -- toggle shorten highlighted URL (default: `s`)
211 | - `top` -- move to first list item (default: `g`)
212 | - `up` -- cursor up (default: `k`)
213 | 
214 | ## Known bugs and limitations
215 | 
216 | - Running urlscan sometimes "messes up" the terminal background. This seems to
217 |   be an urwid bug, but I haven't tracked down just what's going on.
218 | 
219 | - Extraction of context from HTML messages leaves something to be desired.
220 |   Probably the ideal solution would be to extract context on a word basis rather
221 |   than on a paragraph basis.
222 | 
223 | - The HTML message handling is a bit kludgy in general.
224 | 
225 | - multipart/alternative sections are handled by descending into all the
226 |   sub-parts, rather than just picking one, which may lead to URLs and context
227 |   appearing twice. (Bypass this by selecting the '--dedupe' option)
228 | 
229 | ## Build/development
230 | 
231 | - pyproject.toml is configured for [hatch][2] for building and submitting to pypi.
232 | - flake.nix is available for a development shell or building/testing the package
233 |   if desired. `nix develop`
234 | - To update TLD list: `wget https://data.iana.org/TLD/tlds-alpha-by-domain.txt`
235 | - GitHub Action will upload to TestPyPi on each push to `main`. To create a
236 |   GitHub and PyPi release, create a new tag (formatting below) and push tags.
237 | 
238 |         <tag name on first line>
239 |         
240 |         * Release note 1
241 |         * Release note 2
242 |         * ...
243 | 
244 | [1]: http://urwid.org/manual/displayattributes.html#display-attributes  "Urwid display attributes"
245 | [2]: https://hatch.pypa.io/latest/  "Hatch"
246 | 


--------------------------------------------------------------------------------
/flake.lock:
--------------------------------------------------------------------------------
 1 | {
 2 |   "nodes": {
 3 |     "nixpkgs": {
 4 |       "locked": {
 5 |         "lastModified": 1747087824,
 6 |         "narHash": "sha256-13ummRyTs5dVYt4M0w3IpGUeibC3/ZEDLHBU30hJCsg=",
 7 |         "owner": "NixOS",
 8 |         "repo": "nixpkgs",
 9 |         "rev": "deb8833fd813fecbabed844301545ad4b4aca79c",
10 |         "type": "github"
11 |       },
12 |       "original": {
13 |         "owner": "NixOS",
14 |         "repo": "nixpkgs",
15 |         "type": "github"
16 |       }
17 |     },
18 |     "root": {
19 |       "inputs": {
20 |         "nixpkgs": "nixpkgs"
21 |       }
22 |     }
23 |   },
24 |   "root": "root",
25 |   "version": 7
26 | }
27 | 


--------------------------------------------------------------------------------
/flake.nix:
--------------------------------------------------------------------------------
 1 | {
 2 |   description = "View/select the URLs in an email message or file";
 3 | 
 4 |   inputs = {
 5 |     nixpkgs.url = "github:NixOS/nixpkgs";
 6 |   };
 7 | 
 8 |   outputs = {
 9 |     self,
10 |     nixpkgs,
11 |   }: let
12 |     systems = ["x86_64-linux" "i686-linux" "aarch64-linux"];
13 |     forAllSystems = f:
14 |       nixpkgs.lib.genAttrs systems (system:
15 |         f rec {
16 |           pkgs = nixpkgs.legacyPackages.${system};
17 |           commonPackages = builtins.attrValues {
18 |             inherit
19 |               (pkgs.python3Packages)
20 |               python
21 |               urwid
22 |               ;
23 |           };
24 |         });
25 |   in {
26 |     devShells = forAllSystems ({
27 |       pkgs,
28 |       commonPackages,
29 |     }: {
30 |       default = pkgs.mkShell {
31 |         packages = commonPackages ++ [pkgs.pandoc];
32 |         shellHook = ''
33 |           alias urlscan="python -m urlscan"
34 |           export PYTHONPATH="$PYTHONPATH:$PWD"
35 |         '';
36 |       };
37 |     });
38 |     packages = forAllSystems ({
39 |       pkgs,
40 |       commonPackages,
41 |     }: {
42 |       default = pkgs.python3Packages.buildPythonApplication {
43 |         name = "urlscan";
44 |         pname = "urlscan";
45 |         format = "pyproject";
46 |         src = ./.;
47 |         nativeBuildInputs = builtins.attrValues {
48 |           inherit
49 |             (pkgs)
50 |             git
51 |             ;
52 |           inherit
53 |             (pkgs.python3Packages)
54 |             hatchling
55 |             hatch-vcs
56 |             ;
57 |         };
58 |         propagatedBuildInputs = commonPackages;
59 |         meta = {
60 |           description = "View/select the URLs in an email message or file";
61 |           homepage = "https://github.com/firecat53/urlscan";
62 |           license = pkgs.lib.licenses.gpl2Plus;
63 |           maintainers = ["firecat53"];
64 |           platforms = systems;
65 |         };
66 |       };
67 |     });
68 |   };
69 | }
70 | 


--------------------------------------------------------------------------------
/pyproject.toml:
--------------------------------------------------------------------------------
 1 | [build-system]
 2 | requires = ["hatchling", "hatch-vcs"]
 3 | build-backend = "hatchling.build"
 4 | 
 5 | [project]
 6 | name = "urlscan"
 7 | dynamic = ["version"]
 8 | description = "View/select the URLs in an email message or file"
 9 | readme = "README.md"
10 | license = "GPL-2.0-or-later"
11 | authors = [
12 |     { name = "Scott Hansen", email = "tech@firecat53.net" },
13 | ]
14 | keywords = [
15 |     "email",
16 |     "mutt",
17 |     "tmux",
18 |     "urlscan",
19 |     "urlview",
20 | ]
21 | classifiers = [
22 |     "Development Status :: 5 - Production/Stable",
23 |     "Environment :: Console",
24 |     "Environment :: Console :: Curses",
25 |     "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
26 |     "Operating System :: OS Independent",
27 |     "Programming Language :: Python",
28 |     "Programming Language :: Python :: 3.7",
29 |     "Programming Language :: Python :: 3.8",
30 |     "Programming Language :: Python :: 3.9",
31 |     "Programming Language :: Python :: 3.10",
32 |     "Programming Language :: Python :: 3.11",
33 |     "Programming Language :: Python :: 3.12",
34 |     "Programming Language :: Python :: 3.13",
35 |     "Topic :: Utilities",
36 | ]
37 | dependencies = [
38 |     "urwid>=1.2.1",
39 | ]
40 | 
41 | [project.scripts]
42 | urlscan = "urlscan.__main__:main"
43 | 
44 | [project.urls]
45 | Homepage = "https://github.com/firecat53/urlscan"
46 | 
47 | [tool.hatch.version]
48 | source = "vcs"
49 | fallback-version = "0.0.0"
50 | 
51 | [tool.hatch.version.raw-options]
52 | local_scheme = "no-local-version"
53 | 
54 | [tool.hatch.build.hooks.vcs]
55 | version-file = "urlscan/_version.py"
56 | 
57 | [tool.hatch.build.targets.wheel.shared-data]
58 | LICENSE = "share/doc/urlscan/LICENSE"
59 | "README.md" = "share/doc/urlscan/README.md"
60 | "urlscan.1" = "share/man/man1/urlscan.1"
61 | 
62 | [tool.hatch.build.targets.sdist]
63 | include = [
64 |     "/urlscan",
65 |     "urlscan.1",
66 | ]
67 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | urwid>=1.2.1
2 | 


--------------------------------------------------------------------------------
/urlscan.1:
--------------------------------------------------------------------------------
  1 | .\"                                      Hey, EMACS: -*- nroff -*-
  2 | 
  3 | .TH URLSCAN 1 "1 June 2024"
  4 | 
  5 | .SH NAME
  6 | urlscan \- browse the URLs in an email message from a terminal
  7 | .SH SYNOPSIS
  8 | \fBurlscan\fR [options] <
  9 | .I message
 10 | 
 11 | \fBurlscan\fR [options]
 12 | .I message
 13 | 
 14 | .SH DESCRIPTION
 15 | \fBurlscan\fR accepts a single email message on standard
 16 | input, then displays a terminal-based list of the URLs in the given
 17 | message.  Selecting a URL uses the Python webbrowser module to
 18 | determine which browser to open. The \fBBROWSER\fR environment
 19 | variable will be used if it is set.
 20 | 
 21 | \fBurlscan\fR is primarily intended to be used with the
 22 | .B mutt
 23 | (1) mailreader, but it should work well with any terminal-based mail
 24 | program.
 25 | 
 26 | \fBurlscan\fR is similar to \fBurlview\fR(1), but has the following
 27 | additional features:
 28 | 
 29 | \fB1.\fR Support for more message encodings, such as quoted-printable
 30 | and base64.
 31 | 
 32 | \fB2.\fR Extraction and display of the context surrounding each URL. Toggle
 33 | context view on/off with \fBc\fR. Reverse displayed order of URLs with \fBR\fR.
 34 | 
 35 | \fB3.\fR Copy current URL to primary selection with \fBP\fR or to clipboard with
 36 | \fBC\fR.
 37 | 
 38 | \fB4.\fR URLs are shortened by default to fit on one line. Toggle one or all
 39 | shortened URLs with \fBs\fR or \fBS\fR.
 40 | 
 41 | \fB5.\fR Incremental case-insensitive search using \fB/\fR. Footer shows current
 42 | search term. \fB/\fR again resets search.
 43 | 
 44 | \fB6.\fR Cycle through all available palettes (color and black & white available
 45 | by default) using \fBp\fR. Running \fBurlscan \-g\fR will generate a
 46 | ~/.config/urlscan/config.json file for editing or adding additional pallettes
 47 | and keybindings. See
 48 | 
 49 | http://urwid.org/manual/displayattributes.html#display-attributes
 50 | 
 51 | for color options and allowed values. Set display width with `--width`.
 52 | 
 53 | \fB7.\fR \fBu\fR will unescape the highlighted URL if necessary.
 54 | 
 55 | \fB8.\fR Run a command with the selected URL as the argument or pipe the
 56 | selected URL to a command using the \fB--run-safe\fR, \fB--run\fR and
 57 | \fB--pipe\fR arguments.
 58 | 
 59 | \fB9.\fR Use \fBl\fR to cycle through whether URLs are opened using the Python
 60 | webbrowser module (default), xdg-open (if installed) or a function passed on the
 61 | command line with \fB--run-safe\fR or \fB--run\fR. The \fB--run\fR and
 62 | \fB--run-safe\fR functions will respect the value of \fB--pipe\fR.
 63 | 
 64 | \fB10.\fR \fBF1\fR shows the help menu.
 65 | 
 66 | \fB11.\fR Scan certain email headers for URLs. Currently \fBLink\fR,
 67 | \fBArchived-At\fR and \fBList-*\fR are scanned when \fB--headers\fR is passed.
 68 | 
 69 | \fB12.\fR Queue multiple URLs for opening and open them all at once with \fBa\fR and \fBo\fR.
 70 | 
 71 | .SH OPTIONS
 72 | .TP
 73 | .BR \-c ", " \-\-compact
 74 | Compact display mode; don't show the context of each URL.
 75 | .TP
 76 | .B \-d, \-\-dedupe
 77 | Remove duplicated URLs from the list of URLs.
 78 | .TP
 79 | .B \-E, \-\-regex \<expression\>
 80 | Use \<expression\> in place of the default set of regular expressions,
 81 | to be used for any kind of matching. This is useful for example when
 82 | selectively avoiding 'mailto:' links or any other pattern that urlscan
 83 | could interpret as urls (such as '<filename>.<extension>'). Usage
 84 | example:
 85 | 
 86 |     $ urlscan --regex 'https?://.+\.\w+' file.txt
 87 | .TP
 88 | .B \-f, \-\-run\-safe \<expression\>
 89 | Execute \<expression\> in place of opening URL with a browser. Use {} in
 90 | \<expression\> to substitute in the URL. Examples:
 91 | 
 92 |     $ urlscan --run-safe 'tmux set buffer {}'
 93 | .TP
 94 | .B \-g, \-\-genconf
 95 | Generate ~/.config/urlscan/config.json with default options.
 96 | .TP
 97 | .B \-H, \-\-nohelp
 98 | Start with header menu hidden.
 99 | .TP
100 | .BR \-\-headers
101 | Scan certain message headers for URLs. Currently Link, Archived-At and
102 | List-* are scanned.
103 | .TP
104 | .B \-n, \-\-no-browser
105 | Disables the selection interface and print the links to standard output.
106 | Useful for scripting (implies \fB\-\-compact\fR).
107 | .TP
108 | .B \-p, \-\-pipe
109 | Pipe the selected URL to the command specified by `--run-safe` or `--run`. This
110 | is preferred when the command supports it, as it is more secure and tolerant of
111 | special characters in the URL. Example:
112 | 
113 |     $ urlscan --run-safe 'xclip -i' --pipe file.txt
114 | .TP
115 | .B \-R, \-\-reverse
116 | Reverse displayed order of URLs.
117 | .TP
118 | .B \-r, \-\-run \<expression\>
119 | Execute \<expression\> in place of opening URL with a browser. Use {} in
120 | \<expression\> to substitute in the URL. Shell features such as | and \> can be
121 | used, but it is less secure. Examples:
122 | 
123 |     $ urlscan --run 'echo {} | xclip -i' file.txt
124 | .TP
125 | .B \-s, \-\-single
126 | Exit urlscan after opening or copying a single browser link.
127 | 
128 |     $ urlscan -s file.txt
129 | .TP
130 | .B \-w, \-\-width
131 | Set display width.
132 | .TP
133 | .BR \-W ", " \-\-whitespace\-off
134 | Don't display empty lines and ellipses.
135 | .TP
136 | .BR \-C ", " \-\-colors " {true,256,88,16,8,mono}"
137 | Set the urwid color mode. Default is "16". "true" = 2**24 colors and "mono" = 1 color.
138 | .TP
139 | .BR \-V ", " \-\-version
140 | Print urlscan version.
141 | 
142 | .SH MUTT INTEGRATION
143 | 
144 | To integrate urlscan with mutt, include the following two commands in
145 | \fB~/.muttrc\fR:
146 | 
147 | 
148 | .ad l
149 | macro index,pager \\cb "<pipe-message> urlscan<Enter>" "call urlscan to extract URLs out of a message"
150 | 
151 | macro attach,compose \\cb "<pipe-entry> urlscan<Enter>" "call urlscan to extract URLs out of a message"
152 | .ad b
153 | 
154 | 
155 | Once these lines are in your mutt configuration file, pressing
156 | Control-b will allow you to browse and open the URLs in the currently
157 | selected message.
158 | 
159 | Alternately, you can pipe a message into urlscan using the '|' operator. This
160 | can be useful for applying a different flag (such as the '-d' or '-c' options).
161 | 
162 | .SH KEYBINDINGS
163 | 
164 | Run \fBurlscan \-g\fR to generate ~/.config/urlscan/config.json. All of the keys
165 | will be listed. You can either leave in place or delete any that will not be
166 | altered.
167 | 
168 | To unset a binding, set it equal to "". For example: \fB"P": ""\fR
169 | 
170 | The follow actions are supported:
171 | .TP
172 | \fBadd_url\fR \-\- add a URL to the queue (Default: \fBa\fR)
173 | .TP
174 | \fBall_escape\fR \-\- toggle unescape all URLs (Default: \fBu\fR)
175 | .TP
176 | \fBall_shorten\fR \-\- toggle shorten all URLs (Default: \fBS\fR)
177 | .TP
178 | \fBbottom\fR \-\- move cursor to last item (Default: \fBG\fR)
179 | .TP
180 | \fBclear_screen\fR \-\- redraw screen (Default: \fBCtrl-l\fR)
181 | .TP
182 | \fBclipboard\fR \-\- copy highlighted URL to clipboard using xsel/xclip (Default: \fBC\fR)
183 | .TP
184 | \fBclipboard_pri\fR \-\- copy highlighted URL to primary selection using xsel/xclip (Default: \fBP\fR)
185 | .TP
186 | \fBcontext\fR \-\- show/hide context (Default: \fBc\fR)
187 | .TP
188 | \fBdel_url\fR \-\- delete URL from the queue (Default: \fBd\fR)
189 | .TP
190 | \fBdown\fR \-\- cursor down (Default: \fBj\fR)
191 | .TP
192 | \fBhelp_menu\fR \-\- show/hide help menu (Default: \fBF1\fR)
193 | .TP
194 | \fBlink_handler\fR \-\- cycle link handling (webbrowser, xdg-open or custom) (Default: \fBl\fR)
195 | .TP
196 | \fBnext\fR \-\- jump to next URL (Default: \fBJ\fR)
197 | .TP
198 | \fBopen_queue\fR \-\- open all URLs in queue (Default: \fBo\fR)
199 | .TP
200 | \fBopen_queue_win\fR \-\- open all URLs in queue in new window (Default: \fBO\fR)
201 | .TP
202 | \fBopen_url\fR \-\- open selected URL (Default: \fBspace\fR or \fBenter\fR)
203 | .TP
204 | \fBpalette\fR \-\- cycle through palettes (Default: \fBp\fR)
205 | .TP
206 | \fBprevious\fR \-\- jump to previous URL (Default: \fBK\fR)
207 | .TP
208 | \fBquit\fR \-\- quit (Default: \fBq\fR or \fBQ\fR)
209 | .TP
210 | \fBreverse\fR \-\- reverse display order (Default: \fBR\fR)
211 | .TP
212 | \fBshorten\fR \-\- toggle shorten highlighted URL (Default: \fBs\fR)
213 | .TP
214 | \fBtop\fR \-\- move to first list item (Default: \fBg\fR)
215 | .TP
216 | \fBup\fR \-\- cursor up (Default: \fBk\fR)
217 | 
218 | .SH FILES
219 | 
220 | $HOME/.config/urlscan/config.json
221 | 
222 | Only required if additional or modified palettes or keybindings are desired.
223 | 
224 | .SH SEE ALSO
225 | \fI/usr/share/doc/urlscan/README\fR,
226 | \fBurlview\fR(1),
227 | \fBmutt\fR(1)
228 | 
229 | .SH AUTHOR
230 | This manual page was written by Daniel Burrows <dburrows@debian.org> and Scott Hansen <tech@firecat53.net>
231 | 


--------------------------------------------------------------------------------
/urlscan/__init__.py:
--------------------------------------------------------------------------------
1 | __all__ = ['browser', 'urlchoose', 'urlscan']
2 | 


--------------------------------------------------------------------------------
/urlscan/__main__.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | """ A simple urlview replacement that handles things like quoted-printable
  3 | properly.
  4 | 
  5 | """
  6 | #
  7 | #   Copyright (C) 2006-2007 Daniel Burrows
  8 | #   Copyright (C) 2023 Scott Hansen
  9 | #
 10 | # This program is free software; you can redistribute it and/or
 11 | # modify it under the terms of the GNU General Public License
 12 | # as published by the Free Software Foundation; either version 2
 13 | # of the License, or (at your option) any later version.
 14 | #
 15 | # This program is distributed in the hope that it will be useful,
 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 18 | # GNU General Public License for more details.
 19 | #
 20 | # You should have received a copy of the GNU General Public License
 21 | # along with this program; if not, write to the Free Software
 22 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
 23 | 
 24 | import argparse
 25 | import io
 26 | import os
 27 | import sys
 28 | from email import policy
 29 | from email.parser import BytesParser
 30 | from urlscan import urlchoose, urlscan
 31 | from urlscan._version import version
 32 | 
 33 | 
 34 | def parse_arguments():
 35 |     """Parse command line options.
 36 | 
 37 |     Returns: args
 38 | 
 39 |     """
 40 |     arg_parse = argparse.ArgumentParser(description="Parse and display URLs")
 41 |     arg_parse.add_argument('--genconf', '-g',
 42 |                            action='store_true', default=False,
 43 |                            help="Generate config file and exit.")
 44 |     arg_parse.add_argument('--compact', '-c',
 45 |                            action='store_true', default=False,
 46 |                            help="Don't display the context of each URL.")
 47 |     arg_parse.add_argument('--reverse', '-R', dest="reverse",
 48 |                            action='store_true', default=False,
 49 |                            help="Reverse order of displayed URLS/context")
 50 |     arg_parse.add_argument('--no-browser', '-n', dest="nobrowser",
 51 |                            action='store_true', default=False,
 52 |                            help="Pipe URLs to stdout")
 53 |     arg_parse.add_argument('--dedupe', '-d', dest="dedupe",
 54 |                            action='store_true', default=False,
 55 |                            help="Remove duplicate URLs from list")
 56 |     arg_parse.add_argument('--regex', '-E',
 57 |                            help="Alternate custom regex to be used for all "
 58 |                            "kinds of matching. "
 59 |                            r"For example: --regex 'https?://.+\.\w+'")
 60 |     arg_parse.add_argument('--run', '-r',
 61 |                            help="Alternate command to run on selected URL "
 62 |                            "instead of opening URL in browser. Use {} to "
 63 |                            "represent the URL value in the expression. "
 64 |                            "For example: --run 'echo {} | xclip -i'")
 65 |     arg_parse.add_argument('--run-safe', '-f', dest="runsafe",
 66 |                            help="Alternate command to run on selected URL "
 67 |                            "instead of opening URL in browser. Use {} to "
 68 |                            "represent the URL value in the expression. Safest "
 69 |                            "run option but uses `shell=False` which does not "
 70 |                            "allow use of shell features like | or >. Can use "
 71 |                            "with --pipe.")
 72 |     arg_parse.add_argument('--pipe', '-p', dest='pipe',
 73 |                            action='store_true', default=False,
 74 |                            help="Pipe URL into the command specified by --run or --run-safe")
 75 |     arg_parse.add_argument('--nohelp', '-H', dest='nohelp',
 76 |                            action='store_true', default=False,
 77 |                            help='Hide help menu by default')
 78 |     arg_parse.add_argument('--single', '-s', dest='single',
 79 |                            action='store_true', default=False,
 80 |                            help='Quit urlscan after opening/copying a single link.')
 81 |     arg_parse.add_argument('--width', '-w', dest='width',
 82 |                            type=int, default=0,
 83 |                            help='Set width to display')
 84 |     arg_parse.add_argument('--whitespace-off', '-W', dest='whitespaceoff',
 85 |                            action='store_true', default=False,
 86 |                            help="Don't display empty lines and ellipses.")
 87 |     arg_parse.add_argument('--headers', dest='headers',
 88 |                            action='store_true', default=False,
 89 |                            help='Scan certain message headers for URLs.')
 90 |     arg_parse.add_argument('--colors', '-C',
 91 |                            choices=['true', '256', '88', '16', '8', 'mono'],
 92 |                            default='16',
 93 |                            help='Set the terminal color palette (true=2**24, mono=1)')
 94 |     arg_parse.add_argument('--version', '-V', dest='version',
 95 |                            action='store_true', default=False,
 96 |                            help='Print urlscan version')
 97 |     arg_parse.add_argument('message', nargs='?', default=sys.stdin,
 98 |                            help="Filename of the message to parse")
 99 |     return arg_parse.parse_args()
100 | 
101 | 
102 | def close_stdin():
103 |     """This section closes out sys.stdin if necessary so as not to block curses
104 |     keyboard inputs
105 | 
106 |     """
107 |     if not os.isatty(0):
108 |         try:
109 |             fdesc = os.open('/dev/tty', os.O_RDONLY)
110 |         except OSError:
111 |             # This is most likely a non-interactive session, try to open
112 |             # `stdin` directly
113 |             fdesc = os.open('/dev/stdin', os.O_RDONLY)
114 | 
115 |         if fdesc < 0:
116 |             sys.stderr.write('Unable to open an input tty.\n')
117 |             sys.exit(-1)
118 |         else:
119 |             os.dup2(fdesc, 0)
120 |             os.close(fdesc)
121 | 
122 | 
123 | def process_input(fname):
124 |     """Return the parsed text of stdin or the message. Accounts for possible
125 |     file encoding differences.
126 | 
127 |         Args: fname - filename or sys.stdin
128 |         Returns: mesg - EmailMessage object
129 | 
130 |     """
131 |     if fname is sys.stdin:
132 |         try:
133 |             stdin_file = fname.buffer.read()
134 |         except AttributeError:
135 |             stdin_file = fname.read()
136 |     else:
137 |         stdin_file = None
138 |     if stdin_file is not None:
139 |         fobj = io.BytesIO(stdin_file)
140 |     else:
141 |         fobj = io.open(fname, mode='rb')
142 |     f_keep = fobj
143 |     mesg = BytesParser(policy=policy.default.clone(utf8=True)).parse(fobj)
144 |     if 'From' not in mesg.keys() and 'Date' not in mesg.keys():
145 |         # If it's not an email message, don't let the email parser
146 |         # delete the first line. If it is, let the parser do its job so
147 |         # we don't get mailto: links for all the To and From addresses
148 |         fobj = _fix_first_line(f_keep)
149 |         mesg = BytesParser(policy=policy.default.clone(utf8=True)).parse(fobj)
150 |     try:
151 |         fobj.close()
152 |     except NameError:
153 |         pass
154 |     close_stdin()
155 |     return mesg
156 | 
157 | 
158 | def _fix_first_line(fline):
159 |     """If the first line starts with http* or [ or other non-text characters,
160 |     the URLs on that line will not be parsed by email.Parser. Add a blank line
161 |     at the top of the file to ensure everything is read in a non-email file.
162 | 
163 |     """
164 |     fline.seek(0)
165 |     new = io.BytesIO()
166 |     new.write(b"\n" + fline.read())
167 |     fline.close()
168 |     new.seek(0)
169 |     return new
170 | 
171 | 
172 | def main():
173 |     """Entrypoint function for urlscan
174 | 
175 |     """
176 |     args = parse_arguments()
177 |     if args.version is True:
178 |         print(version)
179 |         return
180 |     if args.genconf is True:
181 |         urlchoose.URLChooser([], genconf=True)
182 |         return
183 |     msg = process_input(args.message)
184 |     if args.nobrowser is False:
185 |         tui = urlchoose.URLChooser(urlscan.msgurls(msg, regex=args.regex, headers=args.headers),
186 |                                    compact=args.compact,
187 |                                    reverse=args.reverse,
188 |                                    nohelp=args.nohelp,
189 |                                    dedupe=args.dedupe,
190 |                                    run=args.run,
191 |                                    runsafe=args.runsafe,
192 |                                    single=args.single,
193 |                                    width=args.width,
194 |                                    whitespaceoff=args.whitespaceoff,
195 |                                    pipe=args.pipe,
196 |                                    colors=args.colors)
197 |         tui.main()
198 |     else:
199 |         out = urlchoose.URLChooser(urlscan.msgurls(msg, regex=args.regex, headers=args.headers),
200 |                                    dedupe=args.dedupe,
201 |                                    reverse=args.reverse,
202 |                                    shorten=False)
203 |         if args.reverse is True:
204 |             out.urls.reverse()
205 |         print("\n".join(out.urls))
206 | 
207 | 
208 | if __name__ == "__main__":
209 |     main()
210 | 


--------------------------------------------------------------------------------
/urlscan/assets/tlds-alpha-by-domain.txt:
--------------------------------------------------------------------------------
   1 | # Version 2025051700, Last Updated Sat May 17 07:07:01 2025 UTC
   2 | AAA
   3 | AARP
   4 | ABB
   5 | ABBOTT
   6 | ABBVIE
   7 | ABC
   8 | ABLE
   9 | ABOGADO
  10 | ABUDHABI
  11 | AC
  12 | ACADEMY
  13 | ACCENTURE
  14 | ACCOUNTANT
  15 | ACCOUNTANTS
  16 | ACO
  17 | ACTOR
  18 | AD
  19 | ADS
  20 | ADULT
  21 | AE
  22 | AEG
  23 | AERO
  24 | AETNA
  25 | AF
  26 | AFL
  27 | AFRICA
  28 | AG
  29 | AGAKHAN
  30 | AGENCY
  31 | AI
  32 | AIG
  33 | AIRBUS
  34 | AIRFORCE
  35 | AIRTEL
  36 | AKDN
  37 | AL
  38 | ALIBABA
  39 | ALIPAY
  40 | ALLFINANZ
  41 | ALLSTATE
  42 | ALLY
  43 | ALSACE
  44 | ALSTOM
  45 | AM
  46 | AMAZON
  47 | AMERICANEXPRESS
  48 | AMERICANFAMILY
  49 | AMEX
  50 | AMFAM
  51 | AMICA
  52 | AMSTERDAM
  53 | ANALYTICS
  54 | ANDROID
  55 | ANQUAN
  56 | ANZ
  57 | AO
  58 | AOL
  59 | APARTMENTS
  60 | APP
  61 | APPLE
  62 | AQ
  63 | AQUARELLE
  64 | AR
  65 | ARAB
  66 | ARAMCO
  67 | ARCHI
  68 | ARMY
  69 | ARPA
  70 | ART
  71 | ARTE
  72 | AS
  73 | ASDA
  74 | ASIA
  75 | ASSOCIATES
  76 | AT
  77 | ATHLETA
  78 | ATTORNEY
  79 | AU
  80 | AUCTION
  81 | AUDI
  82 | AUDIBLE
  83 | AUDIO
  84 | AUSPOST
  85 | AUTHOR
  86 | AUTO
  87 | AUTOS
  88 | AW
  89 | AWS
  90 | AX
  91 | AXA
  92 | AZ
  93 | AZURE
  94 | BA
  95 | BABY
  96 | BAIDU
  97 | BANAMEX
  98 | BAND
  99 | BANK
 100 | BAR
 101 | BARCELONA
 102 | BARCLAYCARD
 103 | BARCLAYS
 104 | BAREFOOT
 105 | BARGAINS
 106 | BASEBALL
 107 | BASKETBALL
 108 | BAUHAUS
 109 | BAYERN
 110 | BB
 111 | BBC
 112 | BBT
 113 | BBVA
 114 | BCG
 115 | BCN
 116 | BD
 117 | BE
 118 | BEATS
 119 | BEAUTY
 120 | BEER
 121 | BERLIN
 122 | BEST
 123 | BESTBUY
 124 | BET
 125 | BF
 126 | BG
 127 | BH
 128 | BHARTI
 129 | BI
 130 | BIBLE
 131 | BID
 132 | BIKE
 133 | BING
 134 | BINGO
 135 | BIO
 136 | BIZ
 137 | BJ
 138 | BLACK
 139 | BLACKFRIDAY
 140 | BLOCKBUSTER
 141 | BLOG
 142 | BLOOMBERG
 143 | BLUE
 144 | BM
 145 | BMS
 146 | BMW
 147 | BN
 148 | BNPPARIBAS
 149 | BO
 150 | BOATS
 151 | BOEHRINGER
 152 | BOFA
 153 | BOM
 154 | BOND
 155 | BOO
 156 | BOOK
 157 | BOOKING
 158 | BOSCH
 159 | BOSTIK
 160 | BOSTON
 161 | BOT
 162 | BOUTIQUE
 163 | BOX
 164 | BR
 165 | BRADESCO
 166 | BRIDGESTONE
 167 | BROADWAY
 168 | BROKER
 169 | BROTHER
 170 | BRUSSELS
 171 | BS
 172 | BT
 173 | BUILD
 174 | BUILDERS
 175 | BUSINESS
 176 | BUY
 177 | BUZZ
 178 | BV
 179 | BW
 180 | BY
 181 | BZ
 182 | BZH
 183 | CA
 184 | CAB
 185 | CAFE
 186 | CAL
 187 | CALL
 188 | CALVINKLEIN
 189 | CAM
 190 | CAMERA
 191 | CAMP
 192 | CANON
 193 | CAPETOWN
 194 | CAPITAL
 195 | CAPITALONE
 196 | CAR
 197 | CARAVAN
 198 | CARDS
 199 | CARE
 200 | CAREER
 201 | CAREERS
 202 | CARS
 203 | CASA
 204 | CASE
 205 | CASH
 206 | CASINO
 207 | CAT
 208 | CATERING
 209 | CATHOLIC
 210 | CBA
 211 | CBN
 212 | CBRE
 213 | CC
 214 | CD
 215 | CENTER
 216 | CEO
 217 | CERN
 218 | CF
 219 | CFA
 220 | CFD
 221 | CG
 222 | CH
 223 | CHANEL
 224 | CHANNEL
 225 | CHARITY
 226 | CHASE
 227 | CHAT
 228 | CHEAP
 229 | CHINTAI
 230 | CHRISTMAS
 231 | CHROME
 232 | CHURCH
 233 | CI
 234 | CIPRIANI
 235 | CIRCLE
 236 | CISCO
 237 | CITADEL
 238 | CITI
 239 | CITIC
 240 | CITY
 241 | CK
 242 | CL
 243 | CLAIMS
 244 | CLEANING
 245 | CLICK
 246 | CLINIC
 247 | CLINIQUE
 248 | CLOTHING
 249 | CLOUD
 250 | CLUB
 251 | CLUBMED
 252 | CM
 253 | CN
 254 | CO
 255 | COACH
 256 | CODES
 257 | COFFEE
 258 | COLLEGE
 259 | COLOGNE
 260 | COM
 261 | COMMBANK
 262 | COMMUNITY
 263 | COMPANY
 264 | COMPARE
 265 | COMPUTER
 266 | COMSEC
 267 | CONDOS
 268 | CONSTRUCTION
 269 | CONSULTING
 270 | CONTACT
 271 | CONTRACTORS
 272 | COOKING
 273 | COOL
 274 | COOP
 275 | CORSICA
 276 | COUNTRY
 277 | COUPON
 278 | COUPONS
 279 | COURSES
 280 | CPA
 281 | CR
 282 | CREDIT
 283 | CREDITCARD
 284 | CREDITUNION
 285 | CRICKET
 286 | CROWN
 287 | CRS
 288 | CRUISE
 289 | CRUISES
 290 | CU
 291 | CUISINELLA
 292 | CV
 293 | CW
 294 | CX
 295 | CY
 296 | CYMRU
 297 | CYOU
 298 | CZ
 299 | DAD
 300 | DANCE
 301 | DATA
 302 | DATE
 303 | DATING
 304 | DATSUN
 305 | DAY
 306 | DCLK
 307 | DDS
 308 | DE
 309 | DEAL
 310 | DEALER
 311 | DEALS
 312 | DEGREE
 313 | DELIVERY
 314 | DELL
 315 | DELOITTE
 316 | DELTA
 317 | DEMOCRAT
 318 | DENTAL
 319 | DENTIST
 320 | DESI
 321 | DESIGN
 322 | DEV
 323 | DHL
 324 | DIAMONDS
 325 | DIET
 326 | DIGITAL
 327 | DIRECT
 328 | DIRECTORY
 329 | DISCOUNT
 330 | DISCOVER
 331 | DISH
 332 | DIY
 333 | DJ
 334 | DK
 335 | DM
 336 | DNP
 337 | DO
 338 | DOCS
 339 | DOCTOR
 340 | DOG
 341 | DOMAINS
 342 | DOT
 343 | DOWNLOAD
 344 | DRIVE
 345 | DTV
 346 | DUBAI
 347 | DUNLOP
 348 | DUPONT
 349 | DURBAN
 350 | DVAG
 351 | DVR
 352 | DZ
 353 | EARTH
 354 | EAT
 355 | EC
 356 | ECO
 357 | EDEKA
 358 | EDU
 359 | EDUCATION
 360 | EE
 361 | EG
 362 | EMAIL
 363 | EMERCK
 364 | ENERGY
 365 | ENGINEER
 366 | ENGINEERING
 367 | ENTERPRISES
 368 | EPSON
 369 | EQUIPMENT
 370 | ER
 371 | ERICSSON
 372 | ERNI
 373 | ES
 374 | ESQ
 375 | ESTATE
 376 | ET
 377 | EU
 378 | EUROVISION
 379 | EUS
 380 | EVENTS
 381 | EXCHANGE
 382 | EXPERT
 383 | EXPOSED
 384 | EXPRESS
 385 | EXTRASPACE
 386 | FAGE
 387 | FAIL
 388 | FAIRWINDS
 389 | FAITH
 390 | FAMILY
 391 | FAN
 392 | FANS
 393 | FARM
 394 | FARMERS
 395 | FASHION
 396 | FAST
 397 | FEDEX
 398 | FEEDBACK
 399 | FERRARI
 400 | FERRERO
 401 | FI
 402 | FIDELITY
 403 | FIDO
 404 | FILM
 405 | FINAL
 406 | FINANCE
 407 | FINANCIAL
 408 | FIRE
 409 | FIRESTONE
 410 | FIRMDALE
 411 | FISH
 412 | FISHING
 413 | FIT
 414 | FITNESS
 415 | FJ
 416 | FK
 417 | FLICKR
 418 | FLIGHTS
 419 | FLIR
 420 | FLORIST
 421 | FLOWERS
 422 | FLY
 423 | FM
 424 | FO
 425 | FOO
 426 | FOOD
 427 | FOOTBALL
 428 | FORD
 429 | FOREX
 430 | FORSALE
 431 | FORUM
 432 | FOUNDATION
 433 | FOX
 434 | FR
 435 | FREE
 436 | FRESENIUS
 437 | FRL
 438 | FROGANS
 439 | FRONTIER
 440 | FTR
 441 | FUJITSU
 442 | FUN
 443 | FUND
 444 | FURNITURE
 445 | FUTBOL
 446 | FYI
 447 | GA
 448 | GAL
 449 | GALLERY
 450 | GALLO
 451 | GALLUP
 452 | GAME
 453 | GAMES
 454 | GAP
 455 | GARDEN
 456 | GAY
 457 | GB
 458 | GBIZ
 459 | GD
 460 | GDN
 461 | GE
 462 | GEA
 463 | GENT
 464 | GENTING
 465 | GEORGE
 466 | GF
 467 | GG
 468 | GGEE
 469 | GH
 470 | GI
 471 | GIFT
 472 | GIFTS
 473 | GIVES
 474 | GIVING
 475 | GL
 476 | GLASS
 477 | GLE
 478 | GLOBAL
 479 | GLOBO
 480 | GM
 481 | GMAIL
 482 | GMBH
 483 | GMO
 484 | GMX
 485 | GN
 486 | GODADDY
 487 | GOLD
 488 | GOLDPOINT
 489 | GOLF
 490 | GOO
 491 | GOODYEAR
 492 | GOOG
 493 | GOOGLE
 494 | GOP
 495 | GOT
 496 | GOV
 497 | GP
 498 | GQ
 499 | GR
 500 | GRAINGER
 501 | GRAPHICS
 502 | GRATIS
 503 | GREEN
 504 | GRIPE
 505 | GROCERY
 506 | GROUP
 507 | GS
 508 | GT
 509 | GU
 510 | GUCCI
 511 | GUGE
 512 | GUIDE
 513 | GUITARS
 514 | GURU
 515 | GW
 516 | GY
 517 | HAIR
 518 | HAMBURG
 519 | HANGOUT
 520 | HAUS
 521 | HBO
 522 | HDFC
 523 | HDFCBANK
 524 | HEALTH
 525 | HEALTHCARE
 526 | HELP
 527 | HELSINKI
 528 | HERE
 529 | HERMES
 530 | HIPHOP
 531 | HISAMITSU
 532 | HITACHI
 533 | HIV
 534 | HK
 535 | HKT
 536 | HM
 537 | HN
 538 | HOCKEY
 539 | HOLDINGS
 540 | HOLIDAY
 541 | HOMEDEPOT
 542 | HOMEGOODS
 543 | HOMES
 544 | HOMESENSE
 545 | HONDA
 546 | HORSE
 547 | HOSPITAL
 548 | HOST
 549 | HOSTING
 550 | HOT
 551 | HOTELS
 552 | HOTMAIL
 553 | HOUSE
 554 | HOW
 555 | HR
 556 | HSBC
 557 | HT
 558 | HU
 559 | HUGHES
 560 | HYATT
 561 | HYUNDAI
 562 | IBM
 563 | ICBC
 564 | ICE
 565 | ICU
 566 | ID
 567 | IE
 568 | IEEE
 569 | IFM
 570 | IKANO
 571 | IL
 572 | IM
 573 | IMAMAT
 574 | IMDB
 575 | IMMO
 576 | IMMOBILIEN
 577 | IN
 578 | INC
 579 | INDUSTRIES
 580 | INFINITI
 581 | INFO
 582 | ING
 583 | INK
 584 | INSTITUTE
 585 | INSURANCE
 586 | INSURE
 587 | INT
 588 | INTERNATIONAL
 589 | INTUIT
 590 | INVESTMENTS
 591 | IO
 592 | IPIRANGA
 593 | IQ
 594 | IR
 595 | IRISH
 596 | IS
 597 | ISMAILI
 598 | IST
 599 | ISTANBUL
 600 | IT
 601 | ITAU
 602 | ITV
 603 | JAGUAR
 604 | JAVA
 605 | JCB
 606 | JE
 607 | JEEP
 608 | JETZT
 609 | JEWELRY
 610 | JIO
 611 | JLL
 612 | JM
 613 | JMP
 614 | JNJ
 615 | JO
 616 | JOBS
 617 | JOBURG
 618 | JOT
 619 | JOY
 620 | JP
 621 | JPMORGAN
 622 | JPRS
 623 | JUEGOS
 624 | JUNIPER
 625 | KAUFEN
 626 | KDDI
 627 | KE
 628 | KERRYHOTELS
 629 | KERRYPROPERTIES
 630 | KFH
 631 | KG
 632 | KH
 633 | KI
 634 | KIA
 635 | KIDS
 636 | KIM
 637 | KINDLE
 638 | KITCHEN
 639 | KIWI
 640 | KM
 641 | KN
 642 | KOELN
 643 | KOMATSU
 644 | KOSHER
 645 | KP
 646 | KPMG
 647 | KPN
 648 | KR
 649 | KRD
 650 | KRED
 651 | KUOKGROUP
 652 | KW
 653 | KY
 654 | KYOTO
 655 | KZ
 656 | LA
 657 | LACAIXA
 658 | LAMBORGHINI
 659 | LAMER
 660 | LAND
 661 | LANDROVER
 662 | LANXESS
 663 | LASALLE
 664 | LAT
 665 | LATINO
 666 | LATROBE
 667 | LAW
 668 | LAWYER
 669 | LB
 670 | LC
 671 | LDS
 672 | LEASE
 673 | LECLERC
 674 | LEFRAK
 675 | LEGAL
 676 | LEGO
 677 | LEXUS
 678 | LGBT
 679 | LI
 680 | LIDL
 681 | LIFE
 682 | LIFEINSURANCE
 683 | LIFESTYLE
 684 | LIGHTING
 685 | LIKE
 686 | LILLY
 687 | LIMITED
 688 | LIMO
 689 | LINCOLN
 690 | LINK
 691 | LIVE
 692 | LIVING
 693 | LK
 694 | LLC
 695 | LLP
 696 | LOAN
 697 | LOANS
 698 | LOCKER
 699 | LOCUS
 700 | LOL
 701 | LONDON
 702 | LOTTE
 703 | LOTTO
 704 | LOVE
 705 | LPL
 706 | LPLFINANCIAL
 707 | LR
 708 | LS
 709 | LT
 710 | LTD
 711 | LTDA
 712 | LU
 713 | LUNDBECK
 714 | LUXE
 715 | LUXURY
 716 | LV
 717 | LY
 718 | MA
 719 | MADRID
 720 | MAIF
 721 | MAISON
 722 | MAKEUP
 723 | MAN
 724 | MANAGEMENT
 725 | MANGO
 726 | MAP
 727 | MARKET
 728 | MARKETING
 729 | MARKETS
 730 | MARRIOTT
 731 | MARSHALLS
 732 | MATTEL
 733 | MBA
 734 | MC
 735 | MCKINSEY
 736 | MD
 737 | ME
 738 | MED
 739 | MEDIA
 740 | MEET
 741 | MELBOURNE
 742 | MEME
 743 | MEMORIAL
 744 | MEN
 745 | MENU
 746 | MERCKMSD
 747 | MG
 748 | MH
 749 | MIAMI
 750 | MICROSOFT
 751 | MIL
 752 | MINI
 753 | MINT
 754 | MIT
 755 | MITSUBISHI
 756 | MK
 757 | ML
 758 | MLB
 759 | MLS
 760 | MM
 761 | MMA
 762 | MN
 763 | MO
 764 | MOBI
 765 | MOBILE
 766 | MODA
 767 | MOE
 768 | MOI
 769 | MOM
 770 | MONASH
 771 | MONEY
 772 | MONSTER
 773 | MORMON
 774 | MORTGAGE
 775 | MOSCOW
 776 | MOTO
 777 | MOTORCYCLES
 778 | MOV
 779 | MOVIE
 780 | MP
 781 | MQ
 782 | MR
 783 | MS
 784 | MSD
 785 | MT
 786 | MTN
 787 | MTR
 788 | MU
 789 | MUSEUM
 790 | MUSIC
 791 | MV
 792 | MW
 793 | MX
 794 | MY
 795 | MZ
 796 | NA
 797 | NAB
 798 | NAGOYA
 799 | NAME
 800 | NAVY
 801 | NBA
 802 | NC
 803 | NE
 804 | NEC
 805 | NET
 806 | NETBANK
 807 | NETFLIX
 808 | NETWORK
 809 | NEUSTAR
 810 | NEW
 811 | NEWS
 812 | NEXT
 813 | NEXTDIRECT
 814 | NEXUS
 815 | NF
 816 | NFL
 817 | NG
 818 | NGO
 819 | NHK
 820 | NI
 821 | NICO
 822 | NIKE
 823 | NIKON
 824 | NINJA
 825 | NISSAN
 826 | NISSAY
 827 | NL
 828 | NO
 829 | NOKIA
 830 | NORTON
 831 | NOW
 832 | NOWRUZ
 833 | NOWTV
 834 | NP
 835 | NR
 836 | NRA
 837 | NRW
 838 | NTT
 839 | NU
 840 | NYC
 841 | NZ
 842 | OBI
 843 | OBSERVER
 844 | OFFICE
 845 | OKINAWA
 846 | OLAYAN
 847 | OLAYANGROUP
 848 | OLLO
 849 | OM
 850 | OMEGA
 851 | ONE
 852 | ONG
 853 | ONL
 854 | ONLINE
 855 | OOO
 856 | OPEN
 857 | ORACLE
 858 | ORANGE
 859 | ORG
 860 | ORGANIC
 861 | ORIGINS
 862 | OSAKA
 863 | OTSUKA
 864 | OTT
 865 | OVH
 866 | PA
 867 | PAGE
 868 | PANASONIC
 869 | PARIS
 870 | PARS
 871 | PARTNERS
 872 | PARTS
 873 | PARTY
 874 | PAY
 875 | PCCW
 876 | PE
 877 | PET
 878 | PF
 879 | PFIZER
 880 | PG
 881 | PH
 882 | PHARMACY
 883 | PHD
 884 | PHILIPS
 885 | PHONE
 886 | PHOTO
 887 | PHOTOGRAPHY
 888 | PHOTOS
 889 | PHYSIO
 890 | PICS
 891 | PICTET
 892 | PICTURES
 893 | PID
 894 | PIN
 895 | PING
 896 | PINK
 897 | PIONEER
 898 | PIZZA
 899 | PK
 900 | PL
 901 | PLACE
 902 | PLAY
 903 | PLAYSTATION
 904 | PLUMBING
 905 | PLUS
 906 | PM
 907 | PN
 908 | PNC
 909 | POHL
 910 | POKER
 911 | POLITIE
 912 | PORN
 913 | POST
 914 | PR
 915 | PRAXI
 916 | PRESS
 917 | PRIME
 918 | PRO
 919 | PROD
 920 | PRODUCTIONS
 921 | PROF
 922 | PROGRESSIVE
 923 | PROMO
 924 | PROPERTIES
 925 | PROPERTY
 926 | PROTECTION
 927 | PRU
 928 | PRUDENTIAL
 929 | PS
 930 | PT
 931 | PUB
 932 | PW
 933 | PWC
 934 | PY
 935 | QA
 936 | QPON
 937 | QUEBEC
 938 | QUEST
 939 | RACING
 940 | RADIO
 941 | RE
 942 | READ
 943 | REALESTATE
 944 | REALTOR
 945 | REALTY
 946 | RECIPES
 947 | RED
 948 | REDSTONE
 949 | REDUMBRELLA
 950 | REHAB
 951 | REISE
 952 | REISEN
 953 | REIT
 954 | RELIANCE
 955 | REN
 956 | RENT
 957 | RENTALS
 958 | REPAIR
 959 | REPORT
 960 | REPUBLICAN
 961 | REST
 962 | RESTAURANT
 963 | REVIEW
 964 | REVIEWS
 965 | REXROTH
 966 | RICH
 967 | RICHARDLI
 968 | RICOH
 969 | RIL
 970 | RIO
 971 | RIP
 972 | RO
 973 | ROCKS
 974 | RODEO
 975 | ROGERS
 976 | ROOM
 977 | RS
 978 | RSVP
 979 | RU
 980 | RUGBY
 981 | RUHR
 982 | RUN
 983 | RW
 984 | RWE
 985 | RYUKYU
 986 | SA
 987 | SAARLAND
 988 | SAFE
 989 | SAFETY
 990 | SAKURA
 991 | SALE
 992 | SALON
 993 | SAMSCLUB
 994 | SAMSUNG
 995 | SANDVIK
 996 | SANDVIKCOROMANT
 997 | SANOFI
 998 | SAP
 999 | SARL
1000 | SAS
1001 | SAVE
1002 | SAXO
1003 | SB
1004 | SBI
1005 | SBS
1006 | SC
1007 | SCB
1008 | SCHAEFFLER
1009 | SCHMIDT
1010 | SCHOLARSHIPS
1011 | SCHOOL
1012 | SCHULE
1013 | SCHWARZ
1014 | SCIENCE
1015 | SCOT
1016 | SD
1017 | SE
1018 | SEARCH
1019 | SEAT
1020 | SECURE
1021 | SECURITY
1022 | SEEK
1023 | SELECT
1024 | SENER
1025 | SERVICES
1026 | SEVEN
1027 | SEW
1028 | SEX
1029 | SEXY
1030 | SFR
1031 | SG
1032 | SH
1033 | SHANGRILA
1034 | SHARP
1035 | SHELL
1036 | SHIA
1037 | SHIKSHA
1038 | SHOES
1039 | SHOP
1040 | SHOPPING
1041 | SHOUJI
1042 | SHOW
1043 | SI
1044 | SILK
1045 | SINA
1046 | SINGLES
1047 | SITE
1048 | SJ
1049 | SK
1050 | SKI
1051 | SKIN
1052 | SKY
1053 | SKYPE
1054 | SL
1055 | SLING
1056 | SM
1057 | SMART
1058 | SMILE
1059 | SN
1060 | SNCF
1061 | SO
1062 | SOCCER
1063 | SOCIAL
1064 | SOFTBANK
1065 | SOFTWARE
1066 | SOHU
1067 | SOLAR
1068 | SOLUTIONS
1069 | SONG
1070 | SONY
1071 | SOY
1072 | SPA
1073 | SPACE
1074 | SPORT
1075 | SPOT
1076 | SR
1077 | SRL
1078 | SS
1079 | ST
1080 | STADA
1081 | STAPLES
1082 | STAR
1083 | STATEBANK
1084 | STATEFARM
1085 | STC
1086 | STCGROUP
1087 | STOCKHOLM
1088 | STORAGE
1089 | STORE
1090 | STREAM
1091 | STUDIO
1092 | STUDY
1093 | STYLE
1094 | SU
1095 | SUCKS
1096 | SUPPLIES
1097 | SUPPLY
1098 | SUPPORT
1099 | SURF
1100 | SURGERY
1101 | SUZUKI
1102 | SV
1103 | SWATCH
1104 | SWISS
1105 | SX
1106 | SY
1107 | SYDNEY
1108 | SYSTEMS
1109 | SZ
1110 | TAB
1111 | TAIPEI
1112 | TALK
1113 | TAOBAO
1114 | TARGET
1115 | TATAMOTORS
1116 | TATAR
1117 | TATTOO
1118 | TAX
1119 | TAXI
1120 | TC
1121 | TCI
1122 | TD
1123 | TDK
1124 | TEAM
1125 | TECH
1126 | TECHNOLOGY
1127 | TEL
1128 | TEMASEK
1129 | TENNIS
1130 | TEVA
1131 | TF
1132 | TG
1133 | TH
1134 | THD
1135 | THEATER
1136 | THEATRE
1137 | TIAA
1138 | TICKETS
1139 | TIENDA
1140 | TIPS
1141 | TIRES
1142 | TIROL
1143 | TJ
1144 | TJMAXX
1145 | TJX
1146 | TK
1147 | TKMAXX
1148 | TL
1149 | TM
1150 | TMALL
1151 | TN
1152 | TO
1153 | TODAY
1154 | TOKYO
1155 | TOOLS
1156 | TOP
1157 | TORAY
1158 | TOSHIBA
1159 | TOTAL
1160 | TOURS
1161 | TOWN
1162 | TOYOTA
1163 | TOYS
1164 | TR
1165 | TRADE
1166 | TRADING
1167 | TRAINING
1168 | TRAVEL
1169 | TRAVELERS
1170 | TRAVELERSINSURANCE
1171 | TRUST
1172 | TRV
1173 | TT
1174 | TUBE
1175 | TUI
1176 | TUNES
1177 | TUSHU
1178 | TV
1179 | TVS
1180 | TW
1181 | TZ
1182 | UA
1183 | UBANK
1184 | UBS
1185 | UG
1186 | UK
1187 | UNICOM
1188 | UNIVERSITY
1189 | UNO
1190 | UOL
1191 | UPS
1192 | US
1193 | UY
1194 | UZ
1195 | VA
1196 | VACATIONS
1197 | VANA
1198 | VANGUARD
1199 | VC
1200 | VE
1201 | VEGAS
1202 | VENTURES
1203 | VERISIGN
1204 | VERSICHERUNG
1205 | VET
1206 | VG
1207 | VI
1208 | VIAJES
1209 | VIDEO
1210 | VIG
1211 | VIKING
1212 | VILLAS
1213 | VIN
1214 | VIP
1215 | VIRGIN
1216 | VISA
1217 | VISION
1218 | VIVA
1219 | VIVO
1220 | VLAANDEREN
1221 | VN
1222 | VODKA
1223 | VOLVO
1224 | VOTE
1225 | VOTING
1226 | VOTO
1227 | VOYAGE
1228 | VU
1229 | WALES
1230 | WALMART
1231 | WALTER
1232 | WANG
1233 | WANGGOU
1234 | WATCH
1235 | WATCHES
1236 | WEATHER
1237 | WEATHERCHANNEL
1238 | WEBCAM
1239 | WEBER
1240 | WEBSITE
1241 | WED
1242 | WEDDING
1243 | WEIBO
1244 | WEIR
1245 | WF
1246 | WHOSWHO
1247 | WIEN
1248 | WIKI
1249 | WILLIAMHILL
1250 | WIN
1251 | WINDOWS
1252 | WINE
1253 | WINNERS
1254 | WME
1255 | WOLTERSKLUWER
1256 | WOODSIDE
1257 | WORK
1258 | WORKS
1259 | WORLD
1260 | WOW
1261 | WS
1262 | WTC
1263 | WTF
1264 | XBOX
1265 | XEROX
1266 | XIHUAN
1267 | XIN
1268 | XN--11B4C3D
1269 | XN--1CK2E1B
1270 | XN--1QQW23A
1271 | XN--2SCRJ9C
1272 | XN--30RR7Y
1273 | XN--3BST00M
1274 | XN--3DS443G
1275 | XN--3E0B707E
1276 | XN--3HCRJ9C
1277 | XN--3PXU8K
1278 | XN--42C2D9A
1279 | XN--45BR5CYL
1280 | XN--45BRJ9C
1281 | XN--45Q11C
1282 | XN--4DBRK0CE
1283 | XN--4GBRIM
1284 | XN--54B7FTA0CC
1285 | XN--55QW42G
1286 | XN--55QX5D
1287 | XN--5SU34J936BGSG
1288 | XN--5TZM5G
1289 | XN--6FRZ82G
1290 | XN--6QQ986B3XL
1291 | XN--80ADXHKS
1292 | XN--80AO21A
1293 | XN--80AQECDR1A
1294 | XN--80ASEHDB
1295 | XN--80ASWG
1296 | XN--8Y0A063A
1297 | XN--90A3AC
1298 | XN--90AE
1299 | XN--90AIS
1300 | XN--9DBQ2A
1301 | XN--9ET52U
1302 | XN--9KRT00A
1303 | XN--B4W605FERD
1304 | XN--BCK1B9A5DRE4C
1305 | XN--C1AVG
1306 | XN--C2BR7G
1307 | XN--CCK2B3B
1308 | XN--CCKWCXETD
1309 | XN--CG4BKI
1310 | XN--CLCHC0EA0B2G2A9GCD
1311 | XN--CZR694B
1312 | XN--CZRS0T
1313 | XN--CZRU2D
1314 | XN--D1ACJ3B
1315 | XN--D1ALF
1316 | XN--E1A4C
1317 | XN--ECKVDTC9D
1318 | XN--EFVY88H
1319 | XN--FCT429K
1320 | XN--FHBEI
1321 | XN--FIQ228C5HS
1322 | XN--FIQ64B
1323 | XN--FIQS8S
1324 | XN--FIQZ9S
1325 | XN--FJQ720A
1326 | XN--FLW351E
1327 | XN--FPCRJ9C3D
1328 | XN--FZC2C9E2C
1329 | XN--FZYS8D69UVGM
1330 | XN--G2XX48C
1331 | XN--GCKR3F0F
1332 | XN--GECRJ9C
1333 | XN--GK3AT1E
1334 | XN--H2BREG3EVE
1335 | XN--H2BRJ9C
1336 | XN--H2BRJ9C8C
1337 | XN--HXT814E
1338 | XN--I1B6B1A6A2E
1339 | XN--IMR513N
1340 | XN--IO0A7I
1341 | XN--J1AEF
1342 | XN--J1AMH
1343 | XN--J6W193G
1344 | XN--JLQ480N2RG
1345 | XN--JVR189M
1346 | XN--KCRX77D1X4A
1347 | XN--KPRW13D
1348 | XN--KPRY57D
1349 | XN--KPUT3I
1350 | XN--L1ACC
1351 | XN--LGBBAT1AD8J
1352 | XN--MGB9AWBF
1353 | XN--MGBA3A3EJT
1354 | XN--MGBA3A4F16A
1355 | XN--MGBA7C0BBN0A
1356 | XN--MGBAAM7A8H
1357 | XN--MGBAB2BD
1358 | XN--MGBAH1A3HJKRD
1359 | XN--MGBAI9AZGQP6J
1360 | XN--MGBAYH7GPA
1361 | XN--MGBBH1A
1362 | XN--MGBBH1A71E
1363 | XN--MGBC0A9AZCG
1364 | XN--MGBCA7DZDO
1365 | XN--MGBCPQ6GPA1A
1366 | XN--MGBERP4A5D4AR
1367 | XN--MGBGU82A
1368 | XN--MGBI4ECEXP
1369 | XN--MGBPL2FH
1370 | XN--MGBT3DHD
1371 | XN--MGBTX2B
1372 | XN--MGBX4CD0AB
1373 | XN--MIX891F
1374 | XN--MK1BU44C
1375 | XN--MXTQ1M
1376 | XN--NGBC5AZD
1377 | XN--NGBE9E0A
1378 | XN--NGBRX
1379 | XN--NODE
1380 | XN--NQV7F
1381 | XN--NQV7FS00EMA
1382 | XN--NYQY26A
1383 | XN--O3CW4H
1384 | XN--OGBPF8FL
1385 | XN--OTU796D
1386 | XN--P1ACF
1387 | XN--P1AI
1388 | XN--PGBS0DH
1389 | XN--PSSY2U
1390 | XN--Q7CE6A
1391 | XN--Q9JYB4C
1392 | XN--QCKA1PMC
1393 | XN--QXA6A
1394 | XN--QXAM
1395 | XN--RHQV96G
1396 | XN--ROVU88B
1397 | XN--RVC1E0AM3E
1398 | XN--S9BRJ9C
1399 | XN--SES554G
1400 | XN--T60B56A
1401 | XN--TCKWE
1402 | XN--TIQ49XQYJ
1403 | XN--UNUP4Y
1404 | XN--VERMGENSBERATER-CTB
1405 | XN--VERMGENSBERATUNG-PWB
1406 | XN--VHQUV
1407 | XN--VUQ861B
1408 | XN--W4R85EL8FHU5DNRA
1409 | XN--W4RS40L
1410 | XN--WGBH1C
1411 | XN--WGBL6A
1412 | XN--XHQ521B
1413 | XN--XKC2AL3HYE2A
1414 | XN--XKC2DL3A5EE0H
1415 | XN--Y9A3AQ
1416 | XN--YFRO4I67O
1417 | XN--YGBI2AMMX
1418 | XN--ZFR164B
1419 | XXX
1420 | XYZ
1421 | YACHTS
1422 | YAHOO
1423 | YAMAXUN
1424 | YANDEX
1425 | YE
1426 | YODOBASHI
1427 | YOGA
1428 | YOKOHAMA
1429 | YOU
1430 | YOUTUBE
1431 | YT
1432 | YUN
1433 | ZA
1434 | ZAPPOS
1435 | ZARA
1436 | ZERO
1437 | ZIP
1438 | ZM
1439 | ZONE
1440 | ZUERICH
1441 | ZW
1442 | 


--------------------------------------------------------------------------------
/urlscan/urlchoose.py:
--------------------------------------------------------------------------------
  1 | #   Copyright (C) 2006-2007 Daniel Burrows
  2 | #   Copyright (C) 2023 Scott Hansen
  3 | #
  4 | # This program is free software; you can redistribute it and/or
  5 | # modify it under the terms of the GNU General Public License
  6 | # as published by the Free Software Foundation; either version 2
  7 | # of the License, or (at your option) any later version.
  8 | #
  9 | # This program is distributed in the hope that it will be useful,
 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 12 | # GNU General Public License for more details.
 13 | #
 14 | # You should have received a copy of the GNU General Public License
 15 | # along with this program; if not, write to the Free Software
 16 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
 17 | 
 18 | """An urwid listview-based widget that lets you choose a URL from a list of
 19 | URLs."""
 20 | 
 21 | import contextlib
 22 | import json
 23 | import os
 24 | from os.path import dirname, exists, expanduser
 25 | from sys import platform
 26 | import re
 27 | import shlex
 28 | import subprocess
 29 | import sys
 30 | from threading import Thread
 31 | import webbrowser
 32 | 
 33 | import urwid
 34 | 
 35 | 
 36 | if platform == 'darwin':
 37 |     COPY_COMMANDS = ('pbcopy',)
 38 |     COPY_COMMANDS_PRIMARY = ('pbcopy',)
 39 | elif 'WAYLAND_DISPLAY' in os.environ:
 40 |     COPY_COMMANDS = ('wl-copy',)
 41 |     COPY_COMMANDS_PRIMARY = ('wl-copy --primary',)
 42 | else:
 43 |     COPY_COMMANDS = ("xsel -ib", "xclip -i -selection clipboard")
 44 |     COPY_COMMANDS_PRIMARY = ("xsel -i", "xclip -i")
 45 | 
 46 | 
 47 | def shorten_url(url, cols, shorten):
 48 |     """Shorten long URLs to fit on one line.
 49 | 
 50 |     """
 51 |     cols = ((cols - 6) * .85)  # 6 cols for urlref and don't use while line
 52 |     if shorten is False or len(url) < cols:
 53 |         return url
 54 |     split = int(cols * .5)
 55 |     return url[:split] + "..." + url[-split:]
 56 | 
 57 | 
 58 | def grp_list(items):
 59 |     """Organize list of items [a,2,3,4,a,4,2,a,1, etc...] like:
 60 |         [[a,2,3,4], [a,4,2], [a,1]], where 'a' is a urwid.Divider
 61 | 
 62 |     """
 63 |     grp = []
 64 |     res = []
 65 |     for item in items:
 66 |         if isinstance(item, urwid.Divider):
 67 |             res.append(grp)
 68 |             grp = [items[0]]
 69 |         else:
 70 |             grp.append(item)
 71 |     res.append(grp)
 72 |     return res[1:]
 73 | 
 74 | 
 75 | def splittext(text, search, attr):
 76 |     """Split a text string by search string and add Urwid display attribute to
 77 |     the search term.
 78 | 
 79 |     Args: text - string
 80 |           search - search string
 81 |           attr - attribute string to add
 82 | 
 83 |     Returns: urwid markup list ["string", ("default", " mo"), "re string"]
 84 |              for search="mo", text="string more string" and attr="default"
 85 | 
 86 |     """
 87 |     if search:
 88 |         pat = re.compile(f"({re.escape(search)})", re.IGNORECASE)
 89 |     else:
 90 |         return text
 91 |     final = pat.split(text)
 92 |     final = [(attr, i) if i.lower() == search.lower() else i for i in final]
 93 |     return final
 94 | 
 95 | 
 96 | class URLChooser:
 97 | 
 98 |     def __init__(self, extractedurls, compact=False, reverse=False, nohelp=False, dedupe=False,
 99 |                  shorten=True, run="", runsafe="", single=False, pipe=False,
100 |                  genconf=False, width=0, whitespaceoff=False, colors="true"):
101 |         self.conf = expanduser("~/.config/urlscan/config.json")
102 |         self.keys = {'/': self._search_key,
103 |                      '0': self._digits,
104 |                      '1': self._digits,
105 |                      '2': self._digits,
106 |                      '3': self._digits,
107 |                      '4': self._digits,
108 |                      '5': self._digits,
109 |                      '6': self._digits,
110 |                      '7': self._digits,
111 |                      '8': self._digits,
112 |                      '9': self._digits,
113 |                      'a': self._add_url,
114 |                      'C': self._clipboard,
115 |                      'c': self._context,
116 |                      'ctrl l': self._clear_screen,
117 |                      'ctrl f': self._page_down,
118 |                      'ctrl b': self._page_up,
119 |                      'd': self._del_url,
120 |                      'f1': self._help_menu,
121 |                      'G': self._bottom,
122 |                      'g': self._top,
123 |                      'j': self._down,
124 |                      'k': self._up,
125 |                      'J': self._next,
126 |                      'K': self._previous,
127 |                      'P': self._clipboard_pri,
128 |                      'l': self._link_handler,
129 |                      'o': self._open_queue,
130 |                      'O': self._open_queue_win,
131 |                      'p': self._palette,
132 |                      'Q': self._quit,
133 |                      'q': self._quit,
134 |                      'R': self._reverse,
135 |                      'S': self._all_shorten,
136 |                      's': self._shorten,
137 |                      'u': self._all_escape
138 |                      }
139 |         # Set urwid color mode based on the colors parameter
140 |         self.color = {
141 |             "true": 2**24,
142 |             "256": 256,
143 |             "88": 88,
144 |             "16": 16,
145 |             "8": 8,
146 |             "mono": 1
147 |         }.get(colors, 16)
148 |         self.palettes = {}
149 |         # Default color palette
150 |         default = [('header', 'white', 'dark blue', 'standout', '#ffffff', '#0000aa'),
151 |                    ('footer', 'white', 'dark red', 'standout', '#ffffff', '#aa0000'),
152 |                    ('search', 'white', 'dark green', 'standout', '#ffffff', '#00aa00'),
153 |                    ('msgtext', '', '', '', '', ''),
154 |                    ('msgtext:ellipses', 'light gray', 'black', '', '#aaaaaa', '#000000'),
155 |                    ('urlref:number:braces', 'light gray', 'black', '', '#aaaaaa', '#000000'),
156 |                    ('urlref:number', 'yellow', 'black', 'standout', '#ffff00', '#000000'),
157 |                    ('urlref:url', 'white', 'black', 'standout', '#ffffff', '#000000'),
158 |                    ('url:sel', 'white', 'dark blue', 'bold', '#ffffff', '#0000aa')]
159 |         # Default black & white palette
160 |         blw = [('header', 'black', 'light gray', 'standout', '#000000', '#aaaaaa'),
161 |                ('footer', 'black', 'light gray', 'standout', '#000000', '#aaaaaa'),
162 |                ('search', 'black', 'light gray', 'standout', '#000000', '#aaaaaa'),
163 |                ('msgtext', '', '', '', '', ''),
164 |                ('msgtext:ellipses', 'white', 'black', '', '#ffffff', '#000000'),
165 |                ('urlref:number:braces', 'white', 'black', '', '#ffffff', '#000000'),
166 |                ('urlref:number', 'white', 'black', 'standout', '#ffffff', '#000000'),
167 |                ('urlref:url', 'white', 'black', 'standout', '#ffffff', '#000000'),
168 |                ('url:sel', 'black', 'light gray', 'bold', '#000000', '#aaaaaa')]
169 |         # Default catppuccin palette
170 |         ctp = [("header", "white", "dark blue", "standout", "#CDD6F4", "#89B4FA"),
171 |                ("footer", "white", "dark red", "standout", "#CDD6F4", "#F38BA8"),
172 |                ("search", "white", "dark green", "standout", "#CDD6F4", "#A6E3A1"),
173 |                ("msgtext", "", "", "", "#CDD6F4", "#1E1E2E"),
174 |                ("msgtext:ellipses", "light gray", "black", "", "#B4BEFE", "#1E1E2E"),
175 |                ("urlref:number:braces", "light gray", "black", "", "#B4BEFE", "#1E1E2E"),
176 |                ("urlref:number", "yellow", "black", "standout", "#F9E2AF", "#1E1E2E"),
177 |                ("urlref:url", "white", "black", "standout", "#CBA6F7", "#1E1E2E"),
178 |                ("url:sel", "white", "dark blue", "bold", "#F5E0DC", "#313244")]
179 | 
180 |         if genconf is True:
181 |             # Add default palettes for config generation
182 |             self.palettes.update([("default", default), ("bw", blw), ("catppuccin", ctp)])
183 |             self._config_create()
184 | 
185 |         # Flag to track if we found palettes in config.json
186 |         config_has_palettes = False
187 | 
188 |         try:
189 |             with open(self.conf, 'r', encoding=sys.getdefaultencoding()) as conf_file:
190 |                 data = json.load(conf_file)
191 |                 try:
192 |                     if 'palettes' in data and data['palettes']:
193 |                         # If config has palettes, use only those
194 |                         config_has_palettes = True
195 |                         for pal_name, pal in data['palettes'].items():
196 |                             self.palettes.update([(pal_name, [tuple(i) for i in pal])])
197 |                 except KeyError:
198 |                     pass
199 |                 try:
200 |                     items = data['keys'].items()
201 |                     for key, value in items:
202 |                         if value:
203 |                             if value == "open_url":
204 |                                 urwid.Button._command_map._command[key] = 'activate'
205 |                             value = getattr(self, f"_{value}")
206 |                         else:
207 |                             del self.keys[key]
208 |                             continue
209 |                         self.keys.update([(key, value)])
210 |                 except KeyError:
211 |                     pass
212 |         except FileNotFoundError:
213 |             pass
214 | 
215 |         # If no palettes were found in config, use the default hardcoded ones
216 |         if not config_has_palettes:
217 |             self.palettes.update([("default", default), ("bw", blw), ("catppuccin", ctp)])
218 |         try:
219 |             subprocess.run(['xdg-open'], check=False, stdout=subprocess.DEVNULL)
220 |             self.xdg = True
221 |         except OSError:
222 |             self.xdg = False
223 |         self.shorten = shorten
224 |         self.compact = compact
225 |         self.queue = []
226 |         self.run = run
227 |         self.runsafe = runsafe
228 |         self.single = single
229 |         self.pipe = pipe
230 |         self.search = False
231 |         self.search_string = ""
232 |         self.no_matches = False
233 |         self.enter = False
234 |         self.term_width, _ = urwid.raw_display.Screen().get_cols_rows()
235 |         self.width = min(self.term_width, width or self.term_width)
236 |         self.whitespaceoff = whitespaceoff
237 |         self.activate_keys = [i for i, j in urwid.Button._command_map._command.items()
238 |                               if j == 'activate']
239 |         self.items, self.urls = self.process_urls(extractedurls,
240 |                                                   dedupe=dedupe,
241 |                                                   shorten=self.shorten)
242 |         # Original version of all items
243 |         self.items_orig = self.items
244 |         # Store items grouped into sections
245 |         self.items_org = grp_list(self.items)
246 |         # Store 'compact' mode items
247 |         self.items_com = [i for i in self.items if
248 |                           isinstance(i, urwid.Columns) is True]
249 |         if self.compact is True:
250 |             self.items, self.items_com = self.items_com, self.items
251 |         self.urls_unesc = [i.replace('\\', '') for i in self.urls]
252 |         self.unesc = False
253 |         listbox = urwid.ListBox(self.items)
254 |         self.header = (":: F1 - help/keybindings :: "
255 |                        "q - quit :: "
256 |                        "/ - search :: "
257 |                        "URL opening mode - {} :: "
258 |                        "Queue - {}")
259 |         self.link_open_modes = ["Web Browser", "Xdg-Open"] if self.xdg is True else ["Web Browser"]
260 |         if self.runsafe:
261 |             self.link_open_modes.insert(0, self.runsafe)
262 |         elif self.run:
263 |             self.link_open_modes.insert(0, self.run)
264 |         self.nohelp = nohelp
265 |         if nohelp is False:
266 |             self.headerwid = urwid.AttrMap(urwid.Text(
267 |                 self.header.format(self.link_open_modes[0], len(self.queue))), 'header')
268 |         else:
269 |             self.headerwid = None
270 |         self.top = urwid.Frame(listbox, self.headerwid)
271 |         self.pad = self.term_width - self.width
272 |         self.top = urwid.Padding(self.top, left=0, right=self.pad)
273 |         if self.urls:
274 |             self.top.base_widget.body.focus_position = \
275 |                 (2 if self.compact is False else 0)
276 |         if reverse is True:
277 |             self._reverse()
278 |         self.tui = urwid.raw_display.Screen()
279 |         self.palette_names = list(self.palettes.keys())
280 |         self.palette_idx = 0
281 |         self.number = ""
282 |         self.help_menu = False
283 | 
284 |     def main(self):
285 |         """Urwid main event loop
286 | 
287 |         """
288 |         self.loop = urwid.MainLoop(self.top, self.palettes[self.palette_names[0]], screen=self.tui,
289 |                                    handle_mouse=False, input_filter=self.handle_keys,
290 |                                    unhandled_input=self.unhandled)
291 |         self.loop.screen.set_terminal_properties(self.color)
292 |         self.loop.run()
293 | 
294 |     @property
295 |     def size(self):
296 |         _, rows = self.tui.get_cols_rows()
297 |         return (self.width, rows)
298 | 
299 |     def handle_keys(self, keys, raw):
300 |         """Handle widget default keys
301 | 
302 |             - 'Enter' or 'space' to load URL
303 |             - 'Enter' to end search mode
304 |             - add 'space' to search string in search mode
305 |             - Workaround some small positioning bugs
306 | 
307 |         """
308 |         for j, k in enumerate(keys):
309 |             if self.search is True:
310 |                 text = f"Search: {self.search_string}"
311 |                 if k == 'enter':
312 |                     # Catch 'enter' key to prevent opening URL in mkbrowseto
313 |                     self.enter = True
314 |                     if not self.items:
315 |                         self.search = False
316 |                         self.enter = False
317 |                     if self.search_string:
318 |                         footer = 'search'
319 |                     else:
320 |                         footer = 'default'
321 |                         text = ""
322 |                     footerwid = urwid.AttrMap(urwid.Text(text), footer)
323 |                     self.top.base_widget.footer = footerwid
324 |                 elif k in self.activate_keys:
325 |                     self.search_string += k
326 |                     self._search()
327 |                 elif k == 'backspace':
328 |                     self.search_string = self.search_string[:-1]
329 |                     self._search()
330 |             elif k in self.activate_keys and \
331 |                     self.urls and \
332 |                     self.search is False and \
333 |                     self.help_menu is False:
334 |                 self._open_url()
335 |             elif self.help_menu is True:
336 |                 self._help_menu()
337 |                 return []
338 |             if k == 'up':
339 |                 # Works around bug where the up arrow goes higher than the top list
340 |                 # item and unintentionally triggers context and palette switches.
341 |                 # Remaps 'up' to 'k'
342 |                 keys[j] = 'k'
343 |             if k == 'home':
344 |                 # Remap 'home' to 'g'. Works around small bug where 'home' takes the cursor
345 |                 # above the top list item.
346 |                 keys[j] = 'g'
347 |         # filter backspace out before the widget, it has a weird interaction
348 |         return [i for i in keys if i != 'backspace']
349 | 
350 |     def unhandled(self, key):
351 |         """Handle other keyboard actions not handled by the ListBox widget.
352 | 
353 |         """
354 |         self.key = key
355 |         if self.search is True:
356 |             if self.enter is False and self.no_matches is False:
357 |                 if len(key) == 1 and key.isprintable():
358 |                     self.search_string += key
359 |                 self._search()
360 |             elif self.enter is True and not self.search_string:
361 |                 self.search = False
362 |                 self.enter = False
363 |             return
364 |         if not self.urls and key not in "Qq":
365 |             return  # No other actions are useful with no URLs
366 |         if self.help_menu is False:
367 |             try:
368 |                 self.keys[key]()
369 |             except KeyError:
370 |                 pass
371 | 
372 |     def _quit(self):
373 |         """q/Q"""
374 |         raise urwid.ExitMainLoop()
375 | 
376 |     def _open_url(self):
377 |         """<Enter> or <space>"""
378 |         load_text = "Loading URL..." if self.link_open_modes[0] != (self.run or self.runsafe) \
379 |             else f"Executing: {self.run or self.runsafe}"
380 |         if os.environ.get('BROWSER') not in ['elinks', 'links', 'w3m', 'lynx']:
381 |             self._footer_display(load_text, 5)
382 | 
383 |     def _background_queue(self, mode):
384 |         """Open URLs in background"""
385 |         for url in self.queue:
386 |             self.mkbrowseto(url, mode=mode)()
387 |         self.draw_screen()
388 | 
389 |     def _queue(self, mode=2):
390 |         """Open all URLs in queue
391 | 
392 |             Args: mode - 2 for new tab, 1 for new window
393 | 
394 |         """
395 |         load_text = "Loading URLs in queue..." \
396 |             if self.link_open_modes[0] != (self.run or self.runsafe) \
397 |             else f"Executing: {self.run or self.runsafe}"
398 |         if os.environ.get('BROWSER') in ['elinks', 'links', 'w3m', 'lynx']:
399 |             self._footer_display("Opening multiple links not support in text browsers", 5)
400 |         else:
401 |             self._footer_display(load_text, 5)
402 |         thr = Thread(target=self._background_queue, args=(mode,))
403 |         thr.start()
404 |         self.queue = []
405 |         self.headerwid = urwid.AttrMap(urwid.Text(
406 |             self.header.format(self.link_open_modes[0], len(self.queue))), 'header')
407 |         self.top.base_widget.header = self.headerwid
408 | 
409 |     def _open_queue(self):
410 |         """o (new tab)"""
411 |         if self.queue:
412 |             self._queue()
413 | 
414 |     def _open_queue_win(self):
415 |         """O (new window)"""
416 |         if self.queue:
417 |             self._queue(1)
418 | 
419 |     def _add_url(self):
420 |         """a"""
421 |         fpo = self.top.base_widget.body.focus_position
422 |         url_idx = len([i for i in self.items[:fpo + 1]
423 |                        if isinstance(i, urwid.Columns)]) - 1
424 |         if self.compact is False and fpo <= 1:
425 |             return
426 |         self.queue.append(self.urls[url_idx])
427 |         self.queue = list(set(self.queue))
428 |         self.headerwid = urwid.AttrMap(urwid.Text(
429 |             self.header.format(self.link_open_modes[0], len(self.queue))), 'header')
430 |         self.top.base_widget.header = self.headerwid
431 |         label = self.items[fpo][1].label
432 |         if not label.startswith("* "):
433 |             self.items[fpo][1].set_label(f"* {label}")
434 | 
435 |     def _del_url(self):
436 |         """d"""
437 |         fpo = self.top.base_widget.body.focus_position
438 |         url_idx = len([i for i in self.items[:fpo + 1]
439 |                        if isinstance(i, urwid.Columns)]) - 1
440 |         if self.compact is False and fpo <= 1:
441 |             return
442 |         try:
443 |             self.queue.remove(self.urls[url_idx])
444 |             self.headerwid = urwid.AttrMap(urwid.Text(
445 |                 self.header.format(self.link_open_modes[0], len(self.queue))), 'header')
446 |             self.top.base_widget.header = self.headerwid
447 |             label = self.items[fpo][1].label
448 |             if label.startswith("* "):
449 |                 self.items[fpo][1].set_label(label.lstrip("* "))
450 |         except ValueError:
451 |             pass
452 | 
453 |     def _help_menu(self):
454 |         """F1"""
455 |         if self.help_menu is False:
456 |             self.focus_pos_saved = self.top.base_widget.body.focus_position
457 |             help_men = "\n".join([f"{i} - {j.__name__.strip('_')}"
458 |                                   for i, j in self.keys.items() if j.__name__ !=
459 |                                   '_digits'])
460 |             help_men = "KEYBINDINGS\n" + help_men + "\n<0-9> - Jump to item"
461 |             docs = ("OPTIONS\n"
462 |                     "add_url       -- add URL to queue\n"
463 |                     "all_escape    -- toggle unescape all URLs\n"
464 |                     "all_shorten   -- toggle shorten all URLs\n"
465 |                     "bottom        -- move cursor to last item\n"
466 |                     "clear_screen  -- redraw screen\n"
467 |                     "clipboard     -- copy highlighted URL to clipboard\n"
468 |                     "                 using xsel/xclip\n"
469 |                     "clipboard_pri -- copy highlighted URL to primary\n"
470 |                     "                 selection using xsel/xclip\n"
471 |                     "config_create -- create ~/.config/urlscan/config.json\n"
472 |                     "context       -- show/hide context\n"
473 |                     "del_url       -- delete URL from queue\n"
474 |                     "down          -- cursor down\n"
475 |                     "help_menu     -- show/hide help menu\n"
476 |                     "link_handler  -- cycle through xdg-open, webbrowser \n"
477 |                     "                 and user-defined function\n"
478 |                     "next          -- jump to next URL\n"
479 |                     "open_queue    -- open all URLs in queue\n"
480 |                     "open_queue_win-- open all URLs in queue in new window\n"
481 |                     "open_url      -- open selected URL\n"
482 |                     "palette       -- cycle through palettes\n"
483 |                     "previous      -- jump to previous URL\n"
484 |                     "quit          -- quit\n"
485 |                     "reverse       -- reverse order URLs/context\n"
486 |                     "shorten       -- toggle shorten highlighted URL\n"
487 |                     "single        -- quit urlscan after opening a\n"
488 |                     "                 single link\n"
489 |                     "top           -- move to first list item\n"
490 |                     "up            -- cursor up\n")
491 |             self.top.base_widget.body = \
492 |                 urwid.ListBox(urwid.SimpleListWalker([urwid.Columns([(24, urwid.Text(help_men)),
493 |                                                                      urwid.Text(docs)])]))
494 |         else:
495 |             self.top.base_widget.body = urwid.ListBox(self.items)
496 |             self.top.base_widget.body.focus_position = self.focus_pos_saved
497 |         self.help_menu = not self.help_menu
498 | 
499 |     def _search_key(self):
500 |         """ / """
501 |         if self.urls:
502 |             self.search = True
503 |             if self.compact is True:
504 |                 self._context()
505 |         else:
506 |             return
507 |         self.no_matches = False
508 |         self.search_string = ""
509 |         # Reset the search highlighting
510 |         self._search()
511 |         footerwid = urwid.AttrMap(urwid.Text("Search: "), 'footer')
512 |         self.top.base_widget.footer = footerwid
513 |         self.items = self.items_orig
514 |         self.top.base_widget.body = urwid.ListBox(self.items)
515 | 
516 |     def _digits(self):
517 |         """ 0-9 """
518 |         self.number += self.key
519 |         try:
520 |             if self.compact is False:
521 |                 self.top.base_widget.body.focus_position = \
522 |                     self.items.index(self.items_com[max(int(self.number) - 1, 0)])
523 |             else:
524 |                 self.top.base_widget.body.focus_position = \
525 |                     self.items.index(self.items[max(int(self.number) - 1, 0)])
526 |         except IndexError:
527 |             self.number = self.number[:-1]
528 |         except ValueError:
529 |             pass
530 |         self.top.base_widget.keypress(self.size, "")  # Trick urwid into redisplaying the cursor
531 |         if self.number:
532 |             self._footer_display(f"Selection: {self.number}", 1)
533 | 
534 |     def _clear_screen(self):
535 |         """ Ctrl-l """
536 |         self.draw_screen()
537 | 
538 |     def _down(self):
539 |         """ j """
540 |         self.top.base_widget.keypress(self.size, "down")
541 | 
542 |     def _up(self):
543 |         """ k """
544 |         self.top.base_widget.keypress(self.size, "up")
545 | 
546 |     def _top(self):
547 |         """ g """
548 |         # Goto top of the list
549 |         self.top.base_widget.body.focus_position = 2 if self.compact is False else 0
550 |         self.top.base_widget.keypress(self.size, "")  # Trick urwid into redisplaying the cursor
551 | 
552 |     def _bottom(self):
553 |         """ G """
554 |         # Goto bottom of the list
555 |         self.top.base_widget.body.focus_position = len(self.items) - 1
556 |         self.top.base_widget.keypress(self.size, "")  # Trick urwid into redisplaying the cursor
557 | 
558 |     def _selectable_positions(self):
559 |         return [i for i, item in enumerate(self.items) if item.selectable()]
560 | 
561 |     def _next(self):
562 |         """ J """
563 |         current_position = self.top.base_widget.body.focus_position
564 |         if current_position >= self._selectable_positions()[-1]:
565 |             # Do not jump if focus is on or after the last selectable position
566 |             return
567 |         # Jump to the first selectable position after the currently focused position
568 |         target_position = min(p for p in self._selectable_positions() if p > current_position)
569 |         self.top.base_widget.body.focus_position = target_position
570 |         self.top.base_widget.keypress(self.size, "")  # Trick urwid into redisplaying the cursor
571 | 
572 |     def _page_up(self):
573 |         """ Ctrl-b """
574 |         self.top.base_widget.keypress(self.size, "page up")
575 |         self.top.base_widget.keypress(self.size, "")  # Trick urwid into redisplaying the cursor
576 | 
577 |     def _page_down(self):
578 |         """ Ctrl-f """
579 |         self.top.base_widget.keypress(self.size, "page down")
580 |         self.top.base_widget.keypress(self.size, "")  # Trick urwid into redisplaying the cursor
581 | 
582 |     def _previous(self):
583 |         """ K """
584 |         current_position = self.top.base_widget.body.focus_position
585 |         if current_position <= self._selectable_positions()[0]:
586 |             # Do not jump if focus is on or before the first selectable position
587 |             return
588 |         # Jump to the first selectable position before the currently focused position
589 |         target_position = max(p for p in self._selectable_positions() if p < current_position)
590 |         self.top.base_widget.body.focus_position = target_position
591 |         self.top.base_widget.keypress(self.size, "")  # Trick urwid into redisplaying the cursor
592 | 
593 |     def _shorten(self):
594 |         """ s """
595 |         # Toggle shortened URL for selected item
596 |         fpo = self.top.base_widget.body.focus_position
597 |         url_idx = len([i for i in self.items[:fpo + 1]
598 |                        if isinstance(i, urwid.Columns)]) - 1
599 |         if self.compact is False and fpo <= 1:
600 |             return
601 |         url = self.urls[url_idx]
602 |         short = not "..." in self.items[fpo][1].label
603 |         self.items[fpo][1].set_label(shorten_url(url, self.size[0], short))
604 | 
605 |     def _all_shorten(self):
606 |         """ S """
607 |         # Toggle all shortened URLs
608 |         self.shorten = not self.shorten
609 |         urls = iter(self.urls)
610 |         for item in self.items:
611 |             # Each Column has (Text, Button). Update the Button label
612 |             if isinstance(item, urwid.Columns):
613 |                 item[1].set_label(shorten_url(next(urls),
614 |                                               self.size[0],
615 |                                               self.shorten))
616 | 
617 |     def _all_escape(self):
618 |         """ u """
619 |         # Toggle all escaped URLs
620 |         self.unesc = not self.unesc
621 |         self.urls, self.urls_unesc = self.urls_unesc, self.urls
622 |         urls = iter(self.urls)
623 |         for item in self.items:
624 |             # Each Column has (Text, Button). Update the Button label
625 |             if isinstance(item, urwid.Columns):
626 |                 item[1].set_label(shorten_url(next(urls),
627 |                                               self.size[0],
628 |                                               self.shorten))
629 | 
630 |     def _reverse(self):
631 |         """ R """
632 |         # Reverse items
633 |         fpo = self.top.base_widget.body.focus_position
634 |         if self.compact is True:
635 |             self.items.reverse()
636 |         else:
637 |             rev = []
638 |             for item in self.items:
639 |                 if isinstance(item, urwid.Divider):
640 |                     rev.insert(0, item)
641 |                 elif isinstance(item, urwid.Text):
642 |                     rev.insert(1, item)
643 |                 else:
644 |                     rev.insert(2, item)
645 |             self.items = rev
646 |         self.top.base_widget.body = urwid.ListBox(self.items)
647 |         self.top.base_widget.body.focus_position = self._cur_focus(fpo)
648 | 
649 |     def _context(self):
650 |         """ c """
651 |         # Show/hide context
652 |         if self.search_string:
653 |             # Reset search when toggling compact mode
654 |             footerwid = urwid.AttrMap(urwid.Text(""), 'default')
655 |             self.top.base_widget.footer = footerwid
656 |             self.search_string = ""
657 |             self.items = self.items_orig
658 |         fpo = self.top.base_widget.body.focus_position
659 |         self.items, self.items_com = self.items_com, self.items
660 |         self.top.base_widget.body = urwid.ListBox(self.items)
661 |         self.top.base_widget.body.focus_position = self._cur_focus(fpo)
662 |         self.compact = not self.compact
663 | 
664 |     def _clipboard(self, pri=False):
665 |         """ C """
666 |         # Copy highlighted url to clipboard
667 |         fpo = self.top.base_widget.body.focus_position
668 |         url_idx = len([i for i in self.items[:fpo + 1]
669 |                        if isinstance(i, urwid.Columns)]) - 1
670 |         if self.compact is False and fpo <= 1:
671 |             return
672 |         url = self.urls[url_idx]
673 |         cmds = COPY_COMMANDS_PRIMARY if pri else COPY_COMMANDS
674 |         for cmd in cmds:
675 |             try:
676 |                 subprocess.run(shlex.split(cmd),
677 |                                check=False,
678 |                                input=url.encode(sys.getdefaultencoding()),
679 |                                stdout=subprocess.DEVNULL,
680 |                                stderr=subprocess.DEVNULL)
681 |                 self._footer_display("Copied url to "
682 |                                      f"{'primary' if pri is True else 'clipboard'} selection", 5)
683 |             except OSError:
684 |                 continue
685 |             if self.single is True:
686 |                 self._quit()
687 |             break
688 | 
689 |     def _clipboard_pri(self):
690 |         """ P """
691 |         # Copy highlighted url to primary selection
692 |         self._clipboard(pri=True)
693 | 
694 |     def _palette(self):
695 |         """ p """
696 |         # Loop through available palettes
697 |         self.palette_idx += 1
698 |         try:
699 |             self.loop.screen.register_palette(self.palettes[self.palette_names[self.palette_idx]])
700 |         except IndexError:
701 |             self.loop.screen.register_palette(self.palettes[self.palette_names[0]])
702 |             self.palette_idx = 0
703 |         self.loop.screen.clear()
704 | 
705 |     def _config_create(self):
706 |         """ --genconf """
707 |         # Create ~/.config/urlscan/config.json if if doesn't exist
708 |         if not exists(self.conf):
709 |             os.makedirs(dirname(expanduser(self.conf)), exist_ok=True)
710 |             keys = dict(zip(self.keys.keys(),
711 |                             [i.__name__.strip('_') for i in self.keys.values()]))
712 |             with open(expanduser(self.conf), 'w', encoding=sys.getdefaultencoding()) as pals:
713 |                 pals.writelines(json.dumps({"palettes": self.palettes, "keys": keys},
714 |                                            indent=4))
715 |             print("Created ~/.config/urlscan/config.json")
716 |         else:
717 |             print("~/.config/urlscan/config.json already exists")
718 | 
719 |     def _footer_display(self, text, time):
720 |         """Display given text in the footer. Clears after <time> seconds
721 | 
722 |         """
723 |         footerwid = urwid.AttrMap(urwid.Text(text), 'footer')
724 |         self.top.base_widget.footer = footerwid
725 |         self.loop.set_alarm_in(time, self._footer_callback)
726 | 
727 |     def _footer_callback(self, _loop, _data):
728 |         """Callback for loop set_alarm_in after launching a URL or displaying a
729 |         URL selection number, clearing the screen and clearing the footer
730 |         loading message.
731 | 
732 |         """
733 |         self.number = ""  # Clear URL selection number
734 |         text = f"Search: {self.search_string}"
735 |         if self.search_string:
736 |             footer = 'search'
737 |         else:
738 |             footer = 'default'
739 |             text = ""
740 |         footerwid = urwid.AttrMap(urwid.Text(text), footer)
741 |         self.top.base_widget.footer = footerwid
742 |         self.draw_screen()
743 | 
744 |     def _cur_focus(self, fpo=0):
745 |         # Return correct focus when toggling 'show context'
746 |         if self.compact is False:
747 |             idx = len([i for i in self.items_com[:fpo + 1]
748 |                        if isinstance(i, urwid.Columns)]) - 1
749 |             idx = max(idx, 0)
750 |         elif self.compact is True:
751 |             idx = [i for i in enumerate(self.items)
752 |                    if isinstance(i[1], urwid.Columns)][fpo][0]
753 |         return idx
754 | 
755 |     def _search(self):
756 |         """ Search - search URLs and text.
757 | 
758 |         """
759 |         text = f"Search: {self.search_string}"
760 |         footerwid = urwid.AttrMap(urwid.Text(text), 'footer')
761 |         self.top.base_widget.footer = footerwid
762 |         search_items = []
763 |         for grp in self.items_org:
764 |             done = False
765 |             for idx, item in enumerate(grp):
766 |                 if isinstance(item, urwid.Columns):
767 |                     for col_idx, col in enumerate(item.contents):
768 |                         if isinstance(col[0], urwid.AttrMap):
769 |                             grp[idx][col_idx].set_label(splittext(col[0].base_widget.label,
770 |                                                                   self.search_string,
771 |                                                                   ''))
772 |                             if self.search_string.lower() in col[0].base_widget.label.lower():
773 |                                 grp[idx][col_idx].set_label(splittext(col[0].base_widget.label,
774 |                                                                       self.search_string,
775 |                                                                       'search'))
776 |                                 done = True
777 |                 elif isinstance(item, urwid.Text):
778 |                     grp[idx].set_text(splittext(item.text, self.search_string, ''))
779 |                     if self.search_string.lower() in item.text.lower():
780 |                         grp[idx].set_text(splittext(item.text, self.search_string, 'search'))
781 |                         done = True
782 |             if done is True:
783 |                 search_items.extend(grp)
784 |         self.items = search_items
785 |         self.top.base_widget.body = urwid.ListBox(self.items)
786 |         if self.items:
787 |             self.top.base_widget.body.focus_position = 2 if self.compact is False else 0
788 |             # Trick urwid into redisplaying the cursor
789 |             self.top.base_widget.keypress(self.size, "")
790 |             self.no_matches = False
791 |         else:
792 |             self.no_matches = True
793 |             footerwid = urwid.AttrMap(urwid.Text(text + "  No Matches"), 'footer')
794 |             self.top.base_widget.footer = footerwid
795 | 
796 |     def draw_screen(self):
797 |         """Render curses screen
798 | 
799 |         """
800 |         self.tui.clear()
801 |         canvas = self.top.base_widget.render(self.size, focus=True)
802 |         self.tui.draw_screen(self.size, canvas)
803 | 
804 |     def _get_search(self):
805 |         return lambda: self.search, lambda: self.enter
806 | 
807 |     def _link_handler(self):
808 |         """Function to cycle through opening links via webbrowser module,
809 |         xdg-open or custom expression passed with --run-safe or --run.
810 | 
811 |         """
812 |         mode = self.link_open_modes.pop()
813 |         self.link_open_modes.insert(0, mode)
814 |         if self.nohelp is False:
815 |             self.headerwid = urwid.AttrMap(urwid.Text(
816 |                 self.header.format(self.link_open_modes[0], len(self.queue))), 'header')
817 |             self.top.base_widget.header = self.headerwid
818 | 
819 |     def mkbrowseto(self, url, mode=0):
820 |         """Create the urwid callback function to open the web browser or call
821 |         another function with the URL.
822 | 
823 |         """
824 |         def browse(*args):  # pylint: disable=unused-argument
825 |             # double ()() to ensure self.search evaluated at runtime, not when
826 |             # browse() is _created_. [0] is self.search, [1] is self.enter
827 |             # self.enter prevents opening URL when in search mode
828 |             with redirect_output():
829 |                 if self._get_search()[0]() is True:
830 |                     if self._get_search()[1]() is True:
831 |                         self.search = False
832 |                         self.enter = False
833 |                 elif self.link_open_modes[0] == "Web Browser":
834 |                     webbrowser.open(url, new=mode)
835 |                 elif self.link_open_modes[0] == "Xdg-Open":
836 |                     subprocess.run(shlex.split(f'xdg-open "{url}"'), check=False)
837 |                 elif self.link_open_modes[0] == self.runsafe:
838 |                     if self.pipe:
839 |                         subprocess.run(shlex.split(self.runsafe),
840 |                                        check=False,
841 |                                        input=url.encode(sys.getdefaultencoding()))
842 |                     else:
843 |                         cmd = [i.format(url) for i in shlex.split(self.runsafe)]
844 |                         subprocess.run(cmd, check=False)
845 |                 elif self.link_open_modes[0] == self.run and self.pipe:
846 |                     subprocess.run(shlex.split(self.run),
847 |                                    check=False,
848 |                                    input=url.encode(sys.getdefaultencoding()))
849 |                 else:
850 |                     subprocess.run(self.run.format(url), check=False, shell=True)
851 | 
852 |                 if self.single is True:
853 |                     self._quit()
854 |         return browse
855 | 
856 |     def process_urls(self, extractedurls, dedupe, shorten):
857 |         """Process the 'extractedurls' and ready them for either the curses browser
858 |         or non-interactive output
859 | 
860 |         Args: extractedurls
861 |               dedupe - Remove duplicate URLs from list
862 | 
863 |         Returns: items - List of widgets for the ListBox
864 |                  urls - List of all URLs
865 | 
866 |         """
867 |         items = []
868 |         urls = []
869 |         for group, usedfirst, usedlast in extractedurls:
870 |             items.append(urwid.Divider(div_char='-', top=1, bottom=1))
871 |             if dedupe is True:
872 |                 # If no unique URLs exist, then skip the group completely
873 |                 if not [chunk for chunks in group for chunk in chunks
874 |                         if chunk.url is not None and chunk.url not in urls]:
875 |                     continue
876 |             groupurls = []
877 |             markup = []
878 |             if not usedfirst and not self.whitespaceoff:
879 |                 markup.append(('msgtext:ellipses', '...\n'))
880 |             for chunks in group:
881 |                 i = 0
882 |                 while i < len(chunks):
883 |                     chunk = chunks[i]
884 |                     i += 1
885 |                     if chunk.url is None:
886 |                         markup.append(('msgtext', chunk.markup))
887 |                     else:
888 |                         chunk.url = chunk.url.strip()
889 |                         if (dedupe is True and chunk.url not in urls) \
890 |                                 or dedupe is False:
891 |                             urls.append(chunk.url)
892 |                             groupurls.append(chunk.url)
893 |                         # Collect all immediately adjacent
894 |                         # chunks with the same URL.
895 |                         tmpmarkup = []
896 |                         if chunk.markup:
897 |                             tmpmarkup.append(('msgtext', chunk.markup))
898 |                         while i < len(chunks) and \
899 |                                 (chunks[i].url if chunks[i].url is None
900 |                                     else chunks[i].url.strip()) == chunk.url:
901 |                             if chunks[i].markup:
902 |                                 tmpmarkup.append(chunks[i].markup)
903 |                             i += 1
904 |                         url_idx = urls.index(chunk.url) + 1 if dedupe is True else len(urls)
905 |                         markup += [tmpmarkup or '<URL>',
906 |                                    ('urlref:number:braces', ' ['),
907 |                                    ('urlref:number', repr(url_idx)),
908 |                                    ('urlref:number:braces', ']')]
909 |                 if not self.whitespaceoff:
910 |                     markup += '\n'
911 |             if not usedlast and not self.whitespaceoff:
912 |                 markup += [('msgtext:ellipses', '...\n\n')]
913 |             items.append(urwid.Text(markup))
914 | 
915 |             i = len(urls) - len(groupurls)
916 |             for url in groupurls:
917 |                 i += 1
918 |                 markup = [(6, urwid.Text([('urlref:number:braces', '['),
919 |                                           ('urlref:number', repr(i)),
920 |                                           ('urlref:number:braces', ']'),
921 |                                           ' '])),
922 |                           urwid.AttrMap(urwid.Button(shorten_url(url,
923 |                                                                  self.width,
924 |                                                                  shorten),
925 |                                                      self.mkbrowseto(url),
926 |                                                      user_data=url),
927 |                                         'urlref:url', 'url:sel')]
928 |                 items.append(urwid.Columns(markup))
929 | 
930 |         return items, urls
931 | 
932 | 
933 | @contextlib.contextmanager
934 | def redirect_output():
935 |     """
936 |     A context manager to temporarily redirect stderr and stdout to devnull
937 | 
938 |     Usage:
939 |         with redirect_output():
940 |             webbrowser.open('https://google.com')
941 | 
942 |     """
943 |     try:
944 |         err = os.dup(sys.stderr.fileno())
945 |         out = os.dup(sys.stdout.fileno())
946 |         dest_file = open(os.devnull, 'w')
947 |         os.dup2(dest_file.fileno(), sys.stderr.fileno())
948 |         os.dup2(dest_file.fileno(), sys.stdout.fileno())
949 |         yield
950 |     finally:
951 |         if err is not None:
952 |             os.dup2(err, sys.stderr.fileno())
953 |         if out is not None:
954 |             os.dup2(out, sys.stdout.fileno())
955 |         if dest_file is not None:
956 |             dest_file.close()
957 | 


--------------------------------------------------------------------------------
/urlscan/urlscan.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | #   Copyright (C) 2006-2007 Daniel Burrows
  3 | #   Copyright (C) 2023 Scott Hansen
  4 | #
  5 | # This program is free software; you can redistribute it and/or
  6 | # modify it under the terms of the GNU General Public License
  7 | # as published by the Free Software Foundation; either version 2
  8 | # of the License, or (at your option) any later version.
  9 | #
 10 | # This program is distributed in the hope that it will be useful,
 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13 | # GNU General Public License for more details.
 14 | #
 15 | # You should have received a copy of the GNU General Public License
 16 | # along with this program; if not, write to the Free Software
 17 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 18 | 
 19 | """Contains the backend logic that scans messages for URLs and context."""
 20 | 
 21 | from html.parser import HTMLParser
 22 | import locale
 23 | import os
 24 | import re
 25 | from sys import getdefaultencoding
 26 | 
 27 | 
 28 | class Chunk:
 29 |     '''Represents a chunk of (marked-up) text that
 30 |     may or may not be linked to a URL.
 31 | 
 32 |     Attributes:
 33 |       markup - how this chunk will be rendered via urwid.
 34 |                This may be None if url is set, indicating
 35 |                that a URL footnote should be autogenerated.
 36 | 
 37 |       url    - the URL to which this text is linked, or None
 38 |                if no URL link is present.'''
 39 |     def __init__(self, markup, url):
 40 |         self.markup = markup
 41 |         self.url = url
 42 | 
 43 |     def __str__(self):
 44 |         return f'Chunk(markup = {repr(self.markup)}, url= {repr(self.url)})'
 45 | 
 46 |     def __repr__(self):
 47 |         return self.__str__()
 48 | 
 49 | 
 50 | def isheadertag(tag):
 51 |     """Determine if tag is a header """
 52 |     return len(tag) == 2 and tag[0] == 'h' and tag[1].isdigit()
 53 | 
 54 | 
 55 | class HTMLChunker(HTMLParser):
 56 |     """An HTMLParser that generates a sequence of lists of chunks.
 57 |     Each list represents a single paragraph."""
 58 | 
 59 |     def __init__(self):
 60 |         HTMLParser.__init__(self)
 61 | 
 62 |         # This attribute is the current output list.
 63 |         self.rval = []
 64 | 
 65 |         # If this attribute is True, the next chunk added will start a
 66 |         # new list.
 67 |         self.at_para_start = True
 68 |         self.trailing_space = False
 69 | 
 70 |         self.style_stack = [set()]
 71 |         self.anchor_stack = [None]
 72 |         self.list_stack = []
 73 |         # either 'ul' or 'ol' entries.
 74 | 
 75 |         # Ignore everything inside <style> and <script> elements
 76 |         self.in_style_or_script = False
 77 | 
 78 |     # Styled text uses the named attribute
 79 |     # msgtext:style1style2... where the styles
 80 |     # are always sorted in alphabetic order.
 81 |     # (if the text is in an anchor, substitute
 82 |     # "msgtext:anchor" for "msgtext")
 83 |     tag_styles = {'b': 'bold', 'i': 'italic'}
 84 | 
 85 |     ul_tags = ['*', '+', '-']
 86 | 
 87 |     def cur_url(self):
 88 |         return self.anchor_stack[-1]
 89 | 
 90 |     def add_chunk(self, chunk):
 91 |         if self.at_para_start:
 92 |             self.rval.append([])
 93 |         elif self.trailing_space:
 94 |             self.rval[-1].append(Chunk(' ', self.cur_url()))
 95 | 
 96 |         self.rval[-1].append(chunk)
 97 |         self.at_para_start = False
 98 |         self.trailing_space = False
 99 | 
100 |     def end_para(self):
101 |         if self.at_para_start:
102 |             self.rval.append([])
103 |         else:
104 |             self.at_para_start = True
105 |         self.trailing_space = False
106 |         if self.list_stack:
107 |             self.add_chunk(Chunk(' ' * 3 * len(self.list_stack),
108 |                                  self.cur_url()))
109 | 
110 |     def end_list_para(self):
111 |         if self.at_para_start:
112 |             self.rval.append([])
113 |         if self.list_stack:
114 |             tag = self.list_stack[-1][0]
115 |             if tag == 'ul':
116 |                 depth = len([t for t in self.list_stack if t[0] == tag])
117 |                 ul_tags = HTMLChunker.ul_tags
118 |                 chunk = Chunk(f"{ul_tags[depth % len(ul_tags)]}  ", self.cur_url())
119 |             else:
120 |                 counter = self.list_stack[-1][1]
121 |                 self.list_stack[-1] = (tag, counter + 1)
122 |                 chunk = Chunk(f"{counter:2d}.", self.cur_url())
123 |             self.add_chunk(chunk)
124 |         else:
125 |             self.end_para()
126 | 
127 |     def findattr(self, attrs, searchattr):
128 |         for attr, val in attrs:
129 |             if attr == searchattr:
130 |                 return val
131 | 
132 |         return None
133 | 
134 |     def handle_starttag(self, tag, attrs):
135 |         if tag == 'a':
136 |             self.anchor_stack.append(self.findattr(attrs, 'href'))
137 |         elif tag in ('ul', 'ol'):
138 |             self.list_stack.append((tag, 1))
139 |             self.end_para()
140 |         elif tag in HTMLChunker.tag_styles:
141 |             self.style_stack.append(self.style_stack[-1] |
142 |                                     set([HTMLChunker.tag_styles[tag]]))
143 |         elif isheadertag(tag):
144 |             self.style_stack.append(self.style_stack[-1] | set(['bold']))
145 |         elif tag in ('p', 'br'):
146 |             self.end_para()
147 |         elif tag == 'img':
148 |             # Since we expect HTML *email*, image links
149 |             # should be external (naja?)
150 |             alt = self.findattr(attrs, 'alt')
151 |             if alt is None:
152 |                 alt = '[IMG]'
153 |             src = self.findattr(attrs, 'src')
154 |             if src is not None and not src.startswith(('http://', 'https://')):
155 |                 src = None
156 | 
157 |             if src is not None:
158 |                 self.anchor_stack.append(src)
159 |                 self.handle_data(alt)
160 |                 del self.anchor_stack[-1]
161 |             else:
162 |                 self.handle_data(alt)
163 |         elif tag == 'li':
164 |             self.end_list_para()
165 |         elif tag in ('style', 'script'):
166 |             self.in_style_or_script = True
167 | 
168 |     def handle_startendtag(self, tag, attrs):
169 |         if tag in set(['p', 'br', 'li', 'img']):
170 |             self.handle_starttag(tag, attrs)
171 | 
172 |     def handle_endtag(self, tag):
173 |         if tag == 'a':
174 |             if len(self.anchor_stack) > 1:
175 |                 del self.anchor_stack[-1]
176 |         elif tag in HTMLChunker.tag_styles:
177 |             if len(self.style_stack) > 1:
178 |                 del self.style_stack[-1]
179 |         elif tag in ('ul', 'ol'):
180 |             if len(self.list_stack) > 0:
181 |                 del self.list_stack[-1]
182 |             self.end_para()
183 |         elif isheadertag(tag):
184 |             if len(self.style_stack) > 1:
185 |                 del self.style_stack[-1]
186 |             self.end_para()
187 |         elif tag in ('style', 'script'):
188 |             self.in_style_or_script = False
189 | 
190 |     def handle_data(self, data):
191 |         if self.in_style_or_script:
192 |             return
193 |         future_trailing_space = False
194 |         if data:
195 |             if data[0].isspace():
196 |                 self.trailing_space = True
197 |             if data[-1].isspace():
198 |                 future_trailing_space = True
199 |         data = ' '.join(data.split())
200 |         if self.anchor_stack[-1] is None:
201 |             style = 'msgtext'
202 |         else:
203 |             style = 'anchor'
204 |         if self.style_stack[-1]:
205 |             stylelist = list(self.style_stack[-1])
206 |             stylelist.sort()
207 |             style = style + ':' + ''.join(stylelist)
208 | 
209 |         self.add_chunk(Chunk((style, data), self.cur_url()))
210 |         self.trailing_space = future_trailing_space
211 | 
212 |     extrachars = {8212: "--",
213 |                   8217: "'",
214 |                   8220: "``",
215 |                   8221: "''",
216 |                   8230: "..."}
217 | 
218 |     def handle_charref(self, name):
219 |         if name[0] == 'x':
220 |             char = int(name[1:], 16)
221 |         else:
222 |             char = int(name)
223 |         if char < 128:
224 |             name = chr(char)
225 |         elif char in HTMLChunker.extrachars:
226 |             name = HTMLChunker.extrachars[char]
227 |         else:
228 |             name = f"&#{name};"
229 |         self.handle_data(name)
230 | 
231 |     entities = {'nbsp': ' ',
232 |                 'lt': '<',
233 |                 'gt': '>',
234 |                 'amp': '&',
235 |                 'ldquo': '``',
236 |                 'rdquo': "''",
237 |                 'apos': "'"}
238 | 
239 |     def handle_entityref(self, name):
240 |         if name in HTMLChunker.entities:
241 |             self.handle_data(HTMLChunker.entities[name])
242 |         else:
243 |             # If you see a reference, it needs to be
244 |             # added above.
245 |             self.handle_data(f"&{name};")
246 | 
247 | 
248 | URLINTERNALPATTERN = r'[\[\]{}()@\w/\\\-%?!&.=:;+,#~*]'
249 | URLTRAILINGPATTERN = r'[{}(@\w/\-%&=+#$]'
250 | HTTPURLPATTERN = (r'(?:(https?|file|ftps?)://' + URLINTERNALPATTERN +
251 |                   r'*' + URLTRAILINGPATTERN + r')')
252 | # Used to guess that blah.blah.blah.TLD is a URL.
253 | 
254 | 
255 | def load_tlds():
256 |     """Load all legal TLD extensions from assets
257 | 
258 |     """
259 |     file = os.path.join(os.path.dirname(__file__),
260 |                         'assets',
261 |                         'tlds-alpha-by-domain.txt')
262 |     with open(file, encoding=getdefaultencoding()) as fobj:
263 |         return [elem for elem in fobj.read().lower().splitlines()[1:]
264 |                 if "--" not in elem]
265 | 
266 | 
267 | TLDS = load_tlds()
268 | GUESSEDURLPATTERN = (r'(?:[\w\-%]+(?:\.[\w\-%]+)*\.(?:' +
269 |                      '|'.join(TLDS) + ')$)')
270 | URLRE = re.compile(r'(?:<(?:URL:)?)?(' + HTTPURLPATTERN + '|' +
271 |                    GUESSEDURLPATTERN +
272 |                    r'|(?P<email>(mailto:)?[\w\-.]+@[\w\-.]*[\w\-]))>?',
273 |                    flags=re.U)
274 | 
275 | # Poor man's test cases.
276 | assert URLRE.match('<URL:http://linuxtoday.com>')
277 | assert URLRE.match('http://linuxtoday.com')
278 | assert re.compile(GUESSEDURLPATTERN).match('example.biz')
279 | assert URLRE.match('example.biz')
280 | assert URLRE.match('linuxtoday.com')
281 | assert URLRE.match('master.wizard.edu')
282 | assert URLRE.match('blah.bar.info')
283 | assert URLRE.match('goodpr.org')
284 | assert URLRE.match('http://github.com/firecat53/ürlscan')
285 | assert URLRE.match('https://Schöne_Grüße.es/test')
286 | assert URLRE.match('http://www.pantherhouse.com/newshelton/my-wife-thinks-i’m-a-swan/')
287 | assert not URLRE.match('blah..org')
288 | assert URLRE.match('http://www.testurl.zw')
289 | assert URLRE.match('http://www.testurl.smile')
290 | assert URLRE.match('testurl.smile.smile')
291 | assert URLRE.match('testurl.biz.smile.zw')
292 | assert not URLRE.match('example..biz')
293 | assert not URLRE.match('blah.baz.obviouslynotarealdomain')
294 | assert URLRE.match('http://[2a07:3500:11a0:320::22]:8080')
295 | 
296 | 
297 | def parse_text_urls(mesg, regex=None):
298 |     """Parse a block of text, splitting it into its url and non-url
299 |     components."""
300 | 
301 |     rval = []
302 | 
303 |     loc = 0
304 | 
305 |     global URLRE
306 | 
307 |     if regex:
308 |         URLRE = re.compile(regex)
309 | 
310 |     for match in URLRE.finditer(mesg):
311 |         if loc < match.start():
312 |             rval.append(Chunk(mesg[loc:match.start()], None))
313 |         # Turn email addresses into mailto: links
314 |         if regex:
315 |             rval.append(Chunk(None, match.group(0)))
316 |         else:
317 |             email = match.group("email")
318 |             if email and "mailto" not in email:
319 |                 mailto = f"mailto:{email}"
320 |             else:
321 |                 mailto = match.group(1)
322 |             rval.append(Chunk(None, mailto))
323 |         loc = match.end()
324 | 
325 |     if loc < len(mesg):
326 |         rval.append(Chunk(mesg[loc:], None))
327 | 
328 |     return rval
329 | 
330 | 
331 | def extract_with_context(lst, pred, before_context, after_context):
332 |     """Extract URL and context from a given chunk.
333 | 
334 |     """
335 |     rval = []
336 | 
337 |     start = 0
338 |     length = 0
339 |     while start < len(lst):
340 |         usedfirst = False
341 |         usedlast = False
342 |         # Extend to the next match.
343 |         while start + length < len(lst) and length < before_context + 1 \
344 |                 and not pred(lst[start + length]):
345 |             length += 1
346 | 
347 |         # Slide to the next match.
348 |         while start + length < len(lst) and not pred(lst[start + length]):
349 |             start += 1
350 | 
351 |         # If there was no next match, abort here (it's easier
352 |         # to do this than to try to detect this case later).
353 |         if start + length == len(lst):
354 |             break
355 | 
356 |         # Now extend repeatedly until we can't find anything.
357 |         while start + length < len(lst) and pred(lst[start + length]):
358 |             extendlength = 1
359 |             # Read in the 'after' context and see if it holds a URL.
360 |             while extendlength < after_context + 1 and start + length + \
361 |                     extendlength < len(lst) and \
362 |                     not pred(lst[start + length + extendlength]):
363 |                 extendlength += 1
364 |             length += extendlength
365 |             if start + length < len(lst) and not pred(lst[start + length]):
366 |                 # Didn't find a matching line, so we either
367 |                 # hit the end or extended to after_context + 1..
368 |                 #
369 |                 # Now read in possible 'before' context
370 |                 # from the next URL; if we don't find one,
371 |                 # we discard the readahead.
372 |                 extendlength = 1
373 |                 while extendlength < before_context and start + length + \
374 |                         extendlength < len(lst) and \
375 |                         not pred(lst[start + length + extendlength]):
376 |                     extendlength += 1
377 |                 if start + length + extendlength < len(lst) and \
378 |                         pred(lst[start + length + extendlength]):
379 |                     length += extendlength
380 | 
381 |         if length > 0 and start + length <= len(lst):
382 |             if start == 0:
383 |                 usedfirst = True
384 |             if start + length == len(lst):
385 |                 usedlast = True
386 |             rval.append((lst[start:start + length], usedfirst, usedlast))
387 | 
388 |         start += length
389 |         length = 0
390 |     return rval
391 | 
392 | 
393 | NLRE = re.compile('\r\n|\n|\r')
394 | 
395 | 
396 | def extracturls(mesg, regex=None):
397 |     """Given a text message, extract all the URLs found in the message, along
398 |     with their surrounding context.  The output is a list of sequences of Chunk
399 |     objects, corresponding to the contextual regions extracted from the string.
400 | 
401 |     """
402 |     lines = NLRE.split(mesg)
403 | 
404 |     # The number of lines of context above to provide.
405 |     # above_context = 1
406 |     # The number of lines of context below to provide.
407 |     # below_context = 1
408 | 
409 |     # Plan here is to first transform lines into the form
410 |     # [line_fragments] where each fragment is a chunk as
411 |     # seen by parse_text_urls.  Note that this means that
412 |     # lines with more than one entry or one entry that's
413 |     # a URL are the only lines containing URLs.
414 | 
415 |     linechunks = [parse_text_urls(i, regex=regex) for i in lines]
416 | 
417 |     return extract_with_context(linechunks,
418 |                                 lambda chunk: len(chunk) > 1 or
419 |                                 (len(chunk) == 1 and chunk[0].url is not None),
420 |                                 1, 1)
421 | 
422 | 
423 | def extracthtmlurls(mesg):
424 |     """Extract URLs with context from html type message. Similar to extracturls.
425 | 
426 |     """
427 |     chunk = HTMLChunker()
428 |     chunk.feed(mesg)
429 |     chunk.close()
430 |     # above_context = 1
431 |     # below_context = 1
432 | 
433 |     def somechunkisurl(chunks):
434 |         for chnk in chunks:
435 |             if chnk.url is not None:
436 |                 return True
437 |         return False
438 | 
439 |     return extract_with_context(chunk.rval, somechunkisurl, 1, 1)
440 | 
441 | 
442 | def msgheaders(msg):
443 |     """ Process email message headers for URLs
444 | 
445 |     Args: msg - email message object
446 |     Returns: list
447 | 
448 |     """
449 |     headers = ('Archived-At',
450 |                'Link',
451 |                'List-Archive',
452 |                'List-ID',
453 |                'List-Help',
454 |                'List-Owner',
455 |                'List-Post',
456 |                'List-Subscribe',
457 |                'List-Unsubscribe',
458 |                'List-Unsubscribe-Post')
459 |     res = []
460 |     for hdr in headers:
461 |         hdri = msg.get(hdr)
462 |         if hdri:
463 |             res.append(hdri)
464 |     return res
465 | 
466 | 
467 | def set_charset(message):
468 |     """Get and/or set the message or message part charset. Try the
469 |     content-charset or charset if it exists, or attempt to decode the message
470 |     with a variety of charsets to find the correct one.
471 | 
472 |         Args: message - EmailMessage object
473 |         Returns: message - EmailMessage object
474 | 
475 |     """
476 |     if message.get_content_charset():
477 |         return message
478 |     if message.get_charset():
479 |         return message
480 |     enc_list = ['UTF-8', 'LATIN-1', 'iso8859-1', 'iso8859-2',
481 |                 'UTF-16', 'CP1252', 'CP720', 'CP437']
482 |     locale.setlocale(locale.LC_ALL, '')
483 |     code = locale.getpreferredencoding()
484 |     if code not in enc_list:
485 |         enc_list.insert(0, code)
486 |     for enc in enc_list:
487 |         try:
488 |             message.as_bytes().decode(enc)
489 |         except (UnicodeDecodeError, UnicodeError):
490 |             continue
491 |         else:
492 |             try:
493 |                 message.set_param('charset', enc)
494 |             except (KeyError, UnicodeEncodeError):
495 |                 # Try once to set correct encoding on the message part, then
496 |                 # continue without crashing if it fails
497 |                 continue
498 |             break
499 |         raise Exception("Encoding not detected.")
500 |     return message
501 | 
502 | 
503 | def msgurls(msg, urlidx=1, regex=None, headers=False):
504 |     """Main entry function for urlscan.py
505 | 
506 |     """
507 |     # Written as a generator so I can easily choose only
508 |     # one subpart in the future (e.g., for
509 |     # multipart/alternative).  Actually, I might even add
510 |     # a browser for the message structure?
511 |     if headers is True:
512 |         for part in msgheaders(set_charset(msg)):
513 |             for chunk in extracturls(part):
514 |                 urlidx += 1
515 |                 yield chunk
516 |     msg = set_charset(msg)
517 |     if msg.is_multipart():
518 |         for part in msg.iter_parts():
519 |             for chunk in msgurls(set_charset(part), urlidx, regex=regex):
520 |                 urlidx += 1
521 |                 yield chunk
522 |     elif msg.get_content_type() == "text/plain":
523 |         for chunk in extracturls(msg.get_content(), regex=regex):
524 |             urlidx += 1
525 |             yield chunk
526 |     elif msg.get_content_type() == "text/html":
527 |         for chunk in extracthtmlurls(msg.get_content()):
528 |             urlidx += 1
529 |             yield chunk
530 | 


--------------------------------------------------------------------------------