├── .gitattributes
├── .gitignore
├── LICENSE
├── README.md
├── approximation.ipynb
├── cm_factor.sage
├── cm_factor_python.ipynb
├── coloredlogs
    ├── __init__.py
    ├── cli.py
    ├── converter
    │   ├── __init__.py
    │   └── colors.py
    ├── demo.py
    ├── syslog.py
    └── tests.py
├── data
    ├── .gitkeep
    ├── semifactor_15.json
    └── semifactor_17.json
├── division_poly.ipynb
├── experiments.sage
└── humanfriendly
    ├── __init__.py
    ├── cli.py
    ├── compat.py
    ├── decorators.py
    ├── prompts.py
    ├── sphinx.py
    ├── tables.py
    ├── terminal.py
    ├── testing.py
    ├── tests.py
    ├── text.py
    └── usage.py


/.gitattributes:
--------------------------------------------------------------------------------
1 | data/ filter=lfs diff=lfs merge=lfs -text
2 | data/semifactor_15.json filter=lfs diff=lfs merge=lfs -text
3 | data/semifactor_17.json filter=lfs diff=lfs merge=lfs -text
4 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | # Byte-compiled / optimized / DLL files
  2 | __pycache__/
  3 | *.py[cod]
  4 | *$py.class
  5 | 
  6 | # C extensions
  7 | *.so
  8 | 
  9 | # Distribution / packaging
 10 | .Python
 11 | build/
 12 | develop-eggs/
 13 | dist/
 14 | downloads/
 15 | eggs/
 16 | .eggs/
 17 | lib/
 18 | lib64/
 19 | parts/
 20 | sdist/
 21 | var/
 22 | wheels/
 23 | *.egg-info/
 24 | .installed.cfg
 25 | *.egg
 26 | MANIFEST
 27 | 
 28 | # PyInstaller
 29 | #  Usually these files are written by a python script from a template
 30 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 31 | *.manifest
 32 | *.spec
 33 | 
 34 | # Installer logs
 35 | pip-log.txt
 36 | pip-delete-this-directory.txt
 37 | 
 38 | # Unit test / coverage reports
 39 | htmlcov/
 40 | .tox/
 41 | .coverage
 42 | .coverage.*
 43 | .cache
 44 | nosetests.xml
 45 | coverage.xml
 46 | *.cover
 47 | .hypothesis/
 48 | .pytest_cache/
 49 | 
 50 | # Translations
 51 | *.mo
 52 | *.pot
 53 | 
 54 | # Django stuff:
 55 | *.log
 56 | local_settings.py
 57 | db.sqlite3
 58 | 
 59 | # Flask stuff:
 60 | instance/
 61 | .webassets-cache
 62 | 
 63 | # Scrapy stuff:
 64 | .scrapy
 65 | 
 66 | # Sphinx documentation
 67 | docs/_build/
 68 | 
 69 | # PyBuilder
 70 | target/
 71 | 
 72 | # Jupyter Notebook
 73 | .ipynb_checkpoints
 74 | 
 75 | # pyenv
 76 | .python-version
 77 | 
 78 | # celery beat schedule file
 79 | celerybeat-schedule
 80 | 
 81 | # SageMath parsed files
 82 | *.sage.py
 83 | 
 84 | # Environments
 85 | .env
 86 | .venv
 87 | env/
 88 | venv/
 89 | ENV/
 90 | env.bak/
 91 | venv.bak/
 92 | 
 93 | # Spyder project settings
 94 | .spyderproject
 95 | .spyproject
 96 | 
 97 | # Rope project settings
 98 | .ropeproject
 99 | 
100 | # mkdocs documentation
101 | /site
102 | 
103 | # mypy
104 | .mypy_cache/
105 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2018 CRoCS
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # CM-based factorization
 2 | Complex multiplication based factorization.
 3 | This repository contains proof of concept factorization code and scripts used in experiments.
 4 | 
 5 | ## Proof of concept script usage
 6 | 
 7 | Generating CM primes:
 8 | 
 9 | ```bash
10 | sage cm_factor.sage --action generate --prime-bits 64
11 | 16344322632527439553
12 | ```
13 | 
14 | CM factorization:
15 | 
16 | ```bash
17 | sage cm_factor.sage -N 158697752795669080171615843390068686677 -D 11
18 | Factorization of N: 158697752795669080171615843390068686677 is: 
19 | 14793660019451035033 * 10727416514034371869
20 | ```
21 | 
22 | ## Advanced usage script
23 | 
24 | More complex script that was used to run experiments covered in the paper is `experiments.sage`.
25 | It covers distributed computation of the factorization, class number sampling, batch BCD computation, 
26 | Hilbert polynomial generation, primorial D computation, benchmarking.
27 | 
28 | 
29 | ## Dependencies
30 | 
31 | Sage 8 is required. Included packages `coloredlogs`, `humanfriendly` are optional.
32 | 
33 | 


--------------------------------------------------------------------------------
/cm_factor.sage:
--------------------------------------------------------------------------------
  1 | # packages (optional):
  2 | # sage --pip install coloredlogs
  3 | 
  4 | import time
  5 | import argparse
  6 | import sys
  7 | import logging
  8 | import traceback
  9 | 
 10 | from sage.misc.prandom import randrange
 11 | from sage.parallel.decorate import fork
 12 | 
 13 | 
 14 | debug = False
 15 | logger = logging.getLogger(__name__)
 16 | 
 17 | 
 18 | try:
 19 |     import coloredlogs
 20 |     coloredlogs.CHROOT_FILES = []
 21 |     coloredlogs.install(level=logging.DEBUG, use_chroot=False)
 22 | except:
 23 |     pass
 24 | 
 25 | 
 26 | class AlgException(Exception):
 27 |     pass
 28 | 
 29 | 
 30 | class NotInvertibleException(AlgException):
 31 |     pass
 32 | 
 33 | 
 34 | class NotFactoredException(AlgException):
 35 |     pass
 36 | 
 37 | 
 38 | class FactorRes(object):
 39 |     def __init__(self, r=None, c=None, u=None, a=None, th=None, tq=None):
 40 |         self.r = r
 41 |         self.c = c
 42 |         self.u = u
 43 |         self.a = a
 44 |         self.time_hilbert = th
 45 |         self.time_q = tq
 46 |         self.time_a = None
 47 |         self.time_last_div = None
 48 |         self.time_last_gcd = None
 49 |         self.time_last_nrm = None
 50 |         self.time_total = 0
 51 |         self.time_agg_div = 0
 52 |         self.time_agg_gcd = 0
 53 |         self.time_agg_nrm = 0
 54 |         self.time_qinv_char_poly = 0
 55 |         self.time_qinv_xgcd = 0
 56 |         self.time_qinv_res = 0
 57 |         self.rand_elem = None
 58 |         self.fact_timeout = None
 59 |         self.out_of_time = False
 60 |         self.use_quinv2 = False
 61 |         self.use_cheng = False
 62 | 
 63 | 
 64 | def is_undef(result):
 65 |     return result in ['NO DATA (timed out)', 'NO DATA', 'INVALID DATA', 'INVALID DATA ', None]
 66 | 
 67 | 
 68 | def class_number(d):
 69 |     k = QuadraticField(-d, 'x')
 70 |     return k.class_number()
 71 | 
 72 | 
 73 | def random_cm_prime_sub(thresh_l, thresh_h, rstart, rstop, D):
 74 |     while True:
 75 |         s = -1
 76 | 
 77 |         # Better is sqrt distribution as with uniform on i we get skew on i^2
 78 |         while True:
 79 |             s = randrange(thresh_l, thresh_h)
 80 |             s = int(isqrt((4r * s - 1r) / D))  # memory leak: s = int(sqrt((4 * s - 1) / D))
 81 |             if s & 1r == 1r and s >= rstart and s <= rstop:
 82 |                 break
 83 | 
 84 |         p = int((D * s * s + 1r) / 4r)
 85 |         if p > thresh_h or p < thresh_l:
 86 |             continue
 87 |         if is_prime(p):
 88 |             return p
 89 | 
 90 | 
 91 | def generateCMprime(D, bits, verb = 1):
 92 |     if D % 8 != 3:
 93 |         raise ValueError('D must be congruent to 3 modulo 8')
 94 | 
 95 |     p = None
 96 |     thresh_l = 1 << (bits - 1)
 97 |     thresh_h = (1 << bits) - 1
 98 | 
 99 |     # Exact bounds to cover the whole interval
100 |     rstart = int(isqrt((4r*thresh_l-1r)/D))
101 |     rstop  = int(isqrt((4r*thresh_h-1r)/D))+1r
102 | 
103 |     while True:
104 |         p = random_cm_prime_sub(thresh_l, thresh_h, rstart, rstop, D)
105 |         if p:
106 |             return p
107 | 
108 | 
109 | def xgcd(f, g, N=1):
110 |     toswap = False
111 |     if f.degree() < g.degree():
112 |         toswap = True
113 |         f, g = g, f
114 |     r_i = f
115 |     r_i_plus = g
116 |     r_i_plus_plus = f
117 | 
118 |     s_i, s_i_plus = 1, 0
119 |     t_i, t_i_plus = 0, 1
120 | 
121 |     while (True):
122 |         lc = r_i.lc().lift()
123 |         lc *= r_i_plus.lc().lift()
124 |         lc *= r_i_plus_plus.lc().lift()
125 |         divisor = gcd(lc, N)
126 |         if divisor > 1:
127 |             print('Divisisor of %s is %s'%(N,divisor))
128 |             return divisor, None, None
129 | 
130 |         q = r_i // r_i_plus
131 |         s_i_plus_plus = s_i - q * s_i_plus
132 |         t_i_plus_plus = t_i - q * t_i_plus
133 |         r_i_plus_plus = r_i - q * r_i_plus
134 |         if r_i_plus.degree() <= r_i_plus_plus.degree() or r_i_plus_plus.degree() == -1:
135 |             if toswap == True:
136 |                 assert (r_i_plus == s_i_plus * f + t_i_plus * g)
137 |                 return r_i_plus, t_i_plus, s_i_plus
138 |             else:
139 |                 assert (r_i_plus == s_i_plus * f + t_i_plus * g)
140 |                 return r_i_plus, s_i_plus, t_i_plus,
141 |         r_i, r_i_plus = r_i_plus, r_i_plus_plus
142 |         s_i, s_i_plus = s_i_plus, s_i_plus_plus
143 |         t_i, t_i_plus = t_i_plus, t_i_plus_plus
144 |         check_res = r_i == s_i * f + t_i * g
145 |         if not check_res:
146 |             logger.error('Assertion error: %s, %s != %s * %s + %s * %s' % (check_res, r_i, s_i, f, t_i, g))
147 |             raise ValueError('xgcd assertion error')
148 | 
149 | 
150 | def Qinverse(Q, a, N):
151 |     """
152 |     Q is a quotient ring of Z_N[x], a is the element to be inverted
153 |     :param Q:
154 |     :param a:
155 |     :return:
156 |     """
157 |     j = Q.gens()[0]
158 |     deg =  j.charpoly('X').degree()
159 |     A = Q(a).matrix()
160 |     det_a = det(A)
161 |     logger.debug('DetA: %s' % det_a)
162 | 
163 |     factor = gcd(int(det_a), N)
164 |     if factor!=1:  # a is not invertible
165 |         raise ZeroDivisionError(a)
166 | 
167 |     else:
168 |         Y = vector([1] + (deg-1)*[0])
169 |         X = A.solve_left(Y)
170 |         jvec = vector([j^i for i in [0..deg-1]])
171 |         Xj = jvec*X
172 |         return Xj
173 | 
174 | 
175 | def Qinverse2 (Hx, a, N, time_res):
176 |     ts = time.time()
177 |     r,s,t = xgcd(a.lift(), Hx, N)
178 |     txgcd = time.time()
179 |     if (s,t) == (None, None):
180 |         res = r, 0
181 |     else:
182 |         rinv = r[0]^(-1)
183 |         res = 1, s * rinv
184 |     tres = time.time()
185 | 
186 |     time_res.time_qinv_char_poly = 0
187 |     time_res.time_qinv_xgcd = txgcd - ts
188 |     time_res.time_qinv_res = tres - txgcd
189 |     return res
190 | 
191 | 
192 | def CMfactor(D, N, verb = 1, ctries=10, utries=10, fact_time=None, use_quinv2=False, use_cheng=False):
193 |     """
194 |     Try to factor N with respect to D, with ctries values of c and utries values of u
195 |     """
196 |     ts = time.time()
197 |     Hx = hilbert_class_polynomial(-D)
198 |     tth = time.time()
199 |     if verb == 1:
200 |         logger.debug('Hilbert polynomial computed for -%s!' % D)
201 | 
202 |     res = FactorRes()
203 |     res.use_quinv2 = use_quinv2
204 |     res.use_cheng = use_cheng
205 | 
206 |     ZN = Integers(N)
207 |     R.<x> = PolynomialRing(ZN)
208 | 
209 |     ttq = time.time()
210 |     try:
211 |         if use_quinv2:
212 |             Hx = R(Hx)
213 |             Q.<j> = QuotientRing(R, R.ideal(Hx))
214 |             gcd, inverse = Qinverse2(Hx, 1728 - j, N, res)  
215 |             if gcd == 1:
216 |                 a = Q(j * inverse)                
217 | 
218 |         else:
219 |             Q.<j> = ZN.extension(Hx)
220 |             a = j * Qinverse(Q, 1728 - j, N)
221 | 
222 |     except ZeroDivisionError as noninv:
223 |         logger.warning("is not invertible in Q! %s" % noninv)
224 |         raise NotInvertibleException()
225 |     if gcd != 1:
226 |         exit(-1)
227 |     if verb == 1:
228 |         logger.debug('Q constructed')
229 |         logger.debug('a computed: %s' % a)
230 | 
231 |     tta = time.time()
232 |     res.time_agg_div = 0
233 |     res.time_agg_gcd = 0
234 |     res.time_hilbert = tth - ts
235 |     res.time_q = ttq - tth
236 |     res.time_a = tta - ttq
237 |     res.a = None
238 | 
239 |     core_fnc = CMfactor_core
240 |     if fact_time:
241 |         time_left = fact_time - (tta - ts)
242 |         res.fact_timeout = time_left
243 |         core_fnc = fork(CMfactor_core, time_left)
244 |     cres = core_fnc(N, ctries, utries, a, Q, ZN, Hx, res, use_cheng=use_cheng)
245 |     if is_undef(cres):
246 |         res.out_of_time = True
247 |     else:
248 |         res = cres
249 | 
250 |     tdone = time.time()
251 |     res.time_total = tdone - ts
252 |     return res
253 | 
254 | 
255 | def CMfactor_core(N, ctries, utries, a, Q, ZN, Hx, res, use_cheng=False):
256 |     is_done = False
257 | 
258 |     # We prove this takes only one iteration
259 |     for c in [1..ctries]:
260 |         E = EllipticCurve(Q, [0, 0, 0, 3 * a * c ^ 2, 2 * a * c ^ 3])
261 | 
262 |         # expected number of u iterations: cn^2 / (cn^2 - 1)
263 |         for u in [1..utries]:
264 | 
265 |             # Division polynomial is the most expensive part here
266 |             tcs = time.time()
267 |             rand_elem = ZN.random_element()
268 |             res.rand_elem = int(rand_elem)
269 | 
270 |             w = E.division_polynomial(N, Q(rand_elem), two_torsion_multiplicity=0)
271 |             ttdiv = time.time()
272 |             logger.debug('Division polynomial done')
273 | 
274 |             if use_cheng:
275 |                 poly_gcd = xgcd(w.lift(), Hx, N)[0]
276 |                 ttnrm = time.time()
277 |                 r = gcd(ZZ(poly_gcd), N)
278 |                 ttgcd = time.time()
279 | 
280 |             else:
281 |                 nrm = w.norm()
282 |                 ttnrm = time.time()
283 |                 r = gcd(nrm, N)
284 |                 ttgcd = time.time()
285 | 
286 |             res.time_agg_div += ttdiv - tcs
287 |             res.time_agg_gcd += ttgcd - ttdiv
288 |             res.time_agg_nrm += ttnrm - ttdiv
289 |             res.c = int(c)
290 |             res.u = int(u)
291 |             res.time_last_div = ttdiv - tcs
292 |             res.time_last_gcd = ttgcd - ttdiv
293 |             res.time_last_nrm = ttnrm - ttdiv
294 | 
295 |             if r > 1 and r != N:
296 |                 res.r = int(r)
297 |                 logger.debug('A factor of N: %s' % r)
298 |                 logger.debug('c: %s, u: %s' % (c, u))
299 |                 is_done = True
300 |                 break
301 | 
302 |             else:
303 |                 logger.info('u failed: %s, next_attempt' % u)
304 | 
305 |         if is_done:
306 |             break
307 | 
308 |     return res
309 | 
310 | 
311 | def work_generate(args):
312 |     logger.debug('Generating CM prime with D=%s, bits=%s' % (args.disc, args.prime_bits))
313 |     p = generateCMprime(args.disc, args.prime_bits)
314 |     print(p)
315 | 
316 | 
317 | def work_factor(args):
318 |     disc = args.disc
319 |     class_num = class_number(disc)
320 |     sys.setrecursionlimit(50000)  # for the computation of division polynomials
321 | 
322 |     success = False
323 |     ts = time.time()
324 |     logger.debug('D: %s, class_num: %s' % (disc, class_num))
325 | 
326 |     try:
327 |         factor_timeout = args.timeout if args.timeout > 0 else None
328 |         res = CMfactor(disc, args.mod, 1, fact_time=factor_timeout, use_quinv2=args.qinv2, use_cheng=args.cheng)
329 |         res = -1 if is_undef(res) else res
330 |         result = res if res and not isinstance(res, int) else None
331 | 
332 |         time_total = time.time() - ts
333 |         logger.debug('Total time elapsed: %s' % time_total)
334 | 
335 |         if result.r > 1:
336 |             if args.mod % result.r != 0:
337 |                 raise ValueError('Found result is invalid')
338 | 
339 |             q = args.mod // result.r
340 |             success = True
341 |             print("Factorization of N: %s is: \n%s * %s" % (args.mod, result.r, q))
342 | 
343 |     except Exception as e:
344 |         logger.warning('Exception: %s' % e)
345 |         if args.debug:
346 |             traceback.print_exc()
347 | 
348 |     if not success:
349 |         print('Factorization failed')
350 | 
351 | 
352 | def main():
353 |     parser = argparse.ArgumentParser(description='CM factorization script')
354 |     parser.add_argument('--action', dest='action', action="store", default="factor",
355 |                         help='Action to perform, options: factor, generate')
356 | 
357 |     parser.add_argument('--modulus', '-N', dest='mod', action='store', type=int, default=158697752795669080171615843390068686677,
358 |                         help='Modulus to factorize')
359 |     parser.add_argument('--disc', '-D', dest='disc', action='store', type=int, default=11,
360 |                         help='D, the discriminant to compute factorization for')
361 |     parser.add_argument('--prime-bits', dest='prime_bits', action='store', type=int, default=256,
362 |                         help='Number of prime bits to generate')
363 | 
364 |     parser.add_argument('--timeout', dest='timeout', action="store", type=int, default=4*60,
365 |                         help='Number of seconds for the factorization job, negative for no timeout')
366 |     parser.add_argument('--qinv2', dest='qinv2', default=1, type=int,
367 |                         help='Use optimized inversion algorithm (enabled by default)')
368 |     parser.add_argument('--cheng', dest='cheng', default=1, type=int,
369 |                         help='Use Cheng xgcd instead of norms (enabled by default)')
370 | 
371 |     parser.add_argument('--debug', dest='debug', action='store_const', const=True, default=False,
372 |                         help='Debugging enabled')
373 | 
374 |     args = parser.parse_args()
375 |     if args.action is None or args.action == 'factor':
376 |         work_factor(args)
377 |     elif args.action == 'generate':
378 |         work_generate(args)
379 |     else:
380 |         raise ValueError('Unknown action: %s' % args.action)
381 | 
382 | 
383 | if __name__ == "__main__":
384 |     main()
385 | 
386 | 


--------------------------------------------------------------------------------
/coloredlogs/cli.py:
--------------------------------------------------------------------------------
  1 | # Command line interface for the coloredlogs package.
  2 | #
  3 | # Author: Peter Odding <peter@peterodding.com>
  4 | # Last Change: December 15, 2017
  5 | # URL: https://coloredlogs.readthedocs.io
  6 | 
  7 | """
  8 | Usage: coloredlogs [OPTIONS] [ARGS]
  9 | 
 10 | The coloredlogs program provides a simple command line interface for the Python
 11 | package by the same name.
 12 | 
 13 | Supported options:
 14 | 
 15 |   -c, --convert, --to-html
 16 | 
 17 |     Capture the output of an external command (given by the positional
 18 |     arguments) and convert ANSI escape sequences in the output to HTML.
 19 | 
 20 |     If the `coloredlogs' program is attached to an interactive terminal it will
 21 |     write the generated HTML to a temporary file and open that file in a web
 22 |     browser, otherwise the generated HTML will be written to standard output.
 23 | 
 24 |     This requires the `script' program to fake the external command into
 25 |     thinking that it's attached to an interactive terminal (in order to enable
 26 |     output of ANSI escape sequences).
 27 | 
 28 |     If the command didn't produce any output then no HTML will be produced on
 29 |     standard output, this is to avoid empty emails from cron jobs.
 30 | 
 31 |   -d, --demo
 32 | 
 33 |     Perform a simple demonstration of the coloredlogs package to show the
 34 |     colored logging on an interactive terminal.
 35 | 
 36 |   -h, --help
 37 | 
 38 |     Show this message and exit.
 39 | """
 40 | 
 41 | # Standard library modules.
 42 | import functools
 43 | import getopt
 44 | import logging
 45 | import sys
 46 | import tempfile
 47 | import webbrowser
 48 | 
 49 | # External dependencies.
 50 | from humanfriendly.terminal import connected_to_terminal, output, usage, warning
 51 | 
 52 | # Modules included in our package.
 53 | from coloredlogs.converter import capture, convert
 54 | from coloredlogs.demo import demonstrate_colored_logging
 55 | 
 56 | # Initialize a logger for this module.
 57 | logger = logging.getLogger(__name__)
 58 | 
 59 | 
 60 | def main():
 61 |     """Command line interface for the ``coloredlogs`` program."""
 62 |     actions = []
 63 |     try:
 64 |         # Parse the command line arguments.
 65 |         options, arguments = getopt.getopt(sys.argv[1:], 'cdh', [
 66 |             'convert', 'to-html', 'demo', 'help',
 67 |         ])
 68 |         # Map command line options to actions.
 69 |         for option, value in options:
 70 |             if option in ('-c', '--convert', '--to-html'):
 71 |                 actions.append(functools.partial(convert_command_output, *arguments))
 72 |                 arguments = []
 73 |             elif option in ('-d', '--demo'):
 74 |                 actions.append(demonstrate_colored_logging)
 75 |             elif option in ('-h', '--help'):
 76 |                 usage(__doc__)
 77 |                 return
 78 |             else:
 79 |                 assert False, "Programming error: Unhandled option!"
 80 |         if not actions:
 81 |             usage(__doc__)
 82 |             return
 83 |     except Exception as e:
 84 |         warning("Error: %s", e)
 85 |         sys.exit(1)
 86 |     for function in actions:
 87 |         function()
 88 | 
 89 | 
 90 | def convert_command_output(*command):
 91 |     """
 92 |     Command line interface for ``coloredlogs --to-html``.
 93 | 
 94 |     Takes a command (and its arguments) and runs the program under ``script``
 95 |     (emulating an interactive terminal), intercepts the output of the command
 96 |     and converts ANSI escape sequences in the output to HTML.
 97 |     """
 98 |     captured_output = capture(command)
 99 |     converted_output = convert(captured_output)
100 |     if connected_to_terminal():
101 |         fd, temporary_file = tempfile.mkstemp(suffix='.html')
102 |         with open(temporary_file, 'w') as handle:
103 |             handle.write(converted_output)
104 |         webbrowser.open(temporary_file)
105 |     elif captured_output and not captured_output.isspace():
106 |         output(converted_output)
107 | 


--------------------------------------------------------------------------------
/coloredlogs/converter/__init__.py:
--------------------------------------------------------------------------------
  1 | # Program to convert text with ANSI escape sequences to HTML.
  2 | #
  3 | # Author: Peter Odding <peter@peterodding.com>
  4 | # Last Change: January 17, 2018
  5 | # URL: https://coloredlogs.readthedocs.io
  6 | 
  7 | """Convert text with ANSI escape sequences to HTML."""
  8 | 
  9 | # Standard library modules.
 10 | import codecs
 11 | import os
 12 | import pipes
 13 | import re
 14 | import subprocess
 15 | import tempfile
 16 | 
 17 | # External dependencies.
 18 | from humanfriendly.terminal import (
 19 |     ANSI_CSI,
 20 |     ANSI_TEXT_STYLES,
 21 |     clean_terminal_output,
 22 |     output,
 23 | )
 24 | 
 25 | # Modules included in our package.
 26 | from coloredlogs.converter.colors import (
 27 |     BRIGHT_COLOR_PALETTE,
 28 |     EIGHT_COLOR_PALETTE,
 29 |     EXTENDED_COLOR_PALETTE,
 30 | )
 31 | 
 32 | # Compiled regular expression that matches leading spaces (indentation).
 33 | INDENT_PATTERN = re.compile('^ +', re.MULTILINE)
 34 | 
 35 | # Compiled regular expression that matches a tag followed by a space at the start of a line.
 36 | TAG_INDENT_PATTERN = re.compile('^(<[^>]+>) ', re.MULTILINE)
 37 | 
 38 | # Compiled regular expression that matches strings we want to convert. Used to
 39 | # separate all special strings and literal output in a single pass (this allows
 40 | # us to properly encode the output without resorting to nasty hacks).
 41 | TOKEN_PATTERN = re.compile('''
 42 |     # Wrap the pattern in a capture group so that re.split() includes the
 43 |     # substrings that match the pattern in the resulting list of strings.
 44 |     (
 45 |         # Match URLs with supported schemes and domain names.
 46 |         (?: https?:// | www\\. )
 47 |         # Scan until the end of the URL by matching non-whitespace characters
 48 |         # that are also not escape characters.
 49 |         [^\s\x1b]+
 50 |         # Alternatively ...
 51 |         |
 52 |         # Match (what looks like) ANSI escape sequences.
 53 |         \x1b \[ .*? m
 54 |     )
 55 | ''', re.UNICODE | re.VERBOSE)
 56 | 
 57 | 
 58 | def capture(command, encoding='UTF-8'):
 59 |     """
 60 |     Capture the output of an external command as if it runs in an interactive terminal.
 61 | 
 62 |     :param command: The command name and its arguments (a list of strings).
 63 |     :param encoding: The encoding to use to decode the output (a string).
 64 |     :returns: The output of the command.
 65 | 
 66 |     This function runs an external command under ``script`` (emulating an
 67 |     interactive terminal) to capture the output of the command as if it was
 68 |     running in an interactive terminal (including ANSI escape sequences).
 69 |     """
 70 |     with open(os.devnull, 'wb') as dev_null:
 71 |         # We start by invoking the `script' program in a form that is supported
 72 |         # by the Linux implementation [1] but fails command line validation on
 73 |         # the MacOS (BSD) implementation [2]: The command is specified using
 74 |         # the -c option and the typescript file is /dev/null.
 75 |         #
 76 |         # [1] http://man7.org/linux/man-pages/man1/script.1.html
 77 |         # [2] https://developer.apple.com/legacy/library/documentation/Darwin/Reference/ManPages/man1/script.1.html
 78 |         command_line = ['script', '-qc', ' '.join(map(pipes.quote, command)), '/dev/null']
 79 |         script = subprocess.Popen(command_line, stdout=subprocess.PIPE, stderr=dev_null)
 80 |         stdout, stderr = script.communicate()
 81 |         if script.returncode == 0:
 82 |             # If `script' succeeded we assume that it understood our command line
 83 |             # invocation which means it's the Linux implementation (in this case
 84 |             # we can use standard output instead of a temporary file).
 85 |             output = stdout.decode(encoding)
 86 |         else:
 87 |             # If `script' failed we assume that it didn't understand our command
 88 |             # line invocation which means it's the MacOS (BSD) implementation
 89 |             # (in this case we need a temporary file because the command line
 90 |             # interface requires it).
 91 |             fd, temporary_file = tempfile.mkstemp(prefix='coloredlogs-', suffix='-capture.txt')
 92 |             try:
 93 |                 command_line = ['script', '-q', temporary_file] + list(command)
 94 |                 subprocess.Popen(command_line, stdout=dev_null, stderr=dev_null).wait()
 95 |                 with codecs.open(temporary_file, 'r', encoding) as handle:
 96 |                     output = handle.read()
 97 |             finally:
 98 |                 os.unlink(temporary_file)
 99 |             # On MacOS when standard input is /dev/null I've observed
100 |             # the captured output starting with the characters '^D':
101 |             #
102 |             #   $ script -q capture.txt echo example </dev/null
103 |             #   example
104 |             #   $ xxd capture.txt
105 |             #   00000000: 5e44 0808 6578 616d 706c 650d 0a         ^D..example..
106 |             #
107 |             # I'm not sure why this is here, although I suppose it has to do
108 |             # with ^D in caret notation signifying end-of-file [1]. What I do
109 |             # know is that this is an implementation detail that callers of the
110 |             # capture() function shouldn't be bothered with, so we strip it.
111 |             #
112 |             # [1] https://en.wikipedia.org/wiki/End-of-file
113 |             if output.startswith(b'^D'):
114 |                 output = output[2:]
115 |     # Clean up backspace and carriage return characters and the 'erase line'
116 |     # ANSI escape sequence and return the output as a Unicode string.
117 |     return u'\n'.join(clean_terminal_output(output))
118 | 
119 | 
120 | def convert(text, code=True, tabsize=4):
121 |     """
122 |     Convert text with ANSI escape sequences to HTML.
123 | 
124 |     :param text: The text with ANSI escape sequences (a string).
125 |     :param code: Whether to wrap the returned HTML fragment in a
126 |                  ``<code>...</code>`` element (a boolean, defaults
127 |                  to :data:`True`).
128 |     :param tabsize: Refer to :func:`str.expandtabs()` for details.
129 |     :returns: The text converted to HTML (a string).
130 |     """
131 |     output = []
132 |     in_span = False
133 |     compatible_text_styles = {
134 |         # The following ANSI text styles have an obvious mapping to CSS.
135 |         ANSI_TEXT_STYLES['bold']: {'font-weight': 'bold'},
136 |         ANSI_TEXT_STYLES['strike_through']: {'text-decoration': 'line-through'},
137 |         ANSI_TEXT_STYLES['underline']: {'text-decoration': 'underline'},
138 |     }
139 |     for token in TOKEN_PATTERN.split(text):
140 |         if token.startswith(('http://', 'https://', 'www.')):
141 |             url = token if '://' in token else ('http://' + token)
142 |             token = u'<a href="%s" style="color:inherit">%s</a>' % (html_encode(url), html_encode(token))
143 |         elif token.startswith(ANSI_CSI):
144 |             ansi_codes = token[len(ANSI_CSI):-1].split(';')
145 |             if all(c.isdigit() for c in ansi_codes):
146 |                 ansi_codes = list(map(int, ansi_codes))
147 |             # First we check for a reset code to close the previous <span>
148 |             # element. As explained on Wikipedia [1] an absence of codes
149 |             # implies a reset code as well: "No parameters at all in ESC[m acts
150 |             # like a 0 reset code".
151 |             # [1] https://en.wikipedia.org/wiki/ANSI_escape_code#CSI_sequences
152 |             if in_span and (0 in ansi_codes or not ansi_codes):
153 |                 output.append('</span>')
154 |                 in_span = False
155 |             # Now we're ready to generate the next <span> element (if any) in
156 |             # the knowledge that we're emitting opening <span> and closing
157 |             # </span> tags in the correct order.
158 |             styles = {}
159 |             is_faint = (ANSI_TEXT_STYLES['faint'] in ansi_codes)
160 |             is_inverse = (ANSI_TEXT_STYLES['inverse'] in ansi_codes)
161 |             while ansi_codes:
162 |                 number = ansi_codes.pop(0)
163 |                 # Try to match a compatible text style.
164 |                 if number in compatible_text_styles:
165 |                     styles.update(compatible_text_styles[number])
166 |                     continue
167 |                 # Try to extract a text and/or background color.
168 |                 text_color = None
169 |                 background_color = None
170 |                 if 30 <= number <= 37:
171 |                     # 30-37 sets the text color from the eight color palette.
172 |                     text_color = EIGHT_COLOR_PALETTE[number - 30]
173 |                 elif 40 <= number <= 47:
174 |                     # 40-47 sets the background color from the eight color palette.
175 |                     background_color = EIGHT_COLOR_PALETTE[number - 40]
176 |                 elif 90 <= number <= 97:
177 |                     # 90-97 sets the text color from the high-intensity eight color palette.
178 |                     text_color = BRIGHT_COLOR_PALETTE[number - 90]
179 |                 elif 100 <= number <= 107:
180 |                     # 100-107 sets the background color from the high-intensity eight color palette.
181 |                     background_color = BRIGHT_COLOR_PALETTE[number - 100]
182 |                 elif number in (38, 39) and len(ansi_codes) >= 2 and ansi_codes[0] == 5:
183 |                     # 38;5;N is a text color in the 256 color mode palette,
184 |                     # 39;5;N is a background color in the 256 color mode palette.
185 |                     try:
186 |                         # Consume the 5 following 38 or 39.
187 |                         ansi_codes.pop(0)
188 |                         # Consume the 256 color mode color index.
189 |                         color_index = ansi_codes.pop(0)
190 |                         # Set the variable to the corresponding HTML/CSS color.
191 |                         if number == 38:
192 |                             text_color = EXTENDED_COLOR_PALETTE[color_index]
193 |                         elif number == 39:
194 |                             background_color = EXTENDED_COLOR_PALETTE[color_index]
195 |                     except (ValueError, IndexError):
196 |                         pass
197 |                 # Apply the 'faint' or 'inverse' text style
198 |                 # by manipulating the selected color(s).
199 |                 if text_color and is_inverse:
200 |                     # Use the text color as the background color and pick a
201 |                     # text color that will be visible on the resulting
202 |                     # background color.
203 |                     background_color = text_color
204 |                     text_color = select_text_color(*parse_hex_color(text_color))
205 |                 if text_color and is_faint:
206 |                     # Because I wasn't sure how to implement faint colors
207 |                     # based on normal colors I looked at how gnome-terminal
208 |                     # (my terminal of choice) handles this and it appears
209 |                     # to just pick a somewhat darker color.
210 |                     text_color = '#%02X%02X%02X' % tuple(
211 |                         max(0, n - 40) for n in parse_hex_color(text_color)
212 |                     )
213 |                 if text_color:
214 |                     styles['color'] = text_color
215 |                 if background_color:
216 |                     styles['background-color'] = background_color
217 |             if styles:
218 |                 token = '<span style="%s">' % ';'.join(k + ':' + v for k, v in sorted(styles.items()))
219 |                 in_span = True
220 |             else:
221 |                 token = ''
222 |         else:
223 |             token = html_encode(token)
224 |         output.append(token)
225 |     html = ''.join(output)
226 |     html = encode_whitespace(html, tabsize)
227 |     if code:
228 |         html = '<code>%s</code>' % html
229 |     return html
230 | 
231 | 
232 | def encode_whitespace(text, tabsize=4):
233 |     """
234 |     Encode whitespace so that web browsers properly render it.
235 | 
236 |     :param text: The plain text (a string).
237 |     :param tabsize: Refer to :func:`str.expandtabs()` for details.
238 |     :returns: The text converted to HTML (a string).
239 | 
240 |     The purpose of this function is to encode whitespace in such a way that web
241 |     browsers render the same whitespace regardless of whether 'preformatted'
242 |     styling is used (by wrapping the text in a ``<pre>...</pre>`` element).
243 | 
244 |     .. note:: While the string manipulation performed by this function is
245 |               specifically intended not to corrupt the HTML generated by
246 |               :func:`convert()` it definitely does have the potential to
247 |               corrupt HTML from other sources. You have been warned :-).
248 |     """
249 |     # Convert Windows line endings (CR+LF) to UNIX line endings (LF).
250 |     text = text.replace('\r\n', '\n')
251 |     # Convert UNIX line endings (LF) to HTML line endings (<br>).
252 |     text = text.replace('\n', '<br>\n')
253 |     # Convert tabs to spaces.
254 |     text = text.expandtabs(tabsize)
255 |     # Convert leading spaces (that is to say spaces at the start of the string
256 |     # and/or directly after a line ending) into non-breaking spaces, otherwise
257 |     # HTML rendering engines will simply ignore these spaces.
258 |     text = re.sub(INDENT_PATTERN, encode_whitespace_cb, text)
259 |     # The conversion of leading spaces we just did misses a corner case where a
260 |     # line starts with an HTML tag but the first visible text is a space. Web
261 |     # browsers seem to ignore these spaces, so we need to convert them.
262 |     text = re.sub(TAG_INDENT_PATTERN, r'\1&nbsp;', text)
263 |     # Convert runs of multiple spaces into non-breaking spaces to avoid HTML
264 |     # rendering engines from visually collapsing runs of spaces into a single
265 |     # space. We specifically don't replace single spaces for several reasons:
266 |     # 1. We'd break the HTML emitted by convert() by replacing spaces
267 |     #    inside HTML elements (for example the spaces that separate
268 |     #    element names from attribute names).
269 |     # 2. If every single space is replaced by a non-breaking space,
270 |     #    web browsers perform awkwardly unintuitive word wrapping.
271 |     # 3. The HTML output would be bloated for no good reason.
272 |     text = re.sub(' {2,}', encode_whitespace_cb, text)
273 |     return text
274 | 
275 | 
276 | def encode_whitespace_cb(match):
277 |     """
278 |     Replace runs of multiple spaces with non-breaking spaces.
279 | 
280 |     :param match: A regular expression match object.
281 |     :returns: The replacement string.
282 | 
283 |     This function is used by func:`encode_whitespace()` as a callback for
284 |     replacement using a regular expression pattern.
285 |     """
286 |     return '&nbsp;' * len(match.group(0))
287 | 
288 | 
289 | def html_encode(text):
290 |     """
291 |     Encode characters with a special meaning as HTML.
292 | 
293 |     :param text: The plain text (a string).
294 |     :returns: The text converted to HTML (a string).
295 |     """
296 |     text = text.replace('&', '&amp;')
297 |     text = text.replace('<', '&lt;')
298 |     text = text.replace('>', '&gt;')
299 |     text = text.replace('"', '&quot;')
300 |     return text
301 | 
302 | 
303 | def parse_hex_color(value):
304 |     """
305 |     Convert a CSS color in hexadecimal notation into its R, G, B components.
306 | 
307 |     :param value: A CSS color in hexadecimal notation (a string like '#000000').
308 |     :return: A tuple with three integers (with values between 0 and 255)
309 |              corresponding to the R, G and B components of the color.
310 |     :raises: :exc:`~exceptions.ValueError` on values that can't be parsed.
311 |     """
312 |     if value.startswith('#'):
313 |         value = value[1:]
314 |     if len(value) == 3:
315 |         return (
316 |             int(value[0] * 2, 16),
317 |             int(value[1] * 2, 16),
318 |             int(value[2] * 2, 16),
319 |         )
320 |     elif len(value) == 6:
321 |         return (
322 |             int(value[0:2], 16),
323 |             int(value[2:4], 16),
324 |             int(value[4:6], 16),
325 |         )
326 |     else:
327 |         raise ValueError()
328 | 
329 | 
330 | def select_text_color(r, g, b):
331 |     """
332 |     Choose a suitable color for the inverse text style.
333 | 
334 |     :param r: The amount of red (an integer between 0 and 255).
335 |     :param g: The amount of green (an integer between 0 and 255).
336 |     :param b: The amount of blue (an integer between 0 and 255).
337 |     :returns: A CSS color in hexadecimal notation (a string).
338 | 
339 |     In inverse mode the color that is normally used for the text is instead
340 |     used for the background, however this can render the text unreadable. The
341 |     purpose of :func:`select_text_color()` is to make an effort to select a
342 |     suitable text color. Based on http://stackoverflow.com/a/3943023/112731.
343 |     """
344 |     return '#000' if (r * 0.299 + g * 0.587 + b * 0.114) > 186 else '#FFF'
345 | 
346 | 
347 | class ColoredCronMailer(object):
348 | 
349 |     """
350 |     Easy to use integration between :mod:`coloredlogs` and the UNIX ``cron`` daemon.
351 | 
352 |     By using :class:`ColoredCronMailer` as a context manager in the command
353 |     line interface of your Python program you make it trivially easy for users
354 |     of your program to opt in to HTML output under ``cron``: The only thing the
355 |     user needs to do is set ``CONTENT_TYPE="text/html"`` in their crontab!
356 | 
357 |     Under the hood this requires quite a bit of magic and I must admit that I
358 |     developed this code simply because I was curious whether it could even be
359 |     done :-). It requires my :mod:`capturer` package which you can install
360 |     using ``pip install 'coloredlogs[cron]'``. The ``[cron]`` extra will pull
361 |     in the :mod:`capturer` 2.4 or newer which is required to capture the output
362 |     while silencing it - otherwise you'd get duplicate output in the emails
363 |     sent by ``cron``.
364 |     """
365 | 
366 |     def __init__(self):
367 |         """Initialize output capturing when running under ``cron`` with the correct configuration."""
368 |         self.is_enabled = 'text/html' in os.environ.get('CONTENT_TYPE', 'text/plain')
369 |         self.is_silent = False
370 |         if self.is_enabled:
371 |             # We import capturer here so that the coloredlogs[cron] extra
372 |             # isn't required to use the other functions in this module.
373 |             from capturer import CaptureOutput
374 |             self.capturer = CaptureOutput(merged=True, relay=False)
375 | 
376 |     def __enter__(self):
377 |         """Start capturing output (when applicable)."""
378 |         if self.is_enabled:
379 |             self.capturer.__enter__()
380 |         return self
381 | 
382 |     def __exit__(self, exc_type=None, exc_value=None, traceback=None):
383 |         """Stop capturing output and convert the output to HTML (when applicable)."""
384 |         if self.is_enabled:
385 |             if not self.is_silent:
386 |                 # Only call output() when we captured something useful.
387 |                 text = self.capturer.get_text()
388 |                 if text and not text.isspace():
389 |                     output(convert(text))
390 |             self.capturer.__exit__(exc_type, exc_value, traceback)
391 | 
392 |     def silence(self):
393 |         """
394 |         Tell :func:`__exit__()` to swallow all output (things will be silent).
395 | 
396 |         This can be useful when a Python program is written in such a way that
397 |         it has already produced output by the time it becomes apparent that
398 |         nothing useful can be done (say in a cron job that runs every few
399 |         minutes :-p). By calling :func:`silence()` the output can be swallowed
400 |         retroactively, avoiding useless emails from ``cron``.
401 |         """
402 |         self.is_silent = True
403 | 


--------------------------------------------------------------------------------
/coloredlogs/converter/colors.py:
--------------------------------------------------------------------------------
  1 | # Mapping of ANSI color codes to HTML/CSS colors.
  2 | #
  3 | # Author: Peter Odding <peter@peterodding.com>
  4 | # Last Change: January 14, 2018
  5 | # URL: https://coloredlogs.readthedocs.io
  6 | 
  7 | """Mapping of ANSI color codes to HTML/CSS colors."""
  8 | 
  9 | EIGHT_COLOR_PALETTE = (
 10 |     '#010101',  # black
 11 |     '#DE382B',  # red
 12 |     '#39B54A',  # green
 13 |     '#FFC706',  # yellow
 14 |     '#006FB8',  # blue
 15 |     '#762671',  # magenta
 16 |     '#2CB5E9',  # cyan
 17 |     '#CCC',     # white
 18 | )
 19 | """
 20 | A tuple of strings mapping basic color codes to CSS colors.
 21 | 
 22 | The items in this tuple correspond to the eight basic color codes for black,
 23 | red, green, yellow, blue, magenta, cyan and white as defined in the original
 24 | standard for ANSI escape sequences. The CSS colors are based on the `Ubuntu
 25 | color scheme`_ described on Wikipedia and they are encoded as hexadecimal
 26 | values to get the shortest strings, which reduces the size (in bytes) of
 27 | conversion output.
 28 | 
 29 | .. _Ubuntu color scheme: https://en.wikipedia.org/wiki/ANSI_escape_code#Colors
 30 | """
 31 | 
 32 | BRIGHT_COLOR_PALETTE = (
 33 |     '#808080',  # black
 34 |     '#F00',     # red
 35 |     '#0F0',     # green
 36 |     '#FF0',     # yellow
 37 |     '#00F',     # blue
 38 |     '#F0F',     # magenta
 39 |     '#0FF',     # cyan
 40 |     '#FFF',     # white
 41 | )
 42 | """
 43 | A tuple of strings mapping bright color codes to CSS colors.
 44 | 
 45 | This tuple maps the bright color variants of :data:`EIGHT_COLOR_PALETTE`.
 46 | """
 47 | 
 48 | EXTENDED_COLOR_PALETTE = (
 49 |     '#000000',
 50 |     '#800000',
 51 |     '#008000',
 52 |     '#808000',
 53 |     '#000080',
 54 |     '#800080',
 55 |     '#008080',
 56 |     '#C0C0C0',
 57 |     '#808080',
 58 |     '#FF0000',
 59 |     '#00FF00',
 60 |     '#FFFF00',
 61 |     '#0000FF',
 62 |     '#FF00FF',
 63 |     '#00FFFF',
 64 |     '#FFFFFF',
 65 |     '#000000',
 66 |     '#00005F',
 67 |     '#000087',
 68 |     '#0000AF',
 69 |     '#0000D7',
 70 |     '#0000FF',
 71 |     '#005F00',
 72 |     '#005F5F',
 73 |     '#005F87',
 74 |     '#005FAF',
 75 |     '#005FD7',
 76 |     '#005FFF',
 77 |     '#008700',
 78 |     '#00875F',
 79 |     '#008787',
 80 |     '#0087AF',
 81 |     '#0087D7',
 82 |     '#0087FF',
 83 |     '#00AF00',
 84 |     '#00AF5F',
 85 |     '#00AF87',
 86 |     '#00AFAF',
 87 |     '#00AFD7',
 88 |     '#00AFFF',
 89 |     '#00D700',
 90 |     '#00D75F',
 91 |     '#00D787',
 92 |     '#00D7AF',
 93 |     '#00D7D7',
 94 |     '#00D7FF',
 95 |     '#00FF00',
 96 |     '#00FF5F',
 97 |     '#00FF87',
 98 |     '#00FFAF',
 99 |     '#00FFD7',
100 |     '#00FFFF',
101 |     '#5F0000',
102 |     '#5F005F',
103 |     '#5F0087',
104 |     '#5F00AF',
105 |     '#5F00D7',
106 |     '#5F00FF',
107 |     '#5F5F00',
108 |     '#5F5F5F',
109 |     '#5F5F87',
110 |     '#5F5FAF',
111 |     '#5F5FD7',
112 |     '#5F5FFF',
113 |     '#5F8700',
114 |     '#5F875F',
115 |     '#5F8787',
116 |     '#5F87AF',
117 |     '#5F87D7',
118 |     '#5F87FF',
119 |     '#5FAF00',
120 |     '#5FAF5F',
121 |     '#5FAF87',
122 |     '#5FAFAF',
123 |     '#5FAFD7',
124 |     '#5FAFFF',
125 |     '#5FD700',
126 |     '#5FD75F',
127 |     '#5FD787',
128 |     '#5FD7AF',
129 |     '#5FD7D7',
130 |     '#5FD7FF',
131 |     '#5FFF00',
132 |     '#5FFF5F',
133 |     '#5FFF87',
134 |     '#5FFFAF',
135 |     '#5FFFD7',
136 |     '#5FFFFF',
137 |     '#870000',
138 |     '#87005F',
139 |     '#870087',
140 |     '#8700AF',
141 |     '#8700D7',
142 |     '#8700FF',
143 |     '#875F00',
144 |     '#875F5F',
145 |     '#875F87',
146 |     '#875FAF',
147 |     '#875FD7',
148 |     '#875FFF',
149 |     '#878700',
150 |     '#87875F',
151 |     '#878787',
152 |     '#8787AF',
153 |     '#8787D7',
154 |     '#8787FF',
155 |     '#87AF00',
156 |     '#87AF5F',
157 |     '#87AF87',
158 |     '#87AFAF',
159 |     '#87AFD7',
160 |     '#87AFFF',
161 |     '#87D700',
162 |     '#87D75F',
163 |     '#87D787',
164 |     '#87D7AF',
165 |     '#87D7D7',
166 |     '#87D7FF',
167 |     '#87FF00',
168 |     '#87FF5F',
169 |     '#87FF87',
170 |     '#87FFAF',
171 |     '#87FFD7',
172 |     '#87FFFF',
173 |     '#AF0000',
174 |     '#AF005F',
175 |     '#AF0087',
176 |     '#AF00AF',
177 |     '#AF00D7',
178 |     '#AF00FF',
179 |     '#AF5F00',
180 |     '#AF5F5F',
181 |     '#AF5F87',
182 |     '#AF5FAF',
183 |     '#AF5FD7',
184 |     '#AF5FFF',
185 |     '#AF8700',
186 |     '#AF875F',
187 |     '#AF8787',
188 |     '#AF87AF',
189 |     '#AF87D7',
190 |     '#AF87FF',
191 |     '#AFAF00',
192 |     '#AFAF5F',
193 |     '#AFAF87',
194 |     '#AFAFAF',
195 |     '#AFAFD7',
196 |     '#AFAFFF',
197 |     '#AFD700',
198 |     '#AFD75F',
199 |     '#AFD787',
200 |     '#AFD7AF',
201 |     '#AFD7D7',
202 |     '#AFD7FF',
203 |     '#AFFF00',
204 |     '#AFFF5F',
205 |     '#AFFF87',
206 |     '#AFFFAF',
207 |     '#AFFFD7',
208 |     '#AFFFFF',
209 |     '#D70000',
210 |     '#D7005F',
211 |     '#D70087',
212 |     '#D700AF',
213 |     '#D700D7',
214 |     '#D700FF',
215 |     '#D75F00',
216 |     '#D75F5F',
217 |     '#D75F87',
218 |     '#D75FAF',
219 |     '#D75FD7',
220 |     '#D75FFF',
221 |     '#D78700',
222 |     '#D7875F',
223 |     '#D78787',
224 |     '#D787AF',
225 |     '#D787D7',
226 |     '#D787FF',
227 |     '#D7AF00',
228 |     '#D7AF5F',
229 |     '#D7AF87',
230 |     '#D7AFAF',
231 |     '#D7AFD7',
232 |     '#D7AFFF',
233 |     '#D7D700',
234 |     '#D7D75F',
235 |     '#D7D787',
236 |     '#D7D7AF',
237 |     '#D7D7D7',
238 |     '#D7D7FF',
239 |     '#D7FF00',
240 |     '#D7FF5F',
241 |     '#D7FF87',
242 |     '#D7FFAF',
243 |     '#D7FFD7',
244 |     '#D7FFFF',
245 |     '#FF0000',
246 |     '#FF005F',
247 |     '#FF0087',
248 |     '#FF00AF',
249 |     '#FF00D7',
250 |     '#FF00FF',
251 |     '#FF5F00',
252 |     '#FF5F5F',
253 |     '#FF5F87',
254 |     '#FF5FAF',
255 |     '#FF5FD7',
256 |     '#FF5FFF',
257 |     '#FF8700',
258 |     '#FF875F',
259 |     '#FF8787',
260 |     '#FF87AF',
261 |     '#FF87D7',
262 |     '#FF87FF',
263 |     '#FFAF00',
264 |     '#FFAF5F',
265 |     '#FFAF87',
266 |     '#FFAFAF',
267 |     '#FFAFD7',
268 |     '#FFAFFF',
269 |     '#FFD700',
270 |     '#FFD75F',
271 |     '#FFD787',
272 |     '#FFD7AF',
273 |     '#FFD7D7',
274 |     '#FFD7FF',
275 |     '#FFFF00',
276 |     '#FFFF5F',
277 |     '#FFFF87',
278 |     '#FFFFAF',
279 |     '#FFFFD7',
280 |     '#FFFFFF',
281 |     '#080808',
282 |     '#121212',
283 |     '#1C1C1C',
284 |     '#262626',
285 |     '#303030',
286 |     '#3A3A3A',
287 |     '#444444',
288 |     '#4E4E4E',
289 |     '#585858',
290 |     '#626262',
291 |     '#6C6C6C',
292 |     '#767676',
293 |     '#808080',
294 |     '#8A8A8A',
295 |     '#949494',
296 |     '#9E9E9E',
297 |     '#A8A8A8',
298 |     '#B2B2B2',
299 |     '#BCBCBC',
300 |     '#C6C6C6',
301 |     '#D0D0D0',
302 |     '#DADADA',
303 |     '#E4E4E4',
304 |     '#EEEEEE',
305 | )
306 | """
307 | A tuple of strings mapping 256 color mode color codes to CSS colors.
308 | 
309 | The items in this tuple correspond to the color codes in the 256 color mode palette.
310 | """
311 | 


--------------------------------------------------------------------------------
/coloredlogs/demo.py:
--------------------------------------------------------------------------------
 1 | # Demonstration of the coloredlogs package.
 2 | #
 3 | # Author: Peter Odding <peter@peterodding.com>
 4 | # Last Change: January 14, 2018
 5 | # URL: https://coloredlogs.readthedocs.io
 6 | 
 7 | """A simple demonstration of the `coloredlogs` package."""
 8 | 
 9 | # Standard library modules.
10 | import os
11 | import time
12 | 
13 | # Modules included in our package.
14 | import coloredlogs
15 | 
16 | # If my verbose logger is installed, we'll use that for the demo.
17 | try:
18 |     from verboselogs import VerboseLogger as getLogger
19 | except ImportError:
20 |     from logging import getLogger
21 | 
22 | # Initialize a logger for this module.
23 | logger = getLogger(__name__)
24 | 
25 | DEMO_DELAY = float(os.environ.get('COLOREDLOGS_DEMO_DELAY', '1'))
26 | """The number of seconds between each message emitted by :func:`demonstrate_colored_logging()`."""
27 | 
28 | 
29 | def demonstrate_colored_logging():
30 |     """Interactively demonstrate the :mod:`coloredlogs` package."""
31 |     # Determine the available logging levels and order them by numeric value.
32 |     decorated_levels = []
33 |     defined_levels = coloredlogs.find_defined_levels()
34 |     normalizer = coloredlogs.NameNormalizer()
35 |     for name, level in defined_levels.items():
36 |         if name != 'NOTSET':
37 |             item = (level, normalizer.normalize_name(name))
38 |             if item not in decorated_levels:
39 |                 decorated_levels.append(item)
40 |     ordered_levels = sorted(decorated_levels)
41 |     # Initialize colored output to the terminal, default to the most
42 |     # verbose logging level but enable the user the customize it.
43 |     coloredlogs.install(level=os.environ.get('COLOREDLOGS_LOG_LEVEL', ordered_levels[0][1]))
44 |     # Print some examples with different timestamps.
45 |     for level, name in ordered_levels:
46 |         log_method = getattr(logger, name, None)
47 |         if log_method:
48 |             log_method("message with level %s (%i)", name, level)
49 |             time.sleep(DEMO_DELAY)
50 | 


--------------------------------------------------------------------------------
/coloredlogs/syslog.py:
--------------------------------------------------------------------------------
  1 | # Easy to use system logging for Python's logging module.
  2 | #
  3 | # Author: Peter Odding <peter@peterodding.com>
  4 | # Last Change: April 17, 2017
  5 | # URL: https://coloredlogs.readthedocs.io
  6 | 
  7 | """
  8 | Easy to use UNIX system logging for Python's :mod:`logging` module.
  9 | 
 10 | Admittedly system logging has little to do with colored terminal output, however:
 11 | 
 12 | - The `coloredlogs` package is my attempt to do Python logging right and system
 13 |   logging is an important part of that equation.
 14 | 
 15 | - I've seen a surprising number of quirks and mistakes in system logging done
 16 |   in Python, for example including ``%(asctime)s`` in a format string (the
 17 |   system logging daemon is responsible for adding timestamps and thus you end
 18 |   up with duplicate timestamps that make the logs awful to read :-).
 19 | 
 20 | - The ``%(programname)s`` filter originated in my system logging code and I
 21 |   wanted it in `coloredlogs` so the step to include this module wasn't that big.
 22 | 
 23 | - As a bonus this Python module now has a test suite and proper documentation.
 24 | 
 25 | So there :-P. Go take a look at :func:`enable_system_logging()`.
 26 | """
 27 | 
 28 | # Standard library modules.
 29 | import logging
 30 | import logging.handlers
 31 | import os
 32 | import socket
 33 | import sys
 34 | 
 35 | # Modules included in our package.
 36 | from coloredlogs import (
 37 |     DEFAULT_LOG_LEVEL,
 38 |     ProgramNameFilter,
 39 |     adjust_level,
 40 |     find_program_name,
 41 |     level_to_number,
 42 |     replace_handler,
 43 | )
 44 | 
 45 | LOG_DEVICE_MACOSX = '/var/run/syslog'
 46 | """The pathname of the log device on Mac OS X (a string)."""
 47 | 
 48 | LOG_DEVICE_UNIX = '/dev/log'
 49 | """The pathname of the log device on Linux and most other UNIX systems (a string)."""
 50 | 
 51 | DEFAULT_LOG_FORMAT = '%(programname)s[%(process)d]: %(levelname)s %(message)s'
 52 | """
 53 | The default format for log messages sent to the system log (a string).
 54 | 
 55 | The ``%(programname)s`` format requires :class:`~coloredlogs.ProgramNameFilter`
 56 | but :func:`enable_system_logging()` takes care of this for you.
 57 | 
 58 | The ``name[pid]:`` construct (specifically the colon) in the format allows
 59 | rsyslogd_ to extract the ``$programname`` from each log message, which in turn
 60 | allows configuration files in ``/etc/rsyslog.d/*.conf`` to filter these log
 61 | messages to a separate log file (if the need arises).
 62 | 
 63 | .. _rsyslogd: https://en.wikipedia.org/wiki/Rsyslog
 64 | """
 65 | 
 66 | # Initialize a logger for this module.
 67 | logger = logging.getLogger(__name__)
 68 | 
 69 | 
 70 | class SystemLogging(object):
 71 | 
 72 |     """Context manager to enable system logging."""
 73 | 
 74 |     def __init__(self, *args, **kw):
 75 |         """
 76 |         Initialize a :class:`SystemLogging` object.
 77 | 
 78 |         :param args: Positional arguments to :func:`enable_system_logging()`.
 79 |         :param kw: Keyword arguments to :func:`enable_system_logging()`.
 80 |         """
 81 |         self.args = args
 82 |         self.kw = kw
 83 |         self.handler = None
 84 | 
 85 |     def __enter__(self):
 86 |         """Enable system logging when entering the context."""
 87 |         if self.handler is None:
 88 |             self.handler = enable_system_logging(*self.args, **self.kw)
 89 |         return self.handler
 90 | 
 91 |     def __exit__(self, exc_type=None, exc_value=None, traceback=None):
 92 |         """
 93 |         Disable system logging when leaving the context.
 94 | 
 95 |         .. note:: If an exception is being handled when we leave the context a
 96 |                   warning message including traceback is logged *before* system
 97 |                   logging is disabled.
 98 |         """
 99 |         if self.handler is not None:
100 |             if exc_type is not None:
101 |                 logger.warning("Disabling system logging due to unhandled exception!", exc_info=True)
102 |             (self.kw.get('logger') or logging.getLogger()).removeHandler(self.handler)
103 |             self.handler = None
104 | 
105 | 
106 | def enable_system_logging(programname=None, fmt=None, logger=None, reconfigure=True, **kw):
107 |     """
108 |     Redirect :mod:`logging` messages to the system log (e.g. ``/var/log/syslog``).
109 | 
110 |     :param programname: The program name to embed in log messages (a string, defaults
111 |                          to the result of :func:`~coloredlogs.find_program_name()`).
112 |     :param fmt: The log format for system log messages (a string, defaults to
113 |                 :data:`DEFAULT_LOG_FORMAT`).
114 |     :param logger: The logger to which the :class:`~logging.handlers.SysLogHandler`
115 |                    should be connected (defaults to the root logger).
116 |     :param level: The logging level for the :class:`~logging.handlers.SysLogHandler`
117 |                   (defaults to :data:`.DEFAULT_LOG_LEVEL`). This value is coerced
118 |                   using :func:`~coloredlogs.level_to_number()`.
119 |     :param reconfigure: If :data:`True` (the default) multiple calls to
120 |                         :func:`enable_system_logging()` will each override
121 |                         the previous configuration.
122 |     :param kw: Refer to :func:`connect_to_syslog()`.
123 |     :returns: A :class:`~logging.handlers.SysLogHandler` object or
124 |               :data:`None`. If an existing handler is found and `reconfigure`
125 |               is :data:`False` the existing handler object is returned. If the
126 |               connection to the system logging daemon fails :data:`None` is
127 |               returned.
128 | 
129 |     .. note:: When the logger's effective level is too restrictive it is
130 |               relaxed (refer to `notes about log levels`_ for details).
131 |     """
132 |     # Provide defaults for omitted arguments.
133 |     programname = programname or find_program_name()
134 |     logger = logger or logging.getLogger()
135 |     fmt = fmt or DEFAULT_LOG_FORMAT
136 |     level = level_to_number(kw.get('level', DEFAULT_LOG_LEVEL))
137 |     # Check whether system logging is already enabled.
138 |     handler, logger = replace_handler(logger, match_syslog_handler, reconfigure)
139 |     # Make sure reconfiguration is allowed or not relevant.
140 |     if not (handler and not reconfigure):
141 |         # Create a system logging handler.
142 |         handler = connect_to_syslog(**kw)
143 |         # Make sure the handler was successfully created.
144 |         if handler:
145 |             # Enable the use of %(programname)s.
146 |             ProgramNameFilter.install(handler=handler, fmt=fmt, programname=programname)
147 |             # Connect the formatter, handler and logger.
148 |             handler.setFormatter(logging.Formatter(fmt))
149 |             logger.addHandler(handler)
150 |             # Adjust the level of the selected logger.
151 |             adjust_level(logger, level)
152 |     return handler
153 | 
154 | 
155 | def connect_to_syslog(address=None, facility=None, level=None):
156 |     """
157 |     Create a :class:`~logging.handlers.SysLogHandler`.
158 | 
159 |     :param address: The device file or network address of the system logging
160 |                     daemon (a string or tuple, defaults to the result of
161 |                     :func:`find_syslog_address()`).
162 |     :param facility: Refer to :class:`~logging.handlers.SysLogHandler`.
163 |                      Defaults to ``LOG_USER``.
164 |     :param level: The logging level for the :class:`~logging.handlers.SysLogHandler`
165 |                   (defaults to :data:`.DEFAULT_LOG_LEVEL`). This value is coerced
166 |                   using :func:`~coloredlogs.level_to_number()`.
167 |     :returns: A :class:`~logging.handlers.SysLogHandler` object or :data:`None` (if the
168 |               system logging daemon is unavailable).
169 | 
170 |     The process of connecting to the system logging daemon goes as follows:
171 | 
172 |     - If :class:`~logging.handlers.SysLogHandler` supports the `socktype`
173 |       option (it does since Python 2.7) the following two socket types are
174 |       tried (in decreasing preference):
175 | 
176 |        1. :data:`~socket.SOCK_RAW` avoids truncation of log messages but may
177 |           not be supported.
178 |        2. :data:`~socket.SOCK_STREAM` (TCP) supports longer messages than the
179 |           default (which is UDP).
180 | 
181 |     - If socket types are not supported Python's (2.6) defaults are used to
182 |       connect to the selected `address`.
183 |     """
184 |     if not address:
185 |         address = find_syslog_address()
186 |     if facility is None:
187 |         facility = logging.handlers.SysLogHandler.LOG_USER
188 |     if level is None:
189 |         level = DEFAULT_LOG_LEVEL
190 |     for socktype in socket.SOCK_RAW, socket.SOCK_STREAM, None:
191 |         kw = dict(facility=facility, address=address)
192 |         if socktype is not None:
193 |             kw['socktype'] = socktype
194 |         try:
195 |             handler = logging.handlers.SysLogHandler(**kw)
196 |         except (IOError, TypeError):
197 |             # The socktype argument was added in Python 2.7 and its use will raise a
198 |             # TypeError exception on Python 2.6. IOError is a superclass of socket.error
199 |             # (since Python 2.6) which can be raised if the system logging daemon is
200 |             # unavailable.
201 |             pass
202 |         else:
203 |             handler.setLevel(level_to_number(level))
204 |             return handler
205 | 
206 | 
207 | def find_syslog_address():
208 |     """
209 |     Find the most suitable destination for system log messages.
210 | 
211 |     :returns: The pathname of a log device (a string) or an address/port tuple as
212 |               supported by :class:`~logging.handlers.SysLogHandler`.
213 | 
214 |     On Mac OS X this prefers :data:`LOG_DEVICE_MACOSX`, after that :data:`LOG_DEVICE_UNIX`
215 |     is checked for existence. If both of these device files don't exist the default used
216 |     by :class:`~logging.handlers.SysLogHandler` is returned.
217 |     """
218 |     if sys.platform == 'darwin' and os.path.exists(LOG_DEVICE_MACOSX):
219 |         return LOG_DEVICE_MACOSX
220 |     elif os.path.exists(LOG_DEVICE_UNIX):
221 |         return LOG_DEVICE_UNIX
222 |     else:
223 |         return 'localhost', logging.handlers.SYSLOG_UDP_PORT
224 | 
225 | 
226 | def match_syslog_handler(handler):
227 |     """
228 |     Identify system logging handlers.
229 | 
230 |     :param handler: The :class:`~logging.Handler` class to check.
231 |     :returns: :data:`True` if the handler is a
232 |               :class:`~logging.handlers.SysLogHandler`,
233 |               :data:`False` otherwise.
234 | 
235 |     This function can be used as a callback for :func:`.find_handler()`.
236 |     """
237 |     return isinstance(handler, logging.handlers.SysLogHandler)
238 | 


--------------------------------------------------------------------------------
/coloredlogs/tests.py:
--------------------------------------------------------------------------------
  1 | # Automated tests for the `coloredlogs' package.
  2 | #
  3 | # Author: Peter Odding <peter@peterodding.com>
  4 | # Last Change: April 29, 2018
  5 | # URL: https://coloredlogs.readthedocs.io
  6 | 
  7 | """Automated tests for the `coloredlogs` package."""
  8 | 
  9 | # Standard library modules.
 10 | import contextlib
 11 | import imp
 12 | import logging
 13 | import logging.handlers
 14 | import os
 15 | import re
 16 | import subprocess
 17 | import sys
 18 | import tempfile
 19 | 
 20 | # External dependencies.
 21 | from humanfriendly.compat import StringIO
 22 | from humanfriendly.terminal import ANSI_COLOR_CODES, ansi_style, ansi_wrap
 23 | from humanfriendly.testing import PatchedItem, TestCase, retry
 24 | from humanfriendly.text import format, random_string
 25 | from mock import MagicMock
 26 | 
 27 | # The module we're testing.
 28 | import coloredlogs
 29 | import coloredlogs.cli
 30 | from coloredlogs import (
 31 |     CHROOT_FILES,
 32 |     ColoredFormatter,
 33 |     NameNormalizer,
 34 |     decrease_verbosity,
 35 |     find_defined_levels,
 36 |     find_handler,
 37 |     find_hostname,
 38 |     find_program_name,
 39 |     get_level,
 40 |     increase_verbosity,
 41 |     install,
 42 |     is_verbose,
 43 |     level_to_number,
 44 |     match_stream_handler,
 45 |     parse_encoded_styles,
 46 |     set_level,
 47 |     walk_propagation_tree,
 48 | )
 49 | from coloredlogs.syslog import SystemLogging, match_syslog_handler
 50 | from coloredlogs.converter import (
 51 |     ColoredCronMailer,
 52 |     EIGHT_COLOR_PALETTE,
 53 |     capture,
 54 |     convert,
 55 | )
 56 | 
 57 | # External test dependencies.
 58 | from capturer import CaptureOutput
 59 | from verboselogs import VerboseLogger
 60 | 
 61 | # Compiled regular expression that matches a single line of output produced by
 62 | # the default log format (does not include matching of ANSI escape sequences).
 63 | PLAIN_TEXT_PATTERN = re.compile(r'''
 64 |     (?P<date> \d{4}-\d{2}-\d{2} )
 65 |     \s (?P<time> \d{2}:\d{2}:\d{2} )
 66 |     \s (?P<hostname> \S+ )
 67 |     \s (?P<logger_name> \w+ )
 68 |     \[ (?P<process_id> \d+ ) \]
 69 |     \s (?P<severity> [A-Z]+ )
 70 |     \s (?P<message> .* )
 71 | ''', re.VERBOSE)
 72 | 
 73 | # Compiled regular expression that matches a single line of output produced by
 74 | # the default log format with milliseconds=True.
 75 | PATTERN_INCLUDING_MILLISECONDS = re.compile(r'''
 76 |     (?P<date> \d{4}-\d{2}-\d{2} )
 77 |     \s (?P<time> \d{2}:\d{2}:\d{2},\d{3} )
 78 |     \s (?P<hostname> \S+ )
 79 |     \s (?P<logger_name> \w+ )
 80 |     \[ (?P<process_id> \d+ ) \]
 81 |     \s (?P<severity> [A-Z]+ )
 82 |     \s (?P<message> .* )
 83 | ''', re.VERBOSE)
 84 | 
 85 | 
 86 | def setUpModule():
 87 |     """Speed up the tests by disabling the demo's artificial delay."""
 88 |     os.environ['COLOREDLOGS_DEMO_DELAY'] = '0'
 89 |     coloredlogs.demo.DEMO_DELAY = 0
 90 | 
 91 | 
 92 | class ColoredLogsTestCase(TestCase):
 93 | 
 94 |     """Container for the `coloredlogs` tests."""
 95 | 
 96 |     def find_system_log(self):
 97 |         """Find the system log file or skip the current test."""
 98 |         filename = ('/var/log/system.log' if sys.platform == 'darwin' else (
 99 |             '/var/log/syslog' if 'linux' in sys.platform else None
100 |         ))
101 |         if not filename:
102 |             self.skipTest("Location of system log file unknown!")
103 |         elif not os.path.isfile(filename):
104 |             self.skipTest("System log file not found! (%s)" % filename)
105 |         elif not os.access(filename, os.R_OK):
106 |             self.skipTest("Insufficient permissions to read system log file! (%s)" % filename)
107 |         else:
108 |             return filename
109 | 
110 |     def test_level_to_number(self):
111 |         """Make sure :func:`level_to_number()` works as intended."""
112 |         # Make sure the default levels are translated as expected.
113 |         assert level_to_number('debug') == logging.DEBUG
114 |         assert level_to_number('info') == logging.INFO
115 |         assert level_to_number('warning') == logging.WARNING
116 |         assert level_to_number('error') == logging.ERROR
117 |         assert level_to_number('fatal') == logging.FATAL
118 |         # Make sure bogus level names don't blow up.
119 |         assert level_to_number('bogus-level') == logging.INFO
120 | 
121 |     def test_find_hostname(self):
122 |         """Make sure :func:`~find_hostname()` works correctly."""
123 |         assert find_hostname()
124 |         # Create a temporary file as a placeholder for e.g. /etc/debian_chroot.
125 |         fd, temporary_file = tempfile.mkstemp()
126 |         try:
127 |             with open(temporary_file, 'w') as handle:
128 |                 handle.write('first line\n')
129 |                 handle.write('second line\n')
130 |             CHROOT_FILES.insert(0, temporary_file)
131 |             # Make sure the chroot file is being read.
132 |             assert find_hostname() == 'first line'
133 |         finally:
134 |             # Clean up.
135 |             CHROOT_FILES.pop(0)
136 |             os.unlink(temporary_file)
137 |         # Test that unreadable chroot files don't break coloredlogs.
138 |         try:
139 |             CHROOT_FILES.insert(0, temporary_file)
140 |             # Make sure that a usable value is still produced.
141 |             assert find_hostname()
142 |         finally:
143 |             # Clean up.
144 |             CHROOT_FILES.pop(0)
145 | 
146 |     def test_host_name_filter(self):
147 |         """Make sure :func:`install()` integrates with :class:`~coloredlogs.HostNameFilter()`."""
148 |         install(fmt='%(hostname)s')
149 |         with CaptureOutput() as capturer:
150 |             logging.info("A truly insignificant message ..")
151 |             output = capturer.get_text()
152 |             assert find_hostname() in output
153 | 
154 |     def test_program_name_filter(self):
155 |         """Make sure :func:`install()` integrates with :class:`~coloredlogs.ProgramNameFilter()`."""
156 |         install(fmt='%(programname)s')
157 |         with CaptureOutput() as capturer:
158 |             logging.info("A truly insignificant message ..")
159 |             output = capturer.get_text()
160 |             assert find_program_name() in output
161 | 
162 |     def test_colorama_enabled(self):
163 |         """Test that colorama is enabled (through mocking)."""
164 |         init_function = MagicMock()
165 |         with mocked_colorama_module(init_function):
166 |             # Configure logging to the terminal.
167 |             coloredlogs.install()
168 |             # Ensure that our mock method was called.
169 |             assert init_function.called
170 | 
171 |     def test_colorama_missing(self):
172 |         """Test that colorama is missing (through mocking)."""
173 |         def init_function():
174 |             raise ImportError
175 |         with mocked_colorama_module(init_function):
176 |             # Configure logging to the terminal. It is expected that internally
177 |             # an ImportError is raised, but the exception is caught and colored
178 |             # output is disabled.
179 |             coloredlogs.install()
180 |             # Find the handler that was created by coloredlogs.install().
181 |             handler, logger = find_handler(logging.getLogger(), match_stream_handler)
182 |             # Make sure that logging to the terminal was initialized.
183 |             assert isinstance(handler.formatter, logging.Formatter)
184 |             # Make sure colored logging is disabled.
185 |             assert not isinstance(handler.formatter, ColoredFormatter)
186 | 
187 |     def test_system_logging(self):
188 |         """Make sure the :class:`coloredlogs.syslog.SystemLogging` context manager works."""
189 |         system_log_file = self.find_system_log()
190 |         expected_message = random_string(50)
191 |         with SystemLogging(programname='coloredlogs-test-suite') as syslog:
192 |             if not syslog:
193 |                 return self.skipTest("couldn't connect to syslog daemon")
194 |             # When I tried out the system logging support on macOS 10.13.1 on
195 |             # 2018-01-05 I found that while WARNING and ERROR messages show up
196 |             # in the system log DEBUG and INFO messages don't. This explains
197 |             # the importance of the level of the log message below.
198 |             logging.error("%s", expected_message)
199 |         # Retry the following assertion (for up to 60 seconds) to give the
200 |         # logging daemon time to write our log message to disk. This
201 |         # appears to be needed on MacOS workers on Travis CI, see:
202 |         # https://travis-ci.org/xolox/python-coloredlogs/jobs/325245853
203 |         retry(lambda: check_contents(system_log_file, expected_message, True))
204 | 
205 |     def test_syslog_shortcut_simple(self):
206 |         """Make sure that ``coloredlogs.install(syslog=True)`` works."""
207 |         system_log_file = self.find_system_log()
208 |         expected_message = random_string(50)
209 |         with cleanup_handlers():
210 |             # See test_system_logging() for the importance of this log level.
211 |             coloredlogs.install(syslog=True)
212 |             logging.error("%s", expected_message)
213 |         # See the comments in test_system_logging() on why this is retried.
214 |         retry(lambda: check_contents(system_log_file, expected_message, True))
215 | 
216 |     def test_syslog_shortcut_enhanced(self):
217 |         """Make sure that ``coloredlogs.install(syslog='warning')`` works."""
218 |         system_log_file = self.find_system_log()
219 |         the_expected_message = random_string(50)
220 |         not_an_expected_message = random_string(50)
221 |         with cleanup_handlers():
222 |             # See test_system_logging() for the importance of these log levels.
223 |             coloredlogs.install(syslog='error')
224 |             logging.warning("%s", not_an_expected_message)
225 |             logging.error("%s", the_expected_message)
226 |         # See the comments in test_system_logging() on why this is retried.
227 |         retry(lambda: check_contents(system_log_file, the_expected_message, True))
228 |         retry(lambda: check_contents(system_log_file, not_an_expected_message, False))
229 | 
230 |     def test_name_normalization(self):
231 |         """Make sure :class:`~coloredlogs.NameNormalizer` works as intended."""
232 |         nn = NameNormalizer()
233 |         for canonical_name in ['debug', 'info', 'warning', 'error', 'critical']:
234 |             assert nn.normalize_name(canonical_name) == canonical_name
235 |             assert nn.normalize_name(canonical_name.upper()) == canonical_name
236 |         assert nn.normalize_name('warn') == 'warning'
237 |         assert nn.normalize_name('fatal') == 'critical'
238 | 
239 |     def test_style_parsing(self):
240 |         """Make sure :func:`~coloredlogs.parse_encoded_styles()` works as intended."""
241 |         encoded_styles = 'debug=green;warning=yellow;error=red;critical=red,bold'
242 |         decoded_styles = parse_encoded_styles(encoded_styles, normalize_key=lambda k: k.upper())
243 |         assert sorted(decoded_styles.keys()) == sorted(['debug', 'warning', 'error', 'critical'])
244 |         assert decoded_styles['debug']['color'] == 'green'
245 |         assert decoded_styles['warning']['color'] == 'yellow'
246 |         assert decoded_styles['error']['color'] == 'red'
247 |         assert decoded_styles['critical']['color'] == 'red'
248 |         assert decoded_styles['critical']['bold'] is True
249 | 
250 |     def test_is_verbose(self):
251 |         """Make sure is_verbose() does what it should :-)."""
252 |         set_level(logging.INFO)
253 |         assert not is_verbose()
254 |         set_level(logging.DEBUG)
255 |         assert is_verbose()
256 |         set_level(logging.VERBOSE)
257 |         assert is_verbose()
258 | 
259 |     def test_increase_verbosity(self):
260 |         """Make sure increase_verbosity() respects default and custom levels."""
261 |         # Start from a known state.
262 |         set_level(logging.INFO)
263 |         assert get_level() == logging.INFO
264 |         # INFO -> VERBOSE.
265 |         increase_verbosity()
266 |         assert get_level() == logging.VERBOSE
267 |         # VERBOSE -> DEBUG.
268 |         increase_verbosity()
269 |         assert get_level() == logging.DEBUG
270 |         # DEBUG -> SPAM.
271 |         increase_verbosity()
272 |         assert get_level() == logging.SPAM
273 |         # SPAM -> NOTSET.
274 |         increase_verbosity()
275 |         assert get_level() == logging.NOTSET
276 |         # NOTSET -> NOTSET.
277 |         increase_verbosity()
278 |         assert get_level() == logging.NOTSET
279 | 
280 |     def test_decrease_verbosity(self):
281 |         """Make sure decrease_verbosity() respects default and custom levels."""
282 |         # Start from a known state.
283 |         set_level(logging.INFO)
284 |         assert get_level() == logging.INFO
285 |         # INFO -> NOTICE.
286 |         decrease_verbosity()
287 |         assert get_level() == logging.NOTICE
288 |         # NOTICE -> WARNING.
289 |         decrease_verbosity()
290 |         assert get_level() == logging.WARNING
291 |         # WARNING -> SUCCESS.
292 |         decrease_verbosity()
293 |         assert get_level() == logging.SUCCESS
294 |         # SUCCESS -> ERROR.
295 |         decrease_verbosity()
296 |         assert get_level() == logging.ERROR
297 |         # ERROR -> CRITICAL.
298 |         decrease_verbosity()
299 |         assert get_level() == logging.CRITICAL
300 |         # CRITICAL -> CRITICAL.
301 |         decrease_verbosity()
302 |         assert get_level() == logging.CRITICAL
303 | 
304 |     def test_level_discovery(self):
305 |         """Make sure find_defined_levels() always reports the levels defined in Python's standard library."""
306 |         defined_levels = find_defined_levels()
307 |         level_values = defined_levels.values()
308 |         for number in (0, 10, 20, 30, 40, 50):
309 |             assert number in level_values
310 | 
311 |     def test_walk_propagation_tree(self):
312 |         """Make sure walk_propagation_tree() properly walks the tree of loggers."""
313 |         root, parent, child, grand_child = self.get_logger_tree()
314 |         # Check the default mode of operation.
315 |         loggers = list(walk_propagation_tree(grand_child))
316 |         assert loggers == [grand_child, child, parent, root]
317 |         # Now change the propagation (non-default mode of operation).
318 |         child.propagate = False
319 |         loggers = list(walk_propagation_tree(grand_child))
320 |         assert loggers == [grand_child, child]
321 | 
322 |     def test_find_handler(self):
323 |         """Make sure find_handler() works as intended."""
324 |         root, parent, child, grand_child = self.get_logger_tree()
325 |         # Add some handlers to the tree.
326 |         stream_handler = logging.StreamHandler()
327 |         syslog_handler = logging.handlers.SysLogHandler()
328 |         child.addHandler(stream_handler)
329 |         parent.addHandler(syslog_handler)
330 |         # Make sure the first matching handler is returned.
331 |         matched_handler, matched_logger = find_handler(grand_child, lambda h: isinstance(h, logging.Handler))
332 |         assert matched_handler is stream_handler
333 |         # Make sure the first matching handler of the given type is returned.
334 |         matched_handler, matched_logger = find_handler(child, lambda h: isinstance(h, logging.handlers.SysLogHandler))
335 |         assert matched_handler is syslog_handler
336 | 
337 |     def get_logger_tree(self):
338 |         """Create and return a tree of loggers."""
339 |         # Get the root logger.
340 |         root = logging.getLogger()
341 |         # Create a top level logger for ourselves.
342 |         parent_name = random_string()
343 |         parent = logging.getLogger(parent_name)
344 |         # Create a child logger.
345 |         child_name = '%s.%s' % (parent_name, random_string())
346 |         child = logging.getLogger(child_name)
347 |         # Create a grand child logger.
348 |         grand_child_name = '%s.%s' % (child_name, random_string())
349 |         grand_child = logging.getLogger(grand_child_name)
350 |         return root, parent, child, grand_child
351 | 
352 |     def test_support_for_milliseconds(self):
353 |         """Make sure milliseconds are hidden by default but can be easily enabled."""
354 |         # Check that the default log format doesn't include milliseconds.
355 |         stream = StringIO()
356 |         install(reconfigure=True, stream=stream)
357 |         logging.info("This should not include milliseconds.")
358 |         assert all(map(PLAIN_TEXT_PATTERN.match, stream.getvalue().splitlines()))
359 |         # Check that milliseconds can be enabled via a shortcut.
360 |         stream = StringIO()
361 |         install(milliseconds=True, reconfigure=True, stream=stream)
362 |         logging.info("This should include milliseconds.")
363 |         assert all(map(PATTERN_INCLUDING_MILLISECONDS.match, stream.getvalue().splitlines()))
364 | 
365 |     def test_support_for_milliseconds_directive(self):
366 |         """Make sure milliseconds using the ``%f`` directive are supported."""
367 |         stream = StringIO()
368 |         install(reconfigure=True, stream=stream, datefmt='%Y-%m-%dT%H:%M:%S.%f%z')
369 |         logging.info("This should be timestamped according to #45.")
370 |         assert re.match(r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+\d{4}\s', stream.getvalue())
371 | 
372 |     def test_plain_text_output_format(self):
373 |         """Inspect the plain text output of coloredlogs."""
374 |         logger = VerboseLogger(random_string(25))
375 |         stream = StringIO()
376 |         install(level=logging.NOTSET, logger=logger, stream=stream)
377 |         # Test that filtering on severity works.
378 |         logger.setLevel(logging.INFO)
379 |         logger.debug("No one should see this message.")
380 |         assert len(stream.getvalue().strip()) == 0
381 |         # Test that the default output format looks okay in plain text.
382 |         logger.setLevel(logging.NOTSET)
383 |         for method, severity in ((logger.debug, 'DEBUG'),
384 |                                  (logger.info, 'INFO'),
385 |                                  (logger.verbose, 'VERBOSE'),
386 |                                  (logger.warning, 'WARNING'),
387 |                                  (logger.error, 'ERROR'),
388 |                                  (logger.critical, 'CRITICAL')):
389 |             # Prepare the text.
390 |             text = "This is a message with severity %r." % severity.lower()
391 |             # Log the message with the given severity.
392 |             method(text)
393 |             # Get the line of output generated by the handler.
394 |             output = stream.getvalue()
395 |             lines = output.splitlines()
396 |             last_line = lines[-1]
397 |             assert text in last_line
398 |             assert severity in last_line
399 |             assert PLAIN_TEXT_PATTERN.match(last_line)
400 | 
401 |     def test_html_conversion(self):
402 |         """Check the conversion from ANSI escape sequences to HTML."""
403 |         # Check conversion of colored text.
404 |         for color_name, ansi_code in ANSI_COLOR_CODES.items():
405 |             ansi_encoded_text = 'plain text followed by %s text' % ansi_wrap(color_name, color=color_name)
406 |             expected_html = format(
407 |                 '<code>plain text followed by <span style="color:{css}">{name}</span> text</code>',
408 |                 css=EIGHT_COLOR_PALETTE[ansi_code], name=color_name,
409 |             )
410 |             self.assertEquals(expected_html, convert(ansi_encoded_text))
411 |         # Check conversion of bright colored text.
412 |         expected_html = '<code><span style="color:#FF0">bright yellow</span></code>'
413 |         self.assertEquals(expected_html, convert(ansi_wrap('bright yellow', color='yellow', bright=True)))
414 |         # Check conversion of text with a background color.
415 |         expected_html = '<code><span style="background-color:#DE382B">red background</span></code>'
416 |         self.assertEquals(expected_html, convert(ansi_wrap('red background', background='red')))
417 |         # Check conversion of text with a bright background color.
418 |         expected_html = '<code><span style="background-color:#F00">bright red background</span></code>'
419 |         self.assertEquals(expected_html, convert(ansi_wrap('bright red background', background='red', bright=True)))
420 |         # Check conversion of text that uses the 256 color mode palette as a foreground color.
421 |         expected_html = '<code><span style="color:#FFAF00">256 color mode foreground</span></code>'
422 |         self.assertEquals(expected_html, convert(ansi_wrap('256 color mode foreground', color=214)))
423 |         # Check conversion of text that uses the 256 color mode palette as a background color.
424 |         expected_html = '<code><span style="background-color:#AF0000">256 color mode background</span></code>'
425 |         self.assertEquals(expected_html, convert(ansi_wrap('256 color mode background', background=124)))
426 |         # Check that invalid 256 color mode indexes don't raise exceptions.
427 |         expected_html = '<code>plain text expected</code>'
428 |         self.assertEquals(expected_html, convert('\x1b[38;5;256mplain text expected\x1b[0m'))
429 |         # Check conversion of bold text.
430 |         expected_html = '<code><span style="font-weight:bold">bold text</span></code>'
431 |         self.assertEquals(expected_html, convert(ansi_wrap('bold text', bold=True)))
432 |         # Check conversion of underlined text.
433 |         expected_html = '<code><span style="text-decoration:underline">underlined text</span></code>'
434 |         self.assertEquals(expected_html, convert(ansi_wrap('underlined text', underline=True)))
435 |         # Check conversion of strike-through text.
436 |         expected_html = '<code><span style="text-decoration:line-through">strike-through text</span></code>'
437 |         self.assertEquals(expected_html, convert(ansi_wrap('strike-through text', strike_through=True)))
438 |         # Check conversion of inverse text.
439 |         expected_html = '<code><span style="background-color:#FFC706;color:#000">inverse</span></code>'
440 |         self.assertEquals(expected_html, convert(ansi_wrap('inverse', color='yellow', inverse=True)))
441 |         # Check conversion of URLs.
442 |         for sample_text in 'www.python.org', 'http://coloredlogs.rtfd.org', 'https://coloredlogs.rtfd.org':
443 |             sample_url = sample_text if '://' in sample_text else ('http://' + sample_text)
444 |             expected_html = '<code><a href="%s" style="color:inherit">%s</a></code>' % (sample_url, sample_text)
445 |             self.assertEquals(expected_html, convert(sample_text))
446 |         # Check that the capture pattern for URLs doesn't match ANSI escape
447 |         # sequences and also check that the short hand for the 0 reset code is
448 |         # supported. These are tests for regressions of bugs found in
449 |         # coloredlogs <= 8.0.
450 |         reset_short_hand = '\x1b[0m'
451 |         blue_underlined = ansi_style(color='blue', underline=True)
452 |         ansi_encoded_text = '<%shttps://coloredlogs.readthedocs.io%s>' % (blue_underlined, reset_short_hand)
453 |         expected_html = (
454 |             '<code>&lt;<span style="color:#006FB8;text-decoration:underline">'
455 |             '<a href="https://coloredlogs.readthedocs.io" style="color:inherit">'
456 |             'https://coloredlogs.readthedocs.io'
457 |             '</a></span>&gt;</code>'
458 |         )
459 |         self.assertEquals(expected_html, convert(ansi_encoded_text))
460 | 
461 |     def test_output_interception(self):
462 |         """Test capturing of output from external commands."""
463 |         expected_output = 'testing, 1, 2, 3 ..'
464 |         actual_output = capture(['echo', expected_output])
465 |         assert actual_output.strip() == expected_output.strip()
466 | 
467 |     def test_enable_colored_cron_mailer(self):
468 |         """Test that automatic ANSI to HTML conversion when running under ``cron`` can be enabled."""
469 |         with PatchedItem(os.environ, 'CONTENT_TYPE', 'text/html'):
470 |             with ColoredCronMailer() as mailer:
471 |                 assert mailer.is_enabled
472 | 
473 |     def test_disable_colored_cron_mailer(self):
474 |         """Test that automatic ANSI to HTML conversion when running under ``cron`` can be disabled."""
475 |         with PatchedItem(os.environ, 'CONTENT_TYPE', 'text/plain'):
476 |             with ColoredCronMailer() as mailer:
477 |                 assert not mailer.is_enabled
478 | 
479 |     def test_auto_install(self):
480 |         """Test :func:`coloredlogs.auto_install()`."""
481 |         needle = random_string()
482 |         command_line = [sys.executable, '-c', 'import logging; logging.info(%r)' % needle]
483 |         # Sanity check that log messages aren't enabled by default.
484 |         with CaptureOutput() as capturer:
485 |             os.environ['COLOREDLOGS_AUTO_INSTALL'] = 'false'
486 |             subprocess.call(command_line)
487 |             output = capturer.get_text()
488 |         assert needle not in output
489 |         # Test that the $COLOREDLOGS_AUTO_INSTALL environment variable can be
490 |         # used to automatically call coloredlogs.install() during initialization.
491 |         with CaptureOutput() as capturer:
492 |             os.environ['COLOREDLOGS_AUTO_INSTALL'] = 'true'
493 |             subprocess.call(command_line)
494 |             output = capturer.get_text()
495 |         assert needle in output
496 | 
497 |     def test_cli_demo(self):
498 |         """Test the command line colored logging demonstration."""
499 |         with CaptureOutput() as capturer:
500 |             main('coloredlogs', '--demo')
501 |             output = capturer.get_text()
502 |         # Make sure the output contains all of the expected logging level names.
503 |         for name in 'debug', 'info', 'warning', 'error', 'critical':
504 |             assert name.upper() in output
505 | 
506 |     def test_cli_conversion(self):
507 |         """Test the command line HTML conversion."""
508 |         output = main('coloredlogs', '--convert', 'coloredlogs', '--demo', capture=True)
509 |         # Make sure the output is encoded as HTML.
510 |         assert '<span' in output
511 | 
512 |     def test_empty_conversion(self):
513 |         """
514 |         Test that conversion of empty output produces no HTML.
515 | 
516 |         This test was added because I found that ``coloredlogs --convert`` when
517 |         used in a cron job could cause cron to send out what appeared to be
518 |         empty emails. On more careful inspection the body of those emails was
519 |         ``<code></code>``. By not emitting the wrapper element when no other
520 |         HTML is generated, cron will not send out an email.
521 |         """
522 |         output = main('coloredlogs', '--convert', 'true', capture=True)
523 |         assert not output.strip()
524 | 
525 |     def test_implicit_usage_message(self):
526 |         """Test that the usage message is shown when no actions are given."""
527 |         assert 'Usage:' in main('coloredlogs', capture=True)
528 | 
529 |     def test_explicit_usage_message(self):
530 |         """Test that the usage message is shown when ``--help`` is given."""
531 |         assert 'Usage:' in main('coloredlogs', '--help', capture=True)
532 | 
533 | 
534 | def check_contents(filename, contents, match):
535 |     """Check if a line in a file contains an expected string."""
536 |     with open(filename) as handle:
537 |         assert any(contents in line for line in handle) == match
538 | 
539 | 
540 | def main(*arguments, **options):
541 |     """Wrap the command line interface to make it easier to test."""
542 |     capture = options.get('capture', False)
543 |     saved_argv = sys.argv
544 |     saved_stdout = sys.stdout
545 |     try:
546 |         sys.argv = arguments
547 |         if capture:
548 |             sys.stdout = StringIO()
549 |         coloredlogs.cli.main()
550 |         if capture:
551 |             return sys.stdout.getvalue()
552 |     finally:
553 |         sys.argv = saved_argv
554 |         sys.stdout = saved_stdout
555 | 
556 | 
557 | @contextlib.contextmanager
558 | def mocked_colorama_module(init_function):
559 |     """Context manager to ease testing of colorama integration."""
560 |     module_name = 'colorama'
561 |     # Create a fake module shadowing colorama.
562 |     fake_module = imp.new_module(module_name)
563 |     setattr(fake_module, 'init', init_function)
564 |     # Temporarily reconfigure coloredlogs to use colorama.
565 |     need_colorama = coloredlogs.NEED_COLORAMA
566 |     coloredlogs.NEED_COLORAMA = True
567 |     # Install the fake colorama module.
568 |     saved_module = sys.modules.get(module_name, None)
569 |     sys.modules[module_name] = fake_module
570 |     # We've finished setting up, yield control.
571 |     yield
572 |     # Restore the original setting.
573 |     coloredlogs.NEED_COLORAMA = need_colorama
574 |     # Clean up the mock module.
575 |     if saved_module is not None:
576 |         sys.modules[module_name] = saved_module
577 |     else:
578 |         sys.modules.pop(module_name, None)
579 | 
580 | 
581 | @contextlib.contextmanager
582 | def cleanup_handlers():
583 |     """Context manager to cleanup output handlers."""
584 |     # There's nothing to set up so we immediately yield control.
585 |     yield
586 |     # After the with block ends we cleanup any output handlers.
587 |     for match_func in match_stream_handler, match_syslog_handler:
588 |         handler, logger = find_handler(logging.getLogger(), match_func)
589 |         if handler and logger:
590 |             logger.removeHandler(handler)
591 | 


--------------------------------------------------------------------------------
/data/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/crocs-muni/cm_factorization/bef85394257dc39b451c9eec9c67019ed3224e1a/data/.gitkeep


--------------------------------------------------------------------------------
/data/semifactor_15.json:
--------------------------------------------------------------------------------
1 | version https://git-lfs.github.com/spec/v1
2 | oid sha256:5c5933d50e15df85c89d3c48b441c771124c8fb0d1d94c784c51f502d322485b
3 | size 81541206
4 | 


--------------------------------------------------------------------------------
/data/semifactor_17.json:
--------------------------------------------------------------------------------
1 | version https://git-lfs.github.com/spec/v1
2 | oid sha256:47eeddb29a5de2dd7ec4b13669a6999603adba282f16452fd5d959bfa90c3751
3 | size 44449792
4 | 


--------------------------------------------------------------------------------
/humanfriendly/cli.py:
--------------------------------------------------------------------------------
  1 | # Human friendly input/output in Python.
  2 | #
  3 | # Author: Peter Odding <peter@peterodding.com>
  4 | # Last Change: January 14, 2018
  5 | # URL: https://humanfriendly.readthedocs.io
  6 | 
  7 | """
  8 | Usage: humanfriendly [OPTIONS]
  9 | 
 10 | Human friendly input/output (text formatting) on the command
 11 | line based on the Python package with the same name.
 12 | 
 13 | Supported options:
 14 | 
 15 |   -c, --run-command
 16 | 
 17 |     Execute an external command (given as the positional arguments) and render
 18 |     a spinner and timer while the command is running. The exit status of the
 19 |     command is propagated.
 20 | 
 21 |   --format-table
 22 | 
 23 |     Read tabular data from standard input (each line is a row and each
 24 |     whitespace separated field is a column), format the data as a table and
 25 |     print the resulting table to standard output. See also the --delimiter
 26 |     option.
 27 | 
 28 |   -d, --delimiter=VALUE
 29 | 
 30 |     Change the delimiter used by --format-table to VALUE (a string). By default
 31 |     all whitespace is treated as a delimiter.
 32 | 
 33 |   -l, --format-length=LENGTH
 34 | 
 35 |     Convert a length count (given as the integer or float LENGTH) into a human
 36 |     readable string and print that string to standard output.
 37 | 
 38 |   -n, --format-number=VALUE
 39 | 
 40 |     Format a number (given as the integer or floating point number VALUE) with
 41 |     thousands separators and two decimal places (if needed) and print the
 42 |     formatted number to standard output.
 43 | 
 44 |   -s, --format-size=BYTES
 45 | 
 46 |     Convert a byte count (given as the integer BYTES) into a human readable
 47 |     string and print that string to standard output.
 48 | 
 49 |   -b, --binary
 50 | 
 51 |     Change the output of -s, --format-size to use binary multiples of bytes
 52 |     (base-2) instead of the default decimal multiples of bytes (base-10).
 53 | 
 54 |   -t, --format-timespan=SECONDS
 55 | 
 56 |     Convert a number of seconds (given as the floating point number SECONDS)
 57 |     into a human readable timespan and print that string to standard output.
 58 | 
 59 |   --parse-length=VALUE
 60 | 
 61 |     Parse a human readable length (given as the string VALUE) and print the
 62 |     number of metres to standard output.
 63 | 
 64 |   --parse-size=VALUE
 65 | 
 66 |     Parse a human readable data size (given as the string VALUE) and print the
 67 |     number of bytes to standard output.
 68 | 
 69 |   --demo
 70 | 
 71 |     Demonstrate changing the style and color of the terminal font using ANSI
 72 |     escape sequences.
 73 | 
 74 |   -h, --help
 75 | 
 76 |     Show this message and exit.
 77 | """
 78 | 
 79 | # Standard library modules.
 80 | import functools
 81 | import getopt
 82 | import pipes
 83 | import subprocess
 84 | import sys
 85 | 
 86 | # Modules included in our package.
 87 | from humanfriendly import (
 88 |     Spinner,
 89 |     Timer,
 90 |     format_length,
 91 |     format_number,
 92 |     format_size,
 93 |     format_table,
 94 |     format_timespan,
 95 |     parse_length,
 96 |     parse_size,
 97 | )
 98 | from humanfriendly.tables import format_smart_table
 99 | from humanfriendly.terminal import (
100 |     ANSI_COLOR_CODES,
101 |     ANSI_TEXT_STYLES,
102 |     HIGHLIGHT_COLOR,
103 |     ansi_strip,
104 |     ansi_wrap,
105 |     find_terminal_size,
106 |     output,
107 |     usage,
108 |     warning,
109 | )
110 | 
111 | 
112 | def main():
113 |     """Command line interface for the ``humanfriendly`` program."""
114 |     try:
115 |         options, arguments = getopt.getopt(sys.argv[1:], 'cd:l:n:s:bt:h', [
116 |             'run-command', 'format-table', 'delimiter=', 'format-length=',
117 |             'format-number=', 'format-size=', 'binary', 'format-timespan=',
118 |             'parse-length=', 'parse-size=', 'demo', 'help',
119 |         ])
120 |     except Exception as e:
121 |         warning("Error: %s", e)
122 |         sys.exit(1)
123 |     actions = []
124 |     delimiter = None
125 |     should_format_table = False
126 |     binary = any(o in ('-b', '--binary') for o, v in options)
127 |     for option, value in options:
128 |         if option in ('-d', '--delimiter'):
129 |             delimiter = value
130 |         elif option == '--parse-size':
131 |             actions.append(functools.partial(print_parsed_size, value))
132 |         elif option == '--parse-length':
133 |             actions.append(functools.partial(print_parsed_length, value))
134 |         elif option in ('-c', '--run-command'):
135 |             actions.append(functools.partial(run_command, arguments))
136 |         elif option in ('-l', '--format-length'):
137 |             actions.append(functools.partial(print_formatted_length, value))
138 |         elif option in ('-n', '--format-number'):
139 |             actions.append(functools.partial(print_formatted_number, value))
140 |         elif option in ('-s', '--format-size'):
141 |             actions.append(functools.partial(print_formatted_size, value, binary))
142 |         elif option == '--format-table':
143 |             should_format_table = True
144 |         elif option in ('-t', '--format-timespan'):
145 |             actions.append(functools.partial(print_formatted_timespan, value))
146 |         elif option == '--demo':
147 |             actions.append(demonstrate_ansi_formatting)
148 |         elif option in ('-h', '--help'):
149 |             usage(__doc__)
150 |             return
151 |     if should_format_table:
152 |         actions.append(functools.partial(print_formatted_table, delimiter))
153 |     if not actions:
154 |         usage(__doc__)
155 |         return
156 |     for partial in actions:
157 |         partial()
158 | 
159 | 
160 | def run_command(command_line):
161 |     """Run an external command and show a spinner while the command is running."""
162 |     timer = Timer()
163 |     spinner_label = "Waiting for command: %s" % " ".join(map(pipes.quote, command_line))
164 |     with Spinner(label=spinner_label, timer=timer) as spinner:
165 |         process = subprocess.Popen(command_line)
166 |         while True:
167 |             spinner.step()
168 |             spinner.sleep()
169 |             if process.poll() is not None:
170 |                 break
171 |     sys.exit(process.returncode)
172 | 
173 | 
174 | def print_formatted_length(value):
175 |     """Print a human readable length."""
176 |     if '.' in value:
177 |         output(format_length(float(value)))
178 |     else:
179 |         output(format_length(int(value)))
180 | 
181 | 
182 | def print_formatted_number(value):
183 |     """Print large numbers in a human readable format."""
184 |     output(format_number(float(value)))
185 | 
186 | 
187 | def print_formatted_size(value, binary):
188 |     """Print a human readable size."""
189 |     output(format_size(int(value), binary=binary))
190 | 
191 | 
192 | def print_formatted_table(delimiter):
193 |     """Read tabular data from standard input and print a table."""
194 |     data = []
195 |     for line in sys.stdin:
196 |         line = line.rstrip()
197 |         data.append(line.split(delimiter))
198 |     output(format_table(data))
199 | 
200 | 
201 | def print_formatted_timespan(value):
202 |     """Print a human readable timespan."""
203 |     output(format_timespan(float(value)))
204 | 
205 | 
206 | def print_parsed_length(value):
207 |     """Parse a human readable length and print the number of metres."""
208 |     output(parse_length(value))
209 | 
210 | 
211 | def print_parsed_size(value):
212 |     """Parse a human readable data size and print the number of bytes."""
213 |     output(parse_size(value))
214 | 
215 | 
216 | def demonstrate_ansi_formatting():
217 |     """Demonstrate the use of ANSI escape sequences."""
218 |     # First we demonstrate the supported text styles.
219 |     output('%s', ansi_wrap('Text styles:', bold=True))
220 |     styles = ['normal', 'bright']
221 |     styles.extend(ANSI_TEXT_STYLES.keys())
222 |     for style_name in sorted(styles):
223 |         options = dict(color=HIGHLIGHT_COLOR)
224 |         if style_name != 'normal':
225 |             options[style_name] = True
226 |         style_label = style_name.replace('_', ' ').capitalize()
227 |         output(' - %s', ansi_wrap(style_label, **options))
228 |     # Now we demonstrate named foreground and background colors.
229 |     for color_type, color_label in (('color', 'Foreground colors'),
230 |                                     ('background', 'Background colors')):
231 |         intensities = [
232 |             ('normal', dict()),
233 |             ('bright', dict(bright=True)),
234 |         ]
235 |         if color_type != 'background':
236 |             intensities.insert(0, ('faint', dict(faint=True)))
237 |         output('\n%s' % ansi_wrap('%s:' % color_label, bold=True))
238 |         output(format_smart_table([
239 |             [color_name] + [
240 |                 ansi_wrap(
241 |                     'XXXXXX' if color_type != 'background' else (' ' * 6),
242 |                     **dict(list(kw.items()) + [(color_type, color_name)])
243 |                 ) for label, kw in intensities
244 |             ] for color_name in sorted(ANSI_COLOR_CODES.keys())
245 |         ], column_names=['Color'] + [
246 |             label.capitalize() for label, kw in intensities
247 |         ]))
248 |     # Demonstrate support for 256 colors as well.
249 |     demonstrate_256_colors(0, 7, 'standard colors')
250 |     demonstrate_256_colors(8, 15, 'high-intensity colors')
251 |     demonstrate_256_colors(16, 231, '216 colors')
252 |     demonstrate_256_colors(232, 255, 'gray scale colors')
253 | 
254 | 
255 | def demonstrate_256_colors(i, j, group=None):
256 |     """Demonstrate 256 color mode support."""
257 |     # Generate the label.
258 |     label = '256 color mode'
259 |     if group:
260 |         label += ' (%s)' % group
261 |     output('\n' + ansi_wrap('%s:' % label, bold=True))
262 |     # Generate a simple rendering of the colors in the requested range and
263 |     # check if it will fit on a single line (given the terminal's width).
264 |     single_line = ''.join(' ' + ansi_wrap(str(n), color=n) for n in range(i, j + 1))
265 |     lines, columns = find_terminal_size()
266 |     if columns >= len(ansi_strip(single_line)):
267 |         output(single_line)
268 |     else:
269 |         # Generate a more complex rendering of the colors that will nicely wrap
270 |         # over multiple lines without using too many lines.
271 |         width = len(str(j)) + 1
272 |         colors_per_line = int(columns / width)
273 |         colors = [ansi_wrap(str(n).rjust(width), color=n) for n in range(i, j + 1)]
274 |         blocks = [colors[n:n + colors_per_line] for n in range(0, len(colors), colors_per_line)]
275 |         output('\n'.join(''.join(b) for b in blocks))
276 | 


--------------------------------------------------------------------------------
/humanfriendly/compat.py:
--------------------------------------------------------------------------------
  1 | # Human friendly input/output in Python.
  2 | #
  3 | # Author: Peter Odding <peter@peterodding.com>
  4 | # Last Change: July 14, 2018
  5 | # URL: https://humanfriendly.readthedocs.io
  6 | 
  7 | """
  8 | Compatibility with Python 2 and 3.
  9 | 
 10 | This module exposes aliases and functions that make it easier to write Python
 11 | code that is compatible with Python 2 and Python 3.
 12 | 
 13 | .. data:: basestring
 14 | 
 15 |    Alias for :func:`python2:basestring` (in Python 2) or :class:`python3:str`
 16 |    (in Python 3). See also :func:`is_string()`.
 17 | 
 18 | .. data:: HTMLParser
 19 | 
 20 |    Alias for :func:`python2:HTMLParser.HTMLParser` (in Python 2) or
 21 |    :func:`python3:html.parser.HTMLParser` (in Python 3).
 22 | 
 23 | .. data:: interactive_prompt
 24 | 
 25 |    Alias for :func:`python2:raw_input()` (in Python 2) or
 26 |    :func:`python3:input()` (in Python 3).
 27 | 
 28 | .. data:: StringIO
 29 | 
 30 |    Alias for :class:`python2:StringIO.StringIO` (in Python 2) or
 31 |    :class:`python3:io.StringIO` (in Python 3).
 32 | 
 33 | .. data:: unicode
 34 | 
 35 |    Alias for :func:`python2:unicode` (in Python 2) or :class:`python3:str` (in
 36 |    Python 3). See also :func:`coerce_string()`.
 37 | 
 38 | .. data:: monotonic
 39 | 
 40 |    Alias for :func:`python3:time.monotonic()` (in Python 3.3 and higher) or
 41 |    `monotonic.monotonic()` (a `conditional dependency
 42 |    <https://pypi.python.org/pypi/monotonic/>`_ on older Python versions).
 43 | """
 44 | 
 45 | __all__ = (
 46 |     'HTMLParser',
 47 |     'StringIO',
 48 |     'basestring',
 49 |     'coerce_string',
 50 |     'interactive_prompt',
 51 |     'is_string',
 52 |     'is_unicode',
 53 |     'monotonic',
 54 |     'name2codepoint',
 55 |     'unichr',
 56 |     'unicode',
 57 |     'unittest',
 58 | )
 59 | 
 60 | try:
 61 |     # Python 2.
 62 |     unicode = unicode
 63 |     unichr = unichr
 64 |     basestring = basestring
 65 |     interactive_prompt = raw_input
 66 |     from HTMLParser import HTMLParser
 67 |     from StringIO import StringIO
 68 |     from htmlentitydefs import name2codepoint
 69 | except (ImportError, NameError):
 70 |     # Python 3.
 71 |     unicode = str
 72 |     unichr = chr
 73 |     basestring = str
 74 |     interactive_prompt = input
 75 |     from html.parser import HTMLParser
 76 |     from io import StringIO
 77 |     from html.entities import name2codepoint
 78 | 
 79 | try:
 80 |     # Python 3.3 and higher.
 81 |     from time import monotonic
 82 | except ImportError:
 83 |     # A replacement for older Python versions:
 84 |     # https://pypi.python.org/pypi/monotonic/
 85 |     try:
 86 |         from monotonic import monotonic
 87 |     except (ImportError, RuntimeError):
 88 |         # We fall back to the old behavior of using time.time() instead of
 89 |         # failing when {time,monotonic}.monotonic() are both missing.
 90 |         from time import time as monotonic
 91 | 
 92 | try:
 93 |     # A replacement for Python 2.6:
 94 |     # https://pypi.python.org/pypi/unittest2/
 95 |     import unittest2 as unittest
 96 | except ImportError:
 97 |     # The standard library module (on other Python versions).
 98 |     import unittest
 99 | 
100 | 
101 | def coerce_string(value):
102 |     """
103 |     Coerce any value to a Unicode string (:func:`python2:unicode` in Python 2 and :class:`python3:str` in Python 3).
104 | 
105 |     :param value: The value to coerce.
106 |     :returns: The value coerced to a Unicode string.
107 |     """
108 |     return value if is_string(value) else unicode(value)
109 | 
110 | 
111 | def is_string(value):
112 |     """
113 |     Check if a value is a :func:`python2:basestring` (in Python 2) or :class:`python2:str` (in Python 3) object.
114 | 
115 |     :param value: The value to check.
116 |     :returns: :data:`True` if the value is a string, :data:`False` otherwise.
117 |     """
118 |     return isinstance(value, basestring)
119 | 
120 | 
121 | def is_unicode(value):
122 |     """
123 |     Check if a value is a :func:`python2:unicode` (in Python 2) or :class:`python2:str` (in Python 3) object.
124 | 
125 |     :param value: The value to check.
126 |     :returns: :data:`True` if the value is a Unicode string, :data:`False` otherwise.
127 |     """
128 |     return isinstance(value, unicode)
129 | 


--------------------------------------------------------------------------------
/humanfriendly/decorators.py:
--------------------------------------------------------------------------------
 1 | # Human friendly input/output in Python.
 2 | #
 3 | # Author: Peter Odding <peter@peterodding.com>
 4 | # Last Change: October 20, 2018
 5 | # URL: https://humanfriendly.readthedocs.io
 6 | 
 7 | """Simple function decorators to make Python programming easier."""
 8 | 
 9 | # Public identifiers that require documentation.
10 | __all__ = ('RESULTS_ATTRIBUTE', 'cached')
11 | 
12 | RESULTS_ATTRIBUTE = 'cached_results'
13 | """The name of the property used to cache the return values of functions (a string)."""
14 | 
15 | 
16 | def cached(function):
17 |     """
18 |     Rudimentary caching decorator for functions.
19 | 
20 |     :param function: The function whose return value should be cached.
21 |     :returns: The decorated function.
22 | 
23 |     The given function will only be called once, the first time the wrapper
24 |     function is called. The return value is cached by the wrapper function as
25 |     an attribute of the given function and returned on each subsequent call.
26 | 
27 |     .. note:: Currently no function arguments are supported because only a
28 |               single return value can be cached. Accepting any function
29 |               arguments at all would imply that the cache is parametrized on
30 |               function arguments, which is not currently the case.
31 |     """
32 |     def wrapper():
33 |         try:
34 |             return getattr(wrapper, RESULTS_ATTRIBUTE)
35 |         except AttributeError:
36 |             result = function()
37 |             setattr(wrapper, RESULTS_ATTRIBUTE, result)
38 |             return result
39 |     wrapper.__doc__ = function.__doc__
40 |     return wrapper
41 | 


--------------------------------------------------------------------------------
/humanfriendly/prompts.py:
--------------------------------------------------------------------------------
  1 | # vim: fileencoding=utf-8
  2 | 
  3 | # Human friendly input/output in Python.
  4 | #
  5 | # Author: Peter Odding <peter@peterodding.com>
  6 | # Last Change: June 24, 2017
  7 | # URL: https://humanfriendly.readthedocs.io
  8 | 
  9 | """
 10 | Interactive terminal prompts.
 11 | 
 12 | The :mod:`~humanfriendly.prompts` module enables interaction with the user
 13 | (operator) by asking for confirmation (:func:`prompt_for_confirmation()`) and
 14 | asking to choose from a list of options (:func:`prompt_for_choice()`). It works
 15 | by rendering interactive prompts on the terminal.
 16 | """
 17 | 
 18 | # Standard library modules.
 19 | import logging
 20 | import sys
 21 | 
 22 | # Modules included in our package.
 23 | from humanfriendly.compat import interactive_prompt
 24 | from humanfriendly.terminal import (
 25 |     HIGHLIGHT_COLOR,
 26 |     ansi_strip,
 27 |     ansi_wrap,
 28 |     connected_to_terminal,
 29 |     terminal_supports_colors,
 30 |     warning,
 31 | )
 32 | from humanfriendly.text import format, concatenate
 33 | 
 34 | MAX_ATTEMPTS = 10
 35 | """The number of times an interactive prompt is shown on invalid input (an integer)."""
 36 | 
 37 | # Initialize a logger for this module.
 38 | logger = logging.getLogger(__name__)
 39 | 
 40 | 
 41 | def prompt_for_confirmation(question, default=None, padding=True):
 42 |     """
 43 |     Prompt the user for confirmation.
 44 | 
 45 |     :param question: The text that explains what the user is confirming (a string).
 46 |     :param default: The default value (a boolean) or :data:`None`.
 47 |     :param padding: Refer to the documentation of :func:`prompt_for_input()`.
 48 |     :returns: - If the user enters 'yes' or 'y' then :data:`True` is returned.
 49 |               - If the user enters 'no' or 'n' then :data:`False`  is returned.
 50 |               - If the user doesn't enter any text or standard input is not
 51 |                 connected to a terminal (which makes it impossible to prompt
 52 |                 the user) the value of the keyword argument ``default`` is
 53 |                 returned (if that value is not :data:`None`).
 54 |     :raises: - Any exceptions raised by :func:`retry_limit()`.
 55 |              - Any exceptions raised by :func:`prompt_for_input()`.
 56 | 
 57 |     When `default` is :data:`False` and the user doesn't enter any text an
 58 |     error message is printed and the prompt is repeated:
 59 | 
 60 |     >>> prompt_for_confirmation("Are you sure?")
 61 |      <BLANKLINE>
 62 |      Are you sure? [y/n]
 63 |      <BLANKLINE>
 64 |      Error: Please enter 'yes' or 'no' (there's no default choice).
 65 |      <BLANKLINE>
 66 |      Are you sure? [y/n]
 67 | 
 68 |     The same thing happens when the user enters text that isn't recognized:
 69 | 
 70 |     >>> prompt_for_confirmation("Are you sure?")
 71 |      <BLANKLINE>
 72 |      Are you sure? [y/n] about what?
 73 |      <BLANKLINE>
 74 |      Error: Please enter 'yes' or 'no' (the text 'about what?' is not recognized).
 75 |      <BLANKLINE>
 76 |      Are you sure? [y/n]
 77 |     """
 78 |     # Generate the text for the prompt.
 79 |     prompt_text = prepare_prompt_text(question, bold=True)
 80 |     # Append the valid replies (and default reply) to the prompt text.
 81 |     hint = "[Y/n]" if default else "[y/N]" if default is not None else "[y/n]"
 82 |     prompt_text += " %s " % prepare_prompt_text(hint, color=HIGHLIGHT_COLOR)
 83 |     # Loop until a valid response is given.
 84 |     logger.debug("Requesting interactive confirmation from terminal: %r", ansi_strip(prompt_text).rstrip())
 85 |     for attempt in retry_limit():
 86 |         reply = prompt_for_input(prompt_text, '', padding=padding, strip=True)
 87 |         if reply.lower() in ('y', 'yes'):
 88 |             logger.debug("Confirmation granted by reply (%r).", reply)
 89 |             return True
 90 |         elif reply.lower() in ('n', 'no'):
 91 |             logger.debug("Confirmation denied by reply (%r).", reply)
 92 |             return False
 93 |         elif (not reply) and default is not None:
 94 |             logger.debug("Default choice selected by empty reply (%r).",
 95 |                          "granted" if default else "denied")
 96 |             return default
 97 |         else:
 98 |             details = ("the text '%s' is not recognized" % reply
 99 |                        if reply else "there's no default choice")
100 |             logger.debug("Got %s reply (%s), retrying (%i/%i) ..",
101 |                          "invalid" if reply else "empty", details,
102 |                          attempt, MAX_ATTEMPTS)
103 |             warning("{indent}Error: Please enter 'yes' or 'no' ({details}).",
104 |                     indent=' ' if padding else '', details=details)
105 | 
106 | 
107 | def prompt_for_choice(choices, default=None, padding=True):
108 |     """
109 |     Prompt the user to select a choice from a group of options.
110 | 
111 |     :param choices: A sequence of strings with available options.
112 |     :param default: The default choice if the user simply presses Enter
113 |                     (expected to be a string, defaults to :data:`None`).
114 |     :param padding: Refer to the documentation of :func:`prompt_for_input()`.
115 |     :returns: The string corresponding to the user's choice.
116 |     :raises: - :exc:`~exceptions.ValueError` if `choices` is an empty sequence.
117 |              - Any exceptions raised by :func:`retry_limit()`.
118 |              - Any exceptions raised by :func:`prompt_for_input()`.
119 | 
120 |     When no options are given an exception is raised:
121 | 
122 |     >>> prompt_for_choice([])
123 |     Traceback (most recent call last):
124 |       File "humanfriendly/prompts.py", line 148, in prompt_for_choice
125 |         raise ValueError("Can't prompt for choice without any options!")
126 |     ValueError: Can't prompt for choice without any options!
127 | 
128 |     If a single option is given the user isn't prompted:
129 | 
130 |     >>> prompt_for_choice(['only one choice'])
131 |     'only one choice'
132 | 
133 |     Here's what the actual prompt looks like by default:
134 | 
135 |     >>> prompt_for_choice(['first option', 'second option'])
136 |      <BLANKLINE>
137 |       1. first option
138 |       2. second option
139 |      <BLANKLINE>
140 |      Enter your choice as a number or unique substring (Control-C aborts): second
141 |      <BLANKLINE>
142 |     'second option'
143 | 
144 |     If you don't like the whitespace (empty lines and indentation):
145 | 
146 |     >>> prompt_for_choice(['first option', 'second option'], padding=False)
147 |      1. first option
148 |      2. second option
149 |     Enter your choice as a number or unique substring (Control-C aborts): first
150 |     'first option'
151 |     """
152 |     indent = ' ' if padding else ''
153 |     # Make sure we can use 'choices' more than once (i.e. not a generator).
154 |     choices = list(choices)
155 |     if len(choices) == 1:
156 |         # If there's only one option there's no point in prompting the user.
157 |         logger.debug("Skipping interactive prompt because there's only option (%r).", choices[0])
158 |         return choices[0]
159 |     elif not choices:
160 |         # We can't render a choice prompt without any options.
161 |         raise ValueError("Can't prompt for choice without any options!")
162 |     # Generate the prompt text.
163 |     prompt_text = ('\n\n' if padding else '\n').join([
164 |         # Present the available choices in a user friendly way.
165 |         "\n".join([
166 |             (u" %i. %s" % (i, choice)) + (" (default choice)" if choice == default else "")
167 |             for i, choice in enumerate(choices, start=1)
168 |         ]),
169 |         # Instructions for the user.
170 |         "Enter your choice as a number or unique substring (Control-C aborts): ",
171 |     ])
172 |     prompt_text = prepare_prompt_text(prompt_text, bold=True)
173 |     # Loop until a valid choice is made.
174 |     logger.debug("Requesting interactive choice on terminal (options are %s) ..",
175 |                  concatenate(map(repr, choices)))
176 |     for attempt in retry_limit():
177 |         reply = prompt_for_input(prompt_text, '', padding=padding, strip=True)
178 |         if not reply and default is not None:
179 |             logger.debug("Default choice selected by empty reply (%r).", default)
180 |             return default
181 |         elif reply.isdigit():
182 |             index = int(reply) - 1
183 |             if 0 <= index < len(choices):
184 |                 logger.debug("Option (%r) selected by numeric reply (%s).", choices[index], reply)
185 |                 return choices[index]
186 |         # Check for substring matches.
187 |         matches = []
188 |         for choice in choices:
189 |             lower_reply = reply.lower()
190 |             lower_choice = choice.lower()
191 |             if lower_reply == lower_choice:
192 |                 # If we have an 'exact' match we return it immediately.
193 |                 logger.debug("Option (%r) selected by reply (exact match).", choice)
194 |                 return choice
195 |             elif lower_reply in lower_choice and len(lower_reply) > 0:
196 |                 # Otherwise we gather substring matches.
197 |                 matches.append(choice)
198 |         if len(matches) == 1:
199 |             # If a single choice was matched we return it.
200 |             logger.debug("Option (%r) selected by reply (substring match on %r).", matches[0], reply)
201 |             return matches[0]
202 |         else:
203 |             # Give the user a hint about what went wrong.
204 |             if matches:
205 |                 details = format("text '%s' matches more than one choice: %s", reply, concatenate(matches))
206 |             elif reply.isdigit():
207 |                 details = format("number %i is not a valid choice", int(reply))
208 |             elif reply and not reply.isspace():
209 |                 details = format("text '%s' doesn't match any choices", reply)
210 |             else:
211 |                 details = "there's no default choice"
212 |             logger.debug("Got %s reply (%s), retrying (%i/%i) ..",
213 |                          "invalid" if reply else "empty", details,
214 |                          attempt, MAX_ATTEMPTS)
215 |             warning("%sError: Invalid input (%s).", indent, details)
216 | 
217 | 
218 | def prompt_for_input(question, default=None, padding=True, strip=True):
219 |     """
220 |     Prompt the user for input (free form text).
221 | 
222 |     :param question: An explanation of what is expected from the user (a string).
223 |     :param default: The return value if the user doesn't enter any text or
224 |                     standard input is not connected to a terminal (which
225 |                     makes it impossible to prompt the user).
226 |     :param padding: Render empty lines before and after the prompt to make it
227 |                     stand out from the surrounding text? (a boolean, defaults
228 |                     to :data:`True`)
229 |     :param strip: Strip leading/trailing whitespace from the user's reply?
230 |     :returns: The text entered by the user (a string) or the value of the
231 |               `default` argument.
232 |     :raises: - :exc:`~exceptions.KeyboardInterrupt` when the program is
233 |                interrupted_ while the prompt is active, for example
234 |                because the user presses Control-C_.
235 |              - :exc:`~exceptions.EOFError` when reading from `standard input`_
236 |                fails, for example because the user presses Control-D_ or
237 |                because the standard input stream is redirected (only if
238 |                `default` is :data:`None`).
239 | 
240 |     .. _Control-C: https://en.wikipedia.org/wiki/Control-C#In_command-line_environments
241 |     .. _Control-D: https://en.wikipedia.org/wiki/End-of-transmission_character#Meaning_in_Unix
242 |     .. _interrupted: https://en.wikipedia.org/wiki/Unix_signal#SIGINT
243 |     .. _standard input: https://en.wikipedia.org/wiki/Standard_streams#Standard_input_.28stdin.29
244 |     """
245 |     prepare_friendly_prompts()
246 |     reply = None
247 |     try:
248 |         # Prefix an empty line to the text and indent by one space?
249 |         if padding:
250 |             question = '\n' + question
251 |             question = question.replace('\n', '\n ')
252 |         # Render the prompt and wait for the user's reply.
253 |         try:
254 |             reply = interactive_prompt(question)
255 |         finally:
256 |             if reply is None:
257 |                 # If the user terminated the prompt using Control-C or
258 |                 # Control-D instead of pressing Enter no newline will be
259 |                 # rendered after the prompt's text. The result looks kind of
260 |                 # weird:
261 |                 #
262 |                 #   $ python -c 'print(raw_input("Are you sure? "))'
263 |                 #   Are you sure? ^CTraceback (most recent call last):
264 |                 #     File "<string>", line 1, in <module>
265 |                 #   KeyboardInterrupt
266 |                 #
267 |                 # We can avoid this by emitting a newline ourselves if an
268 |                 # exception was raised (signaled by `reply' being None).
269 |                 sys.stderr.write('\n')
270 |             if padding:
271 |                 # If the caller requested (didn't opt out of) `padding' then we'll
272 |                 # emit a newline regardless of whether an exception is being
273 |                 # handled. This helps to make interactive prompts `stand out' from
274 |                 # a surrounding `wall of text' on the terminal.
275 |                 sys.stderr.write('\n')
276 |     except BaseException as e:
277 |         if isinstance(e, EOFError) and default is not None:
278 |             # If standard input isn't connected to an interactive terminal
279 |             # but the caller provided a default we'll return that.
280 |             logger.debug("Got EOF from terminal, returning default value (%r) ..", default)
281 |             return default
282 |         else:
283 |             # Otherwise we log that the prompt was interrupted but propagate
284 |             # the exception to the caller.
285 |             logger.warning("Interactive prompt was interrupted by exception!", exc_info=True)
286 |             raise
287 |     if default is not None and not reply:
288 |         # If the reply is empty and `default' is None we don't want to return
289 |         # None because it's nicer for callers to be able to assume that the
290 |         # return value is always a string.
291 |         return default
292 |     else:
293 |         return reply.strip()
294 | 
295 | 
296 | def prepare_prompt_text(prompt_text, **options):
297 |     """
298 |     Wrap a text to be rendered as an interactive prompt in ANSI escape sequences.
299 | 
300 |     :param prompt_text: The text to render on the prompt (a string).
301 |     :param options: Any keyword arguments are passed on to :func:`.ansi_wrap()`.
302 |     :returns: The resulting prompt text (a string).
303 | 
304 |     ANSI escape sequences are only used when the standard output stream is
305 |     connected to a terminal. When the standard input stream is connected to a
306 |     terminal any escape sequences are wrapped in "readline hints".
307 |     """
308 |     return (ansi_wrap(prompt_text, readline_hints=connected_to_terminal(sys.stdin), **options)
309 |             if terminal_supports_colors(sys.stdout)
310 |             else prompt_text)
311 | 
312 | 
313 | def prepare_friendly_prompts():
314 |     u"""
315 |     Make interactive prompts more user friendly.
316 | 
317 |     The prompts presented by :func:`python2:raw_input()` (in Python 2) and
318 |     :func:`python3:input()` (in Python 3) are not very user friendly by
319 |     default, for example the cursor keys (:kbd:`←`, :kbd:`↑`, :kbd:`→` and
320 |     :kbd:`↓`) and the :kbd:`Home` and :kbd:`End` keys enter characters instead
321 |     of performing the action you would expect them to. By simply importing the
322 |     :mod:`readline` module these prompts become much friendlier (as mentioned
323 |     in the Python standard library documentation).
324 | 
325 |     This function is called by the other functions in this module to enable
326 |     user friendly prompts.
327 |     """
328 |     import readline  # NOQA
329 | 
330 | 
331 | def retry_limit(limit=MAX_ATTEMPTS):
332 |     """
333 |     Allow the user to provide valid input up to `limit` times.
334 | 
335 |     :param limit: The maximum number of attempts (a number,
336 |                   defaults to :data:`MAX_ATTEMPTS`).
337 |     :returns: A generator of numbers starting from one.
338 |     :raises: :exc:`TooManyInvalidReplies` when an interactive prompt
339 |              receives repeated invalid input (:data:`MAX_ATTEMPTS`).
340 | 
341 |     This function returns a generator for interactive prompts that want to
342 |     repeat on invalid input without getting stuck in infinite loops.
343 |     """
344 |     for i in range(limit):
345 |         yield i + 1
346 |     msg = "Received too many invalid replies on interactive prompt, giving up! (tried %i times)"
347 |     formatted_msg = msg % limit
348 |     # Make sure the event is logged.
349 |     logger.warning(formatted_msg)
350 |     # Force the caller to decide what to do now.
351 |     raise TooManyInvalidReplies(formatted_msg)
352 | 
353 | 
354 | class TooManyInvalidReplies(Exception):
355 | 
356 |     """Raised by interactive prompts when they've received too many invalid inputs."""
357 | 


--------------------------------------------------------------------------------
/humanfriendly/sphinx.py:
--------------------------------------------------------------------------------
  1 | # Human friendly input/output in Python.
  2 | #
  3 | # Author: Peter Odding <peter@peterodding.com>
  4 | # Last Change: February 17, 2016
  5 | # URL: https://humanfriendly.readthedocs.io
  6 | 
  7 | """
  8 | Customizations for and integration with the Sphinx_ documentation generator.
  9 | 
 10 | The :mod:`humanfriendly.sphinx` module uses the `Sphinx extension API`_ to
 11 | customize the process of generating Sphinx based Python documentation.
 12 | The most relevant functions to take a look at are :func:`setup()`,
 13 | :func:`enable_special_methods()` and :func:`enable_usage_formatting()`.
 14 | 
 15 | .. _Sphinx: http://www.sphinx-doc.org/
 16 | .. _Sphinx extension API: http://sphinx-doc.org/extdev/appapi.html
 17 | """
 18 | 
 19 | # Standard library modules.
 20 | import logging
 21 | import types
 22 | 
 23 | # Modules included in our package.
 24 | from humanfriendly.usage import USAGE_MARKER, render_usage
 25 | 
 26 | # Initialize a logger for this module.
 27 | logger = logging.getLogger(__name__)
 28 | 
 29 | 
 30 | def setup(app):
 31 |     """
 32 |     Enable all of the provided Sphinx_ customizations.
 33 | 
 34 |     :param app: The Sphinx application object.
 35 | 
 36 |     The :func:`setup()` function makes it easy to enable all of the Sphinx
 37 |     customizations provided by the :mod:`humanfriendly.sphinx` module with the
 38 |     least amount of code. All you need to do is to add the module name to the
 39 |     ``extensions`` variable in your ``conf.py`` file:
 40 | 
 41 |     .. code-block:: python
 42 | 
 43 |        # Sphinx extension module names.
 44 |        extensions = [
 45 |            'sphinx.ext.autodoc',
 46 |            'sphinx.ext.doctest',
 47 |            'sphinx.ext.intersphinx',
 48 |            'humanfriendly.sphinx',
 49 |        ]
 50 | 
 51 |     When Sphinx sees the :mod:`humanfriendly.sphinx` name it will import the
 52 |     module and call its :func:`setup()` function.
 53 | 
 54 |     At the time of writing this just calls :func:`enable_special_methods()` and
 55 |     :func:`enable_usage_formatting()`, but of course more functionality may be
 56 |     added at a later stage. If you don't like that idea you may be better of
 57 |     calling the individual functions from your own ``setup()`` function.
 58 |     """
 59 |     enable_special_methods(app)
 60 |     enable_usage_formatting(app)
 61 | 
 62 | 
 63 | def enable_special_methods(app):
 64 |     """
 65 |     Enable documenting "special methods" using the autodoc_ extension.
 66 | 
 67 |     :param app: The Sphinx application object.
 68 | 
 69 |     This function connects the :func:`special_methods_callback()` function to
 70 |     ``autodoc-skip-member`` events.
 71 | 
 72 |     .. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html
 73 |     """
 74 |     app.connect('autodoc-skip-member', special_methods_callback)
 75 | 
 76 | 
 77 | def special_methods_callback(app, what, name, obj, skip, options):
 78 |     """
 79 |     Enable documenting "special methods" using the autodoc_ extension.
 80 | 
 81 |     Refer to :func:`enable_special_methods()` to enable the use of this
 82 |     function (you probably don't want to call
 83 |     :func:`special_methods_callback()` directly).
 84 | 
 85 |     This function implements a callback for ``autodoc-skip-member`` events to
 86 |     include documented "special methods" (method names with two leading and two
 87 |     trailing underscores) in your documentation. The result is similar to the
 88 |     use of the ``special-members`` flag with one big difference: Special
 89 |     methods are included but other types of members are ignored. This means
 90 |     that attributes like ``__weakref__`` will always be ignored (this was my
 91 |     main annoyance with the ``special-members`` flag).
 92 | 
 93 |     The parameters expected by this function are those defined for Sphinx event
 94 |     callback functions (i.e. I'm not going to document them here :-).
 95 |     """
 96 |     if getattr(obj, '__doc__', None) and isinstance(obj, (types.FunctionType, types.MethodType)):
 97 |         return False
 98 |     else:
 99 |         return skip
100 | 
101 | 
102 | def enable_usage_formatting(app):
103 |     """
104 |     Reformat human friendly usage messages to reStructuredText_.
105 | 
106 |     :param app: The Sphinx application object (as given to ``setup()``).
107 | 
108 |     This function connects the :func:`usage_message_callback()` function to
109 |     ``autodoc-process-docstring`` events.
110 | 
111 |     .. _reStructuredText: https://en.wikipedia.org/wiki/ReStructuredText
112 |     """
113 |     app.connect('autodoc-process-docstring', usage_message_callback)
114 | 
115 | 
116 | def usage_message_callback(app, what, name, obj, options, lines):
117 |     """
118 |     Reformat human friendly usage messages to reStructuredText_.
119 | 
120 |     Refer to :func:`enable_usage_formatting()` to enable the use of this
121 |     function (you probably don't want to call :func:`usage_message_callback()`
122 |     directly).
123 | 
124 |     This function implements a callback for ``autodoc-process-docstring`` that
125 |     reformats module docstrings using :func:`.render_usage()` so that Sphinx
126 |     doesn't mangle usage messages that were written to be human readable
127 |     instead of machine readable. Only module docstrings whose first line starts
128 |     with :data:`.USAGE_MARKER` are reformatted.
129 | 
130 |     The parameters expected by this function are those defined for Sphinx event
131 |     callback functions (i.e. I'm not going to document them here :-).
132 |     """
133 |     # Make sure we only modify the docstrings of modules.
134 |     if isinstance(obj, types.ModuleType) and lines:
135 |         # Make sure we only modify docstrings containing a usage message.
136 |         if lines[0].startswith(USAGE_MARKER):
137 |             # Convert the usage message to reStructuredText.
138 |             text = render_usage('\n'.join(lines))
139 |             # Clear the existing line buffer.
140 |             while lines:
141 |                 lines.pop()
142 |             # Fill up the buffer with our modified docstring.
143 |             lines.extend(text.splitlines())
144 | 


--------------------------------------------------------------------------------
/humanfriendly/tables.py:
--------------------------------------------------------------------------------
  1 | # Human friendly input/output in Python.
  2 | #
  3 | # Author: Peter Odding <peter@peterodding.com>
  4 | # Last Change: March 25, 2018
  5 | # URL: https://humanfriendly.readthedocs.io
  6 | 
  7 | """
  8 | Functions that render ASCII tables.
  9 | 
 10 | Some generic notes about the table formatting functions in this module:
 11 | 
 12 | - These functions were not written with performance in mind (*at all*) because
 13 |   they're intended to format tabular data to be presented on a terminal. If
 14 |   someone were to run into a performance problem using these functions, they'd
 15 |   be printing so much tabular data to the terminal that a human wouldn't be
 16 |   able to digest the tabular data anyway, so the point is moot :-).
 17 | 
 18 | - These functions ignore ANSI escape sequences (at least the ones generated by
 19 |   the :mod:`~humanfriendly.terminal` module) in the calculation of columns
 20 |   widths. On reason for this is that column names are highlighted in color when
 21 |   connected to a terminal. It also means that you can use ANSI escape sequences
 22 |   to highlight certain column's values if you feel like it (for example to
 23 |   highlight deviations from the norm in an overview of calculated values).
 24 | """
 25 | 
 26 | # Standard library modules.
 27 | import collections
 28 | import re
 29 | 
 30 | # Modules included in our package.
 31 | from humanfriendly.compat import coerce_string
 32 | from humanfriendly.terminal import (
 33 |     ansi_strip,
 34 |     ansi_width,
 35 |     ansi_wrap,
 36 |     terminal_supports_colors,
 37 |     find_terminal_size,
 38 |     HIGHLIGHT_COLOR,
 39 | )
 40 | 
 41 | # Public identifiers that require documentation.
 42 | __all__ = (
 43 |     'format_pretty_table',
 44 |     'format_robust_table',
 45 |     'format_smart_table',
 46 | )
 47 | 
 48 | # Compiled regular expression pattern to recognize table columns containing
 49 | # numeric data (integer and/or floating point numbers). Used to right-align the
 50 | # contents of such columns.
 51 | #
 52 | # Pre-emptive snarky comment: This pattern doesn't match every possible
 53 | # floating point number notation!?!1!1
 54 | #
 55 | # Response: I know, that's intentional. The use of this regular expression
 56 | # pattern has a very high DWIM level and weird floating point notations do not
 57 | # fall under the DWIM umbrella :-).
 58 | NUMERIC_DATA_PATTERN = re.compile(r'^\d+(\.\d+)?$')
 59 | 
 60 | 
 61 | def format_smart_table(data, column_names):
 62 |     """
 63 |     Render tabular data using the most appropriate representation.
 64 | 
 65 |     :param data: An iterable (e.g. a :func:`tuple` or :class:`list`)
 66 |                  containing the rows of the table, where each row is an
 67 |                  iterable containing the columns of the table (strings).
 68 |     :param column_names: An iterable of column names (strings).
 69 |     :returns: The rendered table (a string).
 70 | 
 71 |     If you want an easy way to render tabular data on a terminal in a human
 72 |     friendly format then this function is for you! It works as follows:
 73 | 
 74 |     - If the input data doesn't contain any line breaks the function
 75 |       :func:`format_pretty_table()` is used to render a pretty table. If the
 76 |       resulting table fits in the terminal without wrapping the rendered pretty
 77 |       table is returned.
 78 | 
 79 |     - If the input data does contain line breaks or if a pretty table would
 80 |       wrap (given the width of the terminal) then the function
 81 |       :func:`format_robust_table()` is used to render a more robust table that
 82 |       can deal with data containing line breaks and long text.
 83 |     """
 84 |     # Normalize the input in case we fall back from a pretty table to a robust
 85 |     # table (in which case we'll definitely iterate the input more than once).
 86 |     data = [normalize_columns(r) for r in data]
 87 |     column_names = normalize_columns(column_names)
 88 |     # Make sure the input data doesn't contain any line breaks (because pretty
 89 |     # tables break horribly when a column's text contains a line break :-).
 90 |     if not any(any('\n' in c for c in r) for r in data):
 91 |         # Render a pretty table.
 92 |         pretty_table = format_pretty_table(data, column_names)
 93 |         # Check if the pretty table fits in the terminal.
 94 |         table_width = max(map(ansi_width, pretty_table.splitlines()))
 95 |         num_rows, num_columns = find_terminal_size()
 96 |         if table_width <= num_columns:
 97 |             # The pretty table fits in the terminal without wrapping!
 98 |             return pretty_table
 99 |     # Fall back to a robust table when a pretty table won't work.
100 |     return format_robust_table(data, column_names)
101 | 
102 | 
103 | def format_pretty_table(data, column_names=None, horizontal_bar='-', vertical_bar='|'):
104 |     """
105 |     Render a table using characters like dashes and vertical bars to emulate borders.
106 | 
107 |     :param data: An iterable (e.g. a :func:`tuple` or :class:`list`)
108 |                  containing the rows of the table, where each row is an
109 |                  iterable containing the columns of the table (strings).
110 |     :param column_names: An iterable of column names (strings).
111 |     :param horizontal_bar: The character used to represent a horizontal bar (a
112 |                            string).
113 |     :param vertical_bar: The character used to represent a vertical bar (a
114 |                          string).
115 |     :returns: The rendered table (a string).
116 | 
117 |     Here's an example:
118 | 
119 |     >>> from humanfriendly.tables import format_pretty_table
120 |     >>> column_names = ['Version', 'Uploaded on', 'Downloads']
121 |     >>> humanfriendly_releases = [
122 |     ... ['1.23', '2015-05-25', '218'],
123 |     ... ['1.23.1', '2015-05-26', '1354'],
124 |     ... ['1.24', '2015-05-26', '223'],
125 |     ... ['1.25', '2015-05-26', '4319'],
126 |     ... ['1.25.1', '2015-06-02', '197'],
127 |     ... ]
128 |     >>> print(format_pretty_table(humanfriendly_releases, column_names))
129 |     -------------------------------------
130 |     | Version | Uploaded on | Downloads |
131 |     -------------------------------------
132 |     | 1.23    | 2015-05-25  |       218 |
133 |     | 1.23.1  | 2015-05-26  |      1354 |
134 |     | 1.24    | 2015-05-26  |       223 |
135 |     | 1.25    | 2015-05-26  |      4319 |
136 |     | 1.25.1  | 2015-06-02  |       197 |
137 |     -------------------------------------
138 | 
139 |     Notes about the resulting table:
140 | 
141 |     - If a column contains numeric data (integer and/or floating point
142 |       numbers) in all rows (ignoring column names of course) then the content
143 |       of that column is right-aligned, as can be seen in the example above. The
144 |       idea here is to make it easier to compare the numbers in different
145 |       columns to each other.
146 | 
147 |     - The column names are highlighted in color so they stand out a bit more
148 |       (see also :data:`.HIGHLIGHT_COLOR`). The following screen shot shows what
149 |       that looks like (my terminals are always set to white text on a black
150 |       background):
151 | 
152 |       .. image:: images/pretty-table.png
153 |     """
154 |     # Normalize the input because we'll have to iterate it more than once.
155 |     data = [normalize_columns(r) for r in data]
156 |     if column_names is not None:
157 |         column_names = normalize_columns(column_names)
158 |         if column_names:
159 |             if terminal_supports_colors():
160 |                 column_names = [highlight_column_name(n) for n in column_names]
161 |             data.insert(0, column_names)
162 |     # Calculate the maximum width of each column.
163 |     widths = collections.defaultdict(int)
164 |     numeric_data = collections.defaultdict(list)
165 |     for row_index, row in enumerate(data):
166 |         for column_index, column in enumerate(row):
167 |             widths[column_index] = max(widths[column_index], ansi_width(column))
168 |             if not (column_names and row_index == 0):
169 |                 numeric_data[column_index].append(bool(NUMERIC_DATA_PATTERN.match(ansi_strip(column))))
170 |     # Create a horizontal bar of dashes as a delimiter.
171 |     line_delimiter = horizontal_bar * (sum(widths.values()) + len(widths) * 3 + 1)
172 |     # Start the table with a vertical bar.
173 |     lines = [line_delimiter]
174 |     # Format the rows and columns.
175 |     for row_index, row in enumerate(data):
176 |         line = [vertical_bar]
177 |         for column_index, column in enumerate(row):
178 |             padding = ' ' * (widths[column_index] - ansi_width(column))
179 |             if all(numeric_data[column_index]):
180 |                 line.append(' ' + padding + column + ' ')
181 |             else:
182 |                 line.append(' ' + column + padding + ' ')
183 |             line.append(vertical_bar)
184 |         lines.append(u''.join(line))
185 |         if column_names and row_index == 0:
186 |             lines.append(line_delimiter)
187 |     # End the table with a vertical bar.
188 |     lines.append(line_delimiter)
189 |     # Join the lines, returning a single string.
190 |     return u'\n'.join(lines)
191 | 
192 | 
193 | def format_robust_table(data, column_names):
194 |     """
195 |     Render tabular data with one column per line (allowing columns with line breaks).
196 | 
197 |     :param data: An iterable (e.g. a :func:`tuple` or :class:`list`)
198 |                  containing the rows of the table, where each row is an
199 |                  iterable containing the columns of the table (strings).
200 |     :param column_names: An iterable of column names (strings).
201 |     :returns: The rendered table (a string).
202 | 
203 |     Here's an example:
204 | 
205 |     >>> from humanfriendly.tables import format_robust_table
206 |     >>> column_names = ['Version', 'Uploaded on', 'Downloads']
207 |     >>> humanfriendly_releases = [
208 |     ... ['1.23', '2015-05-25', '218'],
209 |     ... ['1.23.1', '2015-05-26', '1354'],
210 |     ... ['1.24', '2015-05-26', '223'],
211 |     ... ['1.25', '2015-05-26', '4319'],
212 |     ... ['1.25.1', '2015-06-02', '197'],
213 |     ... ]
214 |     >>> print(format_robust_table(humanfriendly_releases, column_names))
215 |     -----------------------
216 |     Version: 1.23
217 |     Uploaded on: 2015-05-25
218 |     Downloads: 218
219 |     -----------------------
220 |     Version: 1.23.1
221 |     Uploaded on: 2015-05-26
222 |     Downloads: 1354
223 |     -----------------------
224 |     Version: 1.24
225 |     Uploaded on: 2015-05-26
226 |     Downloads: 223
227 |     -----------------------
228 |     Version: 1.25
229 |     Uploaded on: 2015-05-26
230 |     Downloads: 4319
231 |     -----------------------
232 |     Version: 1.25.1
233 |     Uploaded on: 2015-06-02
234 |     Downloads: 197
235 |     -----------------------
236 | 
237 |     The column names are highlighted in bold font and color so they stand out a
238 |     bit more (see :data:`.HIGHLIGHT_COLOR`).
239 |     """
240 |     blocks = []
241 |     column_names = ["%s:" % n for n in normalize_columns(column_names)]
242 |     if terminal_supports_colors():
243 |         column_names = [highlight_column_name(n) for n in column_names]
244 |     # Convert each row into one or more `name: value' lines (one per column)
245 |     # and group each `row of lines' into a block (i.e. rows become blocks).
246 |     for row in data:
247 |         lines = []
248 |         for column_index, column_text in enumerate(normalize_columns(row)):
249 |             stripped_column = column_text.strip()
250 |             if '\n' not in stripped_column:
251 |                 # Columns without line breaks are formatted inline.
252 |                 lines.append("%s %s" % (column_names[column_index], stripped_column))
253 |             else:
254 |                 # Columns with line breaks could very well contain indented
255 |                 # lines, so we'll put the column name on a separate line. This
256 |                 # way any indentation remains intact, and it's easier to
257 |                 # copy/paste the text.
258 |                 lines.append(column_names[column_index])
259 |                 lines.extend(column_text.rstrip().splitlines())
260 |         blocks.append(lines)
261 |     # Calculate the width of the row delimiter.
262 |     num_rows, num_columns = find_terminal_size()
263 |     longest_line = max(max(map(ansi_width, lines)) for lines in blocks)
264 |     delimiter = u"\n%s\n" % ('-' * min(longest_line, num_columns))
265 |     # Force a delimiter at the start and end of the table.
266 |     blocks.insert(0, "")
267 |     blocks.append("")
268 |     # Embed the row delimiter between every two blocks.
269 |     return delimiter.join(u"\n".join(b) for b in blocks).strip()
270 | 
271 | 
272 | def format_rst_table(data, column_names=None):
273 |     """
274 |     Render a table in reStructuredText_ format.
275 | 
276 |     :param data: An iterable (e.g. a :func:`tuple` or :class:`list`)
277 |                  containing the rows of the table, where each row is an
278 |                  iterable containing the columns of the table (strings).
279 |     :param column_names: An iterable of column names (strings).
280 |     :returns: The rendered table (a string).
281 | 
282 |     Here's an example:
283 | 
284 |     >>> from humanfriendly.tables import format_rst_table
285 |     >>> column_names = ['Version', 'Uploaded on', 'Downloads']
286 |     >>> humanfriendly_releases = [
287 |     ... ['1.23', '2015-05-25', '218'],
288 |     ... ['1.23.1', '2015-05-26', '1354'],
289 |     ... ['1.24', '2015-05-26', '223'],
290 |     ... ['1.25', '2015-05-26', '4319'],
291 |     ... ['1.25.1', '2015-06-02', '197'],
292 |     ... ]
293 |     >>> print(format_rst_table(humanfriendly_releases, column_names))
294 |     =======  ===========  =========
295 |     Version  Uploaded on  Downloads
296 |     =======  ===========  =========
297 |     1.23     2015-05-25   218
298 |     1.23.1   2015-05-26   1354
299 |     1.24     2015-05-26   223
300 |     1.25     2015-05-26   4319
301 |     1.25.1   2015-06-02   197
302 |     =======  ===========  =========
303 | 
304 |     .. _reStructuredText: https://en.wikipedia.org/wiki/ReStructuredText
305 |     """
306 |     data = [normalize_columns(r) for r in data]
307 |     if column_names:
308 |         data.insert(0, normalize_columns(column_names))
309 |     # Calculate the maximum width of each column.
310 |     widths = collections.defaultdict(int)
311 |     for row in data:
312 |         for index, column in enumerate(row):
313 |             widths[index] = max(widths[index], len(column))
314 |     # Pad the columns using whitespace.
315 |     for row in data:
316 |         for index, column in enumerate(row):
317 |             if index < (len(row) - 1):
318 |                 row[index] = column.ljust(widths[index])
319 |     # Add table markers.
320 |     delimiter = ['=' * w for i, w in sorted(widths.items())]
321 |     if column_names:
322 |         data.insert(1, delimiter)
323 |     data.insert(0, delimiter)
324 |     data.append(delimiter)
325 |     # Join the lines and columns together.
326 |     return '\n'.join('  '.join(r) for r in data)
327 | 
328 | 
329 | def normalize_columns(row):
330 |     return [coerce_string(c) for c in row]
331 | 
332 | 
333 | def highlight_column_name(name):
334 |     return ansi_wrap(name, bold=True, color=HIGHLIGHT_COLOR)
335 | 


--------------------------------------------------------------------------------
/humanfriendly/testing.py:
--------------------------------------------------------------------------------
  1 | # Human friendly input/output in Python.
  2 | #
  3 | # Author: Peter Odding <peter@peterodding.com>
  4 | # Last Change: July 16, 2017
  5 | # URL: https://humanfriendly.readthedocs.io
  6 | 
  7 | """
  8 | Utility classes and functions that make it easy to write :mod:`unittest` compatible test suites.
  9 | 
 10 | Over the years I've developed the habit of writing test suites for Python
 11 | projects using the :mod:`unittest` module. During those years I've come to know
 12 | pytest_ and in fact I use pytest to run my test suites (due to its much better
 13 | error reporting) but I've yet to publish a test suite that *requires* pytest.
 14 | I have several reasons for doing so:
 15 | 
 16 | - It's nice to keep my test suites as simple and accessible as possible and
 17 |   not requiring a specific test runner is part of that attitude.
 18 | 
 19 | - Whereas :mod:`unittest` is quite explicit, pytest contains a lot of magic,
 20 |   which kind of contradicts the Python mantra "explicit is better than
 21 |   implicit" (IMHO).
 22 | 
 23 | .. _pytest: https://docs.pytest.org
 24 | """
 25 | 
 26 | # Standard library module
 27 | import functools
 28 | import logging
 29 | import os
 30 | import pipes
 31 | import shutil
 32 | import sys
 33 | import tempfile
 34 | import time
 35 | 
 36 | # Modules included in our package.
 37 | from humanfriendly.compat import StringIO, unicode, unittest
 38 | from humanfriendly.text import compact, random_string
 39 | 
 40 | # Initialize a logger for this module.
 41 | logger = logging.getLogger(__name__)
 42 | 
 43 | # A unique object reference used to detect missing attributes.
 44 | NOTHING = object()
 45 | 
 46 | # Public identifiers that require documentation.
 47 | __all__ = (
 48 |     'CallableTimedOut',
 49 |     'CaptureOutput',
 50 |     'ContextManager',
 51 |     'CustomSearchPath',
 52 |     'MockedProgram',
 53 |     'PatchedAttribute',
 54 |     'PatchedItem',
 55 |     'TemporaryDirectory',
 56 |     'TestCase',
 57 |     'configure_logging',
 58 |     'make_dirs',
 59 |     'retry',
 60 |     'run_cli',
 61 |     'touch',
 62 | )
 63 | 
 64 | 
 65 | def configure_logging(log_level=logging.DEBUG):
 66 |     """configure_logging(log_level=logging.DEBUG)
 67 |     Automatically configure logging to the terminal.
 68 | 
 69 |     :param log_level: The log verbosity (a number, defaults to
 70 |                       :data:`logging.DEBUG`).
 71 | 
 72 |     When :mod:`coloredlogs` is installed :func:`coloredlogs.install()` will be
 73 |     used to configure logging to the terminal. When this fails with an
 74 |     :exc:`~exceptions.ImportError` then :func:`logging.basicConfig()` is used
 75 |     as a fall back.
 76 |     """
 77 |     try:
 78 |         import coloredlogs
 79 |         coloredlogs.install(level=log_level)
 80 |     except ImportError:
 81 |         logging.basicConfig(
 82 |             level=log_level,
 83 |             format='%(asctime)s %(name)s[%(process)d] %(levelname)s %(message)s',
 84 |             datefmt='%Y-%m-%d %H:%M:%S')
 85 | 
 86 | 
 87 | def make_dirs(pathname):
 88 |     """
 89 |     Create missing directories.
 90 | 
 91 |     :param pathname: The pathname of a directory (a string).
 92 |     """
 93 |     if not os.path.isdir(pathname):
 94 |         os.makedirs(pathname)
 95 | 
 96 | 
 97 | def retry(func, timeout=60, exc_type=AssertionError):
 98 |     """retry(func, timeout=60, exc_type=AssertionError)
 99 |     Retry a function until assertions no longer fail.
100 | 
101 |     :param func: A callable. When the callable returns
102 |                  :data:`False` it will also be retried.
103 |     :param timeout: The number of seconds after which to abort (a number,
104 |                     defaults to 60).
105 |     :param exc_type: The type of exceptions to retry (defaults
106 |                      to :exc:`~exceptions.AssertionError`).
107 |     :returns: The value returned by `func`.
108 |     :raises: Once the timeout has expired :func:`retry()` will raise the
109 |              previously retried assertion error. When `func` keeps returning
110 |              :data:`False` until `timeout` expires :exc:`CallableTimedOut`
111 |              will be raised.
112 | 
113 |     This function sleeps between retries to avoid claiming CPU cycles we don't
114 |     need. It starts by sleeping for 0.1 second but adjusts this to one second
115 |     as the number of retries grows.
116 |     """
117 |     pause = 0.1
118 |     timeout += time.time()
119 |     while True:
120 |         try:
121 |             result = func()
122 |             if result is not False:
123 |                 return result
124 |         except exc_type:
125 |             if time.time() > timeout:
126 |                 raise
127 |         else:
128 |             if time.time() > timeout:
129 |                 raise CallableTimedOut()
130 |         time.sleep(pause)
131 |         if pause < 1:
132 |             pause *= 2
133 | 
134 | 
135 | def run_cli(entry_point, *arguments, **options):
136 |     """
137 |     Test a command line entry point.
138 | 
139 |     :param entry_point: The function that implements the command line interface
140 |                         (a callable).
141 |     :param arguments: Any positional arguments (strings) become the command
142 |                       line arguments (:data:`sys.argv` items 1-N).
143 |     :param options: The following keyword arguments are supported:
144 | 
145 |                     **input**
146 |                      Refer to :class:`CaptureOutput`.
147 |                     **merged**
148 |                      Refer to :class:`CaptureOutput`.
149 |                     **program_name**
150 |                      Used to set :data:`sys.argv` item 0.
151 |     :returns: A tuple with two values:
152 | 
153 |               1. The return code (an integer).
154 |               2. The captured output (a string).
155 |     """
156 |     merged = options.get('merged', False)
157 |     # Add the `program_name' option to the arguments.
158 |     arguments = list(arguments)
159 |     arguments.insert(0, options.pop('program_name', sys.executable))
160 |     # Log the command line arguments (and the fact that we're about to call the
161 |     # command line entry point function).
162 |     logger.debug("Calling command line entry point with arguments: %s", arguments)
163 |     # Prepare to capture the return code and output even if the command line
164 |     # interface raises an exception (whether the exception type is SystemExit
165 |     # or something else).
166 |     returncode = 0
167 |     stdout = None
168 |     stderr = None
169 |     try:
170 |         # Temporarily override sys.argv.
171 |         with PatchedAttribute(sys, 'argv', arguments):
172 |             # Manipulate the standard input/output/error streams.
173 |             with CaptureOutput(**options) as capturer:
174 |                 try:
175 |                     # Call the command line interface.
176 |                     entry_point()
177 |                 finally:
178 |                     # Get the output even if an exception is raised.
179 |                     stdout = capturer.stdout.getvalue()
180 |                     stderr = capturer.stderr.getvalue()
181 |                     # Reconfigure logging to the terminal because it is very
182 |                     # likely that the entry point function has changed the
183 |                     # configured log level.
184 |                     configure_logging()
185 |     except BaseException as e:
186 |         if isinstance(e, SystemExit):
187 |             logger.debug("Intercepting return code %s from SystemExit exception.", e.code)
188 |             returncode = e.code
189 |         else:
190 |             logger.warning("Defaulting return code to 1 due to raised exception.", exc_info=True)
191 |             returncode = 1
192 |     else:
193 |         logger.debug("Command line entry point returned successfully!")
194 |     # Always log the output captured on stdout/stderr, to make it easier to
195 |     # diagnose test failures (but avoid duplicate logging when merged=True).
196 |     merged_streams = [('merged streams', stdout)]
197 |     separate_streams = [('stdout', stdout), ('stderr', stderr)]
198 |     streams = merged_streams if merged else separate_streams
199 |     for name, value in streams:
200 |         if value:
201 |             logger.debug("Output on %s:\n%s", name, value)
202 |         else:
203 |             logger.debug("No output on %s.", name)
204 |     return returncode, stdout
205 | 
206 | 
207 | def touch(filename):
208 |     """
209 |     The equivalent of the UNIX ``touch`` program in Python.
210 | 
211 |     :param filename: The pathname of the file to touch (a string).
212 | 
213 |     Note that missing directories are automatically created using
214 |     :func:`make_dirs()`.
215 |     """
216 |     make_dirs(os.path.dirname(filename))
217 |     with open(filename, 'a'):
218 |         os.utime(filename, None)
219 | 
220 | 
221 | class CallableTimedOut(Exception):
222 | 
223 |     """Raised by :func:`retry()` when the timeout expires."""
224 | 
225 | 
226 | class ContextManager(object):
227 | 
228 |     """Base class to enable composition of context managers."""
229 | 
230 |     def __enter__(self):
231 |         """Enable use as context managers."""
232 |         return self
233 | 
234 |     def __exit__(self, exc_type=None, exc_value=None, traceback=None):
235 |         """Enable use as context managers."""
236 | 
237 | 
238 | class PatchedAttribute(ContextManager):
239 | 
240 |     """Context manager that temporary replaces an object attribute using :func:`setattr()`."""
241 | 
242 |     def __init__(self, obj, name, value):
243 |         """
244 |         Initialize a :class:`PatchedAttribute` object.
245 | 
246 |         :param obj: The object to patch.
247 |         :param name: An attribute name.
248 |         :param value: The value to set.
249 |         """
250 |         self.object_to_patch = obj
251 |         self.attribute_to_patch = name
252 |         self.patched_value = value
253 |         self.original_value = NOTHING
254 | 
255 |     def __enter__(self):
256 |         """
257 |         Replace (patch) the attribute.
258 | 
259 |         :returns: The object whose attribute was patched.
260 |         """
261 |         # Enable composition of context managers.
262 |         super(PatchedAttribute, self).__enter__()
263 |         # Patch the object's attribute.
264 |         self.original_value = getattr(self.object_to_patch, self.attribute_to_patch, NOTHING)
265 |         setattr(self.object_to_patch, self.attribute_to_patch, self.patched_value)
266 |         return self.object_to_patch
267 | 
268 |     def __exit__(self, exc_type=None, exc_value=None, traceback=None):
269 |         """Restore the attribute to its original value."""
270 |         # Enable composition of context managers.
271 |         super(PatchedAttribute, self).__exit__(exc_type, exc_value, traceback)
272 |         # Restore the object's attribute.
273 |         if self.original_value is NOTHING:
274 |             delattr(self.object_to_patch, self.attribute_to_patch)
275 |         else:
276 |             setattr(self.object_to_patch, self.attribute_to_patch, self.original_value)
277 | 
278 | 
279 | class PatchedItem(ContextManager):
280 | 
281 |     """Context manager that temporary replaces an object item using :func:`~object.__setitem__()`."""
282 | 
283 |     def __init__(self, obj, item, value):
284 |         """
285 |         Initialize a :class:`PatchedItem` object.
286 | 
287 |         :param obj: The object to patch.
288 |         :param item: The item to patch.
289 |         :param value: The value to set.
290 |         """
291 |         self.object_to_patch = obj
292 |         self.item_to_patch = item
293 |         self.patched_value = value
294 |         self.original_value = NOTHING
295 | 
296 |     def __enter__(self):
297 |         """
298 |         Replace (patch) the item.
299 | 
300 |         :returns: The object whose item was patched.
301 |         """
302 |         # Enable composition of context managers.
303 |         super(PatchedItem, self).__enter__()
304 |         # Patch the object's item.
305 |         try:
306 |             self.original_value = self.object_to_patch[self.item_to_patch]
307 |         except KeyError:
308 |             self.original_value = NOTHING
309 |         self.object_to_patch[self.item_to_patch] = self.patched_value
310 |         return self.object_to_patch
311 | 
312 |     def __exit__(self, exc_type=None, exc_value=None, traceback=None):
313 |         """Restore the item to its original value."""
314 |         # Enable composition of context managers.
315 |         super(PatchedItem, self).__exit__(exc_type, exc_value, traceback)
316 |         # Restore the object's item.
317 |         if self.original_value is NOTHING:
318 |             del self.object_to_patch[self.item_to_patch]
319 |         else:
320 |             self.object_to_patch[self.item_to_patch] = self.original_value
321 | 
322 | 
323 | class TemporaryDirectory(ContextManager):
324 | 
325 |     """
326 |     Easy temporary directory creation & cleanup using the :keyword:`with` statement.
327 | 
328 |     Here's an example of how to use this:
329 | 
330 |     .. code-block:: python
331 | 
332 |        with TemporaryDirectory() as directory:
333 |            # Do something useful here.
334 |            assert os.path.isdir(directory)
335 |     """
336 | 
337 |     def __init__(self, **options):
338 |         """
339 |         Initialize a :class:`TemporaryDirectory` object.
340 | 
341 |         :param options: Any keyword arguments are passed on to
342 |                         :func:`tempfile.mkdtemp()`.
343 |         """
344 |         self.mkdtemp_options = options
345 |         self.temporary_directory = None
346 | 
347 |     def __enter__(self):
348 |         """
349 |         Create the temporary directory using :func:`tempfile.mkdtemp()`.
350 | 
351 |         :returns: The pathname of the directory (a string).
352 |         """
353 |         # Enable composition of context managers.
354 |         super(TemporaryDirectory, self).__enter__()
355 |         # Create the temporary directory.
356 |         self.temporary_directory = tempfile.mkdtemp(**self.mkdtemp_options)
357 |         return self.temporary_directory
358 | 
359 |     def __exit__(self, exc_type=None, exc_value=None, traceback=None):
360 |         """Cleanup the temporary directory using :func:`shutil.rmtree()`."""
361 |         # Enable composition of context managers.
362 |         super(TemporaryDirectory, self).__exit__(exc_type, exc_value, traceback)
363 |         # Cleanup the temporary directory.
364 |         if self.temporary_directory is not None:
365 |             shutil.rmtree(self.temporary_directory)
366 |             self.temporary_directory = None
367 | 
368 | 
369 | class MockedHomeDirectory(PatchedItem, TemporaryDirectory):
370 | 
371 |     """
372 |     Context manager to temporarily change ``$HOME`` (the current user's profile directory).
373 | 
374 |     This class is a composition of the :class:`PatchedItem` and
375 |     :class:`TemporaryDirectory` context managers.
376 |     """
377 | 
378 |     def __init__(self):
379 |         """Initialize a :class:`MockedHomeDirectory` object."""
380 |         PatchedItem.__init__(self, os.environ, 'HOME', os.environ.get('HOME'))
381 |         TemporaryDirectory.__init__(self)
382 | 
383 |     def __enter__(self):
384 |         """
385 |         Activate the custom ``$PATH``.
386 | 
387 |         :returns: The pathname of the directory that has
388 |                   been added to ``$PATH`` (a string).
389 |         """
390 |         # Get the temporary directory.
391 |         directory = TemporaryDirectory.__enter__(self)
392 |         # Override the value to patch now that we have
393 |         # the pathname of the temporary directory.
394 |         self.patched_value = directory
395 |         # Temporary patch $HOME.
396 |         PatchedItem.__enter__(self)
397 |         # Pass the pathname of the temporary directory to the caller.
398 |         return directory
399 | 
400 |     def __exit__(self, exc_type=None, exc_value=None, traceback=None):
401 |         """Deactivate the custom ``$HOME``."""
402 |         super(MockedHomeDirectory, self).__exit__(exc_type, exc_value, traceback)
403 | 
404 | 
405 | class CustomSearchPath(PatchedItem, TemporaryDirectory):
406 | 
407 |     """
408 |     Context manager to temporarily customize ``$PATH`` (the executable search path).
409 | 
410 |     This class is a composition of the :class:`PatchedItem` and
411 |     :class:`TemporaryDirectory` context managers.
412 |     """
413 | 
414 |     def __init__(self, isolated=False):
415 |         """
416 |         Initialize a :class:`CustomSearchPath` object.
417 | 
418 |         :param isolated: :data:`True` to clear the original search path,
419 |                          :data:`False` to add the temporary directory to the
420 |                          start of the search path.
421 |         """
422 |         # Initialize our own instance variables.
423 |         self.isolated_search_path = isolated
424 |         # Selectively initialize our superclasses.
425 |         PatchedItem.__init__(self, os.environ, 'PATH', self.current_search_path)
426 |         TemporaryDirectory.__init__(self)
427 | 
428 |     def __enter__(self):
429 |         """
430 |         Activate the custom ``$PATH``.
431 | 
432 |         :returns: The pathname of the directory that has
433 |                   been added to ``$PATH`` (a string).
434 |         """
435 |         # Get the temporary directory.
436 |         directory = TemporaryDirectory.__enter__(self)
437 |         # Override the value to patch now that we have
438 |         # the pathname of the temporary directory.
439 |         self.patched_value = (
440 |             directory if self.isolated_search_path
441 |             else os.pathsep.join([directory] + self.current_search_path.split(os.pathsep))
442 |         )
443 |         # Temporary patch the $PATH.
444 |         PatchedItem.__enter__(self)
445 |         # Pass the pathname of the temporary directory to the caller
446 |         # because they may want to `install' custom executables.
447 |         return directory
448 | 
449 |     def __exit__(self, exc_type=None, exc_value=None, traceback=None):
450 |         """Deactivate the custom ``$PATH``."""
451 |         super(CustomSearchPath, self).__exit__(exc_type, exc_value, traceback)
452 | 
453 |     @property
454 |     def current_search_path(self):
455 |         """The value of ``$PATH`` or :data:`os.defpath` (a string)."""
456 |         return os.environ.get('PATH', os.defpath)
457 | 
458 | 
459 | class MockedProgram(CustomSearchPath):
460 | 
461 |     """
462 |     Context manager to mock the existence of a program (executable).
463 | 
464 |     This class extends the functionality of :class:`CustomSearchPath`.
465 |     """
466 | 
467 |     def __init__(self, name, returncode=0):
468 |         """
469 |         Initialize a :class:`MockedProgram` object.
470 | 
471 |         :param name: The name of the program (a string).
472 |         :param returncode: The return code that the program should emit (a
473 |                            number, defaults to zero).
474 |         """
475 |         # Initialize our own instance variables.
476 |         self.program_name = name
477 |         self.program_returncode = returncode
478 |         self.program_signal_file = None
479 |         # Initialize our superclasses.
480 |         super(MockedProgram, self).__init__()
481 | 
482 |     def __enter__(self):
483 |         """
484 |         Create the mock program.
485 | 
486 |         :returns: The pathname of the directory that has
487 |                   been added to ``$PATH`` (a string).
488 |         """
489 |         directory = super(MockedProgram, self).__enter__()
490 |         self.program_signal_file = os.path.join(directory, 'program-was-run-%s' % random_string(10))
491 |         pathname = os.path.join(directory, self.program_name)
492 |         with open(pathname, 'w') as handle:
493 |             handle.write('#!/bin/sh\n')
494 |             handle.write('echo > %s\n' % pipes.quote(self.program_signal_file))
495 |             handle.write('exit %i\n' % self.program_returncode)
496 |         os.chmod(pathname, 0o755)
497 |         return directory
498 | 
499 |     def __exit__(self, *args, **kw):
500 |         """
501 |         Ensure that the mock program was run.
502 | 
503 |         :raises: :exc:`~exceptions.AssertionError` when
504 |                  the mock program hasn't been run.
505 |         """
506 |         try:
507 |             assert self.program_signal_file and os.path.isfile(self.program_signal_file), \
508 |                 ("It looks like %r was never run!" % self.program_name)
509 |         finally:
510 |             return super(MockedProgram, self).__exit__(*args, **kw)
511 | 
512 | 
513 | class CaptureOutput(ContextManager):
514 | 
515 |     """Context manager that captures what's written to :data:`sys.stdout` and :data:`sys.stderr`."""
516 | 
517 |     def __init__(self, merged=False, input=''):
518 |         """
519 |         Initialize a :class:`CaptureOutput` object.
520 | 
521 |         :param merged: :data:`True` to merge the streams,
522 |                        :data:`False` to capture them separately.
523 |         :param input: The data that reads from :data:`sys.stdin`
524 |                       should return (a string).
525 |         """
526 |         self.stdin = StringIO(input)
527 |         self.stdout = StringIO()
528 |         self.stderr = self.stdout if merged else StringIO()
529 |         self.patched_attributes = [
530 |             PatchedAttribute(sys, name, getattr(self, name))
531 |             for name in ('stdin', 'stdout', 'stderr')
532 |         ]
533 | 
534 |     stdin = None
535 |     """The :class:`~humanfriendly.compat.StringIO` object used to feed the standard input stream."""
536 | 
537 |     stdout = None
538 |     """The :class:`~humanfriendly.compat.StringIO` object used to capture the standard output stream."""
539 | 
540 |     stderr = None
541 |     """The :class:`~humanfriendly.compat.StringIO` object used to capture the standard error stream."""
542 | 
543 |     def __enter__(self):
544 |         """Start capturing what's written to :data:`sys.stdout` and :data:`sys.stderr`."""
545 |         super(CaptureOutput, self).__enter__()
546 |         for context in self.patched_attributes:
547 |             context.__enter__()
548 |         return self
549 | 
550 |     def __exit__(self, exc_type=None, exc_value=None, traceback=None):
551 |         """Stop capturing what's written to :data:`sys.stdout` and :data:`sys.stderr`."""
552 |         super(CaptureOutput, self).__exit__(exc_type, exc_value, traceback)
553 |         for context in self.patched_attributes:
554 |             context.__exit__(exc_type, exc_value, traceback)
555 | 
556 |     def getvalue(self):
557 |         """Get the text written to :data:`sys.stdout`."""
558 |         return self.stdout.getvalue()
559 | 
560 | 
561 | class TestCase(unittest.TestCase):
562 | 
563 |     """Subclass of :class:`unittest.TestCase` with automatic logging and other miscellaneous features."""
564 | 
565 |     exceptionsToSkip = []
566 |     """A list of exception types that are translated into skipped tests."""
567 | 
568 |     def __init__(self, *args, **kw):
569 |         """Wrap test methods using :func:`skipTestWrapper()`."""
570 |         # Initialize our superclass.
571 |         super(TestCase, self).__init__(*args, **kw)
572 |         # Wrap all of the test methods so that we can customize the
573 |         # skipping of tests based on the exceptions they raise.
574 |         for name in dir(self.__class__):
575 |             if name.startswith('test_'):
576 |                 setattr(self, name, functools.partial(
577 |                     self.skipTestWrapper,
578 |                     getattr(self, name),
579 |                 ))
580 | 
581 |     def assertRaises(self, exception, callable, *args, **kwds):
582 |         """
583 |         Replacement for :func:`unittest.TestCase.assertRaises()` that returns the exception.
584 | 
585 |         Refer to the :func:`unittest.TestCase.assertRaises()` documentation for
586 |         details on argument handling. The return value is the caught exception.
587 | 
588 |         .. warning:: This method does not support use as a context manager.
589 |         """
590 |         try:
591 |             callable(*args, **kwds)
592 |         except exception as e:
593 |             # Return the expected exception as a regular return value.
594 |             return e
595 |         else:
596 |             # Raise an exception when no exception was raised :-).
597 |             assert False, "Expected an exception to be raised!"
598 | 
599 |     def setUp(self, log_level=logging.DEBUG):
600 |         """setUp(log_level=logging.DEBUG)
601 |         Automatically configure logging to the terminal.
602 | 
603 |         :param log_level: Refer to :func:`configure_logging()`.
604 | 
605 |         The :func:`setUp()` method is automatically called by
606 |         :class:`unittest.TestCase` before each test method starts.
607 |         It does two things:
608 | 
609 |         - Logging to the terminal is configured using
610 |           :func:`configure_logging()`.
611 | 
612 |         - Before the test method starts a newline is emitted, to separate the
613 |           name of the test method (which will be printed to the terminal by
614 |           :mod:`unittest` and/or pytest_) from the first line of logging output
615 |           that the test method is likely going to generate.
616 |         """
617 |         # Configure logging to the terminal.
618 |         configure_logging(log_level)
619 |         # Separate the name of the test method (printed by the superclass
620 |         # and/or py.test without a newline at the end) from the first line of
621 |         # logging output that the test method is likely going to generate.
622 |         sys.stderr.write("\n")
623 | 
624 |     def shouldSkipTest(self, exception):
625 |         """
626 |         Decide whether a test that raised an exception should be skipped.
627 | 
628 |         :param exception: The exception that was raised by the test.
629 |         :returns: :data:`True` to translate the exception into a skipped test,
630 |                   :data:`False` to propagate the exception as usual.
631 | 
632 |         The :func:`shouldSkipTest()` method skips exceptions listed in the
633 |         :attr:`exceptionsToSkip` attribute. This enables subclasses of
634 |         :class:`TestCase` to customize the default behavior with a one liner.
635 |         """
636 |         return isinstance(exception, tuple(self.exceptionsToSkip))
637 | 
638 |     def skipTest(self, text, *args, **kw):
639 |         """
640 |         Enable skipping of tests.
641 | 
642 |         This method was added in humanfriendly 3.3 as a fall back for the
643 |         :func:`unittest.TestCase.skipTest()` method that was added in Python
644 |         2.7 and 3.1 (because humanfriendly also supports Python 2.6).
645 | 
646 |         Since then `humanfriendly` has gained a conditional dependency on
647 |         unittest2_ which enables actual skipping of tests (instead of just
648 |         mocking it) on Python 2.6.
649 | 
650 |         This method now remains for backwards compatibility (and just because
651 |         it's a nice shortcut).
652 | 
653 |         .. _unittest2: https://pypi.python.org/pypi/unittest2
654 |         """
655 |         raise unittest.SkipTest(compact(text, *args, **kw))
656 | 
657 |     def skipTestWrapper(self, test_method, *args, **kw):
658 |         """
659 |         Wrap test methods to translate exceptions into skipped tests.
660 | 
661 |         :param test_method: The test method to wrap.
662 |         :param args: The positional arguments to the test method.
663 |         :param kw: The keyword arguments to the test method.
664 |         :returns: The return value of the test method.
665 | 
666 |         When a :class:`TestCase` object is initialized, :func:`__init__()`
667 |         wraps all of the ``test_*`` methods with :func:`skipTestWrapper()`.
668 | 
669 |         When a test method raises an exception, :func:`skipTestWrapper()` will
670 |         catch the exception and call :func:`shouldSkipTest()` to decide whether
671 |         to translate the exception into a skipped test.
672 | 
673 |         When :func:`shouldSkipTest()` returns :data:`True` the exception is
674 |         swallowed and :exc:`unittest.SkipTest` is raised instead of the
675 |         original exception.
676 |         """
677 |         try:
678 |             return test_method(*args, **kw)
679 |         except BaseException as e:
680 |             if self.shouldSkipTest(e):
681 |                 if isinstance(e, unittest.SkipTest):
682 |                     # We prefer to preserve the original
683 |                     # exception and stack trace.
684 |                     raise
685 |                 else:
686 |                     # If the original exception wasn't a unittest.SkipTest
687 |                     # exception then we will translate it into one.
688 |                     raise unittest.SkipTest(unicode(e))
689 |             else:
690 |                 raise
691 | 


--------------------------------------------------------------------------------
/humanfriendly/text.py:
--------------------------------------------------------------------------------
  1 | # Human friendly input/output in Python.
  2 | #
  3 | # Author: Peter Odding <peter@peterodding.com>
  4 | # Last Change: February 21, 2019
  5 | # URL: https://humanfriendly.readthedocs.io
  6 | 
  7 | """
  8 | Simple text manipulation functions.
  9 | 
 10 | The :mod:`~humanfriendly.text` module contains simple functions to manipulate text:
 11 | 
 12 | - The :func:`concatenate()` and :func:`pluralize()` functions make it easy to
 13 |   generate human friendly output.
 14 | 
 15 | - The :func:`format()`, :func:`compact()` and :func:`dedent()` functions
 16 |   provide a clean and simple to use syntax for composing large text fragments
 17 |   with interpolated variables.
 18 | 
 19 | - The :func:`tokenize()` function parses simple user input.
 20 | """
 21 | 
 22 | # Standard library modules.
 23 | import math
 24 | import numbers
 25 | import random
 26 | import re
 27 | import string
 28 | import textwrap
 29 | 
 30 | # Public identifiers that require documentation.
 31 | __all__ = (
 32 |     'compact',
 33 |     'compact_empty_lines',
 34 |     'concatenate',
 35 |     'dedent',
 36 |     'format',
 37 |     'is_empty_line',
 38 |     'join_lines',
 39 |     'pluralize',
 40 |     'random_string',
 41 |     'split',
 42 |     'split_paragraphs',
 43 |     'tokenize',
 44 |     'trim_empty_lines',
 45 | )
 46 | 
 47 | 
 48 | def compact(text, *args, **kw):
 49 |     '''
 50 |     Compact whitespace in a string.
 51 | 
 52 |     Trims leading and trailing whitespace, replaces runs of whitespace
 53 |     characters with a single space and interpolates any arguments using
 54 |     :func:`format()`.
 55 | 
 56 |     :param text: The text to compact (a string).
 57 |     :param args: Any positional arguments are interpolated using :func:`format()`.
 58 |     :param kw: Any keyword arguments are interpolated using :func:`format()`.
 59 |     :returns: The compacted text (a string).
 60 | 
 61 |     Here's an example of how I like to use the :func:`compact()` function, this
 62 |     is an example from a random unrelated project I'm working on at the moment::
 63 | 
 64 |         raise PortDiscoveryError(compact("""
 65 |             Failed to discover port(s) that Apache is listening on!
 66 |             Maybe I'm parsing the wrong configuration file? ({filename})
 67 |         """, filename=self.ports_config))
 68 | 
 69 |     The combination of :func:`compact()` and Python's multi line strings allows
 70 |     me to write long text fragments with interpolated variables that are easy
 71 |     to write, easy to read and work well with Python's whitespace
 72 |     sensitivity.
 73 |     '''
 74 |     non_whitespace_tokens = text.split()
 75 |     compacted_text = ' '.join(non_whitespace_tokens)
 76 |     return format(compacted_text, *args, **kw)
 77 | 
 78 | 
 79 | def compact_empty_lines(text):
 80 |     """
 81 |     Replace repeating empty lines with a single empty line (similar to ``cat -s``).
 82 | 
 83 |     :param text: The text in which to compact empty lines (a string).
 84 |     :returns: The text with empty lines compacted (a string).
 85 |     """
 86 |     i = 0
 87 |     lines = text.splitlines(True)
 88 |     while i < len(lines):
 89 |         if i > 0 and is_empty_line(lines[i - 1]) and is_empty_line(lines[i]):
 90 |             lines.pop(i)
 91 |         else:
 92 |             i += 1
 93 |     return ''.join(lines)
 94 | 
 95 | 
 96 | def concatenate(items):
 97 |     """
 98 |     Concatenate a list of items in a human friendly way.
 99 | 
100 |     :param items: A sequence of strings.
101 |     :returns: A single string.
102 | 
103 |     >>> from humanfriendly.text import concatenate
104 |     >>> concatenate(["eggs", "milk", "bread"])
105 |     'eggs, milk and bread'
106 |     """
107 |     items = list(items)
108 |     if len(items) > 1:
109 |         return ', '.join(items[:-1]) + ' and ' + items[-1]
110 |     elif items:
111 |         return items[0]
112 |     else:
113 |         return ''
114 | 
115 | 
116 | def dedent(text, *args, **kw):
117 |     """
118 |     Dedent a string (remove common leading whitespace from all lines).
119 | 
120 |     Removes common leading whitespace from all lines in the string using
121 |     :func:`textwrap.dedent()`, removes leading and trailing empty lines using
122 |     :func:`trim_empty_lines()` and interpolates any arguments using
123 |     :func:`format()`.
124 | 
125 |     :param text: The text to dedent (a string).
126 |     :param args: Any positional arguments are interpolated using :func:`format()`.
127 |     :param kw: Any keyword arguments are interpolated using :func:`format()`.
128 |     :returns: The dedented text (a string).
129 | 
130 |     The :func:`compact()` function's documentation contains an example of how I
131 |     like to use the :func:`compact()` and :func:`dedent()` functions. The main
132 |     difference is that I use :func:`compact()` for text that will be presented
133 |     to the user (where whitespace is not so significant) and :func:`dedent()`
134 |     for data file and code generation tasks (where newlines and indentation are
135 |     very significant).
136 |     """
137 |     dedented_text = textwrap.dedent(text)
138 |     trimmed_text = trim_empty_lines(dedented_text)
139 |     return format(trimmed_text, *args, **kw)
140 | 
141 | 
142 | def format(text, *args, **kw):
143 |     """
144 |     Format a string using the string formatting operator and/or :func:`str.format()`.
145 | 
146 |     :param text: The text to format (a string).
147 |     :param args: Any positional arguments are interpolated into the text using
148 |                  the string formatting operator (``%``). If no positional
149 |                  arguments are given no interpolation is done.
150 |     :param kw: Any keyword arguments are interpolated into the text using the
151 |                :func:`str.format()` function. If no keyword arguments are given
152 |                no interpolation is done.
153 |     :returns: The text with any positional and/or keyword arguments
154 |               interpolated (a string).
155 | 
156 |     The implementation of this function is so trivial that it seems silly to
157 |     even bother writing and documenting it. Justifying this requires some
158 |     context :-).
159 | 
160 |     **Why format() instead of the string formatting operator?**
161 | 
162 |     For really simple string interpolation Python's string formatting operator
163 |     is ideal, but it does have some strange quirks:
164 | 
165 |     - When you switch from interpolating a single value to interpolating
166 |       multiple values you have to wrap them in tuple syntax. Because
167 |       :func:`format()` takes a `variable number of arguments`_ it always
168 |       receives a tuple (which saves me a context switch :-). Here's an
169 |       example:
170 | 
171 |       >>> from humanfriendly.text import format
172 |       >>> # The string formatting operator.
173 |       >>> print('the magic number is %s' % 42)
174 |       the magic number is 42
175 |       >>> print('the magic numbers are %s and %s' % (12, 42))
176 |       the magic numbers are 12 and 42
177 |       >>> # The format() function.
178 |       >>> print(format('the magic number is %s', 42))
179 |       the magic number is 42
180 |       >>> print(format('the magic numbers are %s and %s', 12, 42))
181 |       the magic numbers are 12 and 42
182 | 
183 |     - When you interpolate a single value and someone accidentally passes in a
184 |       tuple your code raises a :exc:`~exceptions.TypeError`. Because
185 |       :func:`format()` takes a `variable number of arguments`_ it always
186 |       receives a tuple so this can never happen. Here's an example:
187 | 
188 |       >>> # How expecting to interpolate a single value can fail.
189 |       >>> value = (12, 42)
190 |       >>> print('the magic value is %s' % value)
191 |       Traceback (most recent call last):
192 |         File "<stdin>", line 1, in <module>
193 |       TypeError: not all arguments converted during string formatting
194 |       >>> # The following line works as intended, no surprises here!
195 |       >>> print(format('the magic value is %s', value))
196 |       the magic value is (12, 42)
197 | 
198 |     **Why format() instead of the str.format() method?**
199 | 
200 |     When you're doing complex string interpolation the :func:`str.format()`
201 |     function results in more readable code, however I frequently find myself
202 |     adding parentheses to force evaluation order. The :func:`format()` function
203 |     avoids this because of the relative priority between the comma and dot
204 |     operators. Here's an example:
205 | 
206 |     >>> "{adjective} example" + " " + "(can't think of anything less {adjective})".format(adjective='silly')
207 |     "{adjective} example (can't think of anything less silly)"
208 |     >>> ("{adjective} example" + " " + "(can't think of anything less {adjective})").format(adjective='silly')
209 |     "silly example (can't think of anything less silly)"
210 |     >>> format("{adjective} example" + " " + "(can't think of anything less {adjective})", adjective='silly')
211 |     "silly example (can't think of anything less silly)"
212 | 
213 |     The :func:`compact()` and :func:`dedent()` functions are wrappers that
214 |     combine :func:`format()` with whitespace manipulation to make it easy to
215 |     write nice to read Python code.
216 | 
217 |     .. _variable number of arguments: https://docs.python.org/2/tutorial/controlflow.html#arbitrary-argument-lists
218 |     """
219 |     if args:
220 |         text %= args
221 |     if kw:
222 |         text = text.format(**kw)
223 |     return text
224 | 
225 | 
226 | def generate_slug(text, delimiter="-"):
227 |     """
228 |     Convert text to a normalized "slug" without whitespace.
229 | 
230 |     :param text: The original text, for example ``Some Random Text!``.
231 |     :param delimiter: The delimiter used to separate words
232 |                       (defaults to the ``-`` character).
233 |     :returns: The slug text, for example ``some-random-text``.
234 |     :raises: :exc:`~exceptions.ValueError` when the provided
235 |              text is nonempty but results in an empty slug.
236 |     """
237 |     slug = text.lower()
238 |     escaped = delimiter.replace("\\", "\\\\")
239 |     slug = re.sub("[^a-z0-9]+", escaped, slug)
240 |     slug = slug.strip(delimiter)
241 |     if text and not slug:
242 |         msg = "The provided text %r results in an empty slug!"
243 |         raise ValueError(format(msg, text))
244 |     return slug
245 | 
246 | 
247 | def is_empty_line(text):
248 |     """
249 |     Check if a text is empty or contains only whitespace.
250 | 
251 |     :param text: The text to check for "emptiness" (a string).
252 |     :returns: :data:`True` if the text is empty or contains only whitespace,
253 |               :data:`False` otherwise.
254 |     """
255 |     return len(text) == 0 or text.isspace()
256 | 
257 | 
258 | def join_lines(text):
259 |     """
260 |     Remove "hard wrapping" from the paragraphs in a string.
261 | 
262 |     :param text: The text to reformat (a string).
263 |     :returns: The text without hard wrapping (a string).
264 | 
265 |     This function works by removing line breaks when the last character before
266 |     a line break and the first character after the line break are both
267 |     non-whitespace characters. This means that common leading indentation will
268 |     break :func:`join_lines()` (in that case you can use :func:`dedent()`
269 |     before calling :func:`join_lines()`).
270 |     """
271 |     return re.sub(r'(\S)\n(\S)', r'\1 \2', text)
272 | 
273 | 
274 | def pluralize(count, singular, plural=None):
275 |     """
276 |     Combine a count with the singular or plural form of a word.
277 | 
278 |     If the plural form of the word is not provided it is obtained by
279 |     concatenating the singular form of the word with the letter "s". Of course
280 |     this will not always be correct, which is why you have the option to
281 |     specify both forms.
282 | 
283 |     :param count: The count (a number).
284 |     :param singular: The singular form of the word (a string).
285 |     :param plural: The plural form of the word (a string or :data:`None`).
286 |     :returns: The count and singular/plural word concatenated (a string).
287 |     """
288 |     if not plural:
289 |         plural = singular + 's'
290 |     return '%s %s' % (count, singular if math.floor(float(count)) == 1 else plural)
291 | 
292 | 
293 | def random_string(length=(25, 100), characters=string.ascii_letters):
294 |     """random_string(length=(25, 100), characters=string.ascii_letters)
295 |     Generate a random string.
296 | 
297 |     :param length: The length of the string to be generated (a number or a
298 |                    tuple with two numbers). If this is a tuple then a random
299 |                    number between the two numbers given in the tuple is used.
300 |     :param characters: The characters to be used (a string, defaults
301 |                        to :data:`string.ascii_letters`).
302 |     :returns: A random string.
303 | 
304 |     The :func:`random_string()` function is very useful in test suites; by the
305 |     time I included it in :mod:`humanfriendly.text` I had already included
306 |     variants of this function in seven different test suites :-).
307 |     """
308 |     if not isinstance(length, numbers.Number):
309 |         length = random.randint(length[0], length[1])
310 |     return ''.join(random.choice(characters) for _ in range(length))
311 | 
312 | 
313 | def split(text, delimiter=','):
314 |     """
315 |     Split a comma-separated list of strings.
316 | 
317 |     :param text: The text to split (a string).
318 |     :param delimiter: The delimiter to split on (a string).
319 |     :returns: A list of zero or more nonempty strings.
320 | 
321 |     Here's the default behavior of Python's built in :func:`str.split()`
322 |     function:
323 | 
324 |     >>> 'foo,bar, baz,'.split(',')
325 |     ['foo', 'bar', ' baz', '']
326 | 
327 |     In contrast here's the default behavior of the :func:`split()` function:
328 | 
329 |     >>> from humanfriendly.text import split
330 |     >>> split('foo,bar, baz,')
331 |     ['foo', 'bar', 'baz']
332 | 
333 |     Here is an example that parses a nested data structure (a mapping of
334 |     logging level names to one or more styles per level) that's encoded in a
335 |     string so it can be set as an environment variable:
336 | 
337 |     >>> from pprint import pprint
338 |     >>> encoded_data = 'debug=green;warning=yellow;error=red;critical=red,bold'
339 |     >>> parsed_data = dict((k, split(v, ',')) for k, v in (split(kv, '=') for kv in split(encoded_data, ';')))
340 |     >>> pprint(parsed_data)
341 |     {'debug': ['green'],
342 |      'warning': ['yellow'],
343 |      'error': ['red'],
344 |      'critical': ['red', 'bold']}
345 |     """
346 |     return [token.strip() for token in text.split(delimiter) if token and not token.isspace()]
347 | 
348 | 
349 | def split_paragraphs(text):
350 |     """
351 |     Split a string into paragraphs (one or more lines delimited by an empty line).
352 | 
353 |     :param text: The text to split into paragraphs (a string).
354 |     :returns: A list of strings.
355 |     """
356 |     paragraphs = []
357 |     for chunk in text.split('\n\n'):
358 |         chunk = trim_empty_lines(chunk)
359 |         if chunk and not chunk.isspace():
360 |             paragraphs.append(chunk)
361 |     return paragraphs
362 | 
363 | 
364 | def tokenize(text):
365 |     """
366 |     Tokenize a text into numbers and strings.
367 | 
368 |     :param text: The text to tokenize (a string).
369 |     :returns: A list of strings and/or numbers.
370 | 
371 |     This function is used to implement robust tokenization of user input in
372 |     functions like :func:`.parse_size()` and :func:`.parse_timespan()`. It
373 |     automatically coerces integer and floating point numbers, ignores
374 |     whitespace and knows how to separate numbers from strings even without
375 |     whitespace. Some examples to make this more concrete:
376 | 
377 |     >>> from humanfriendly.text import tokenize
378 |     >>> tokenize('42')
379 |     [42]
380 |     >>> tokenize('42MB')
381 |     [42, 'MB']
382 |     >>> tokenize('42.5MB')
383 |     [42.5, 'MB']
384 |     >>> tokenize('42.5 MB')
385 |     [42.5, 'MB']
386 |     """
387 |     tokenized_input = []
388 |     for token in re.split(r'(\d+(?:\.\d+)?)', text):
389 |         token = token.strip()
390 |         if re.match(r'\d+\.\d+', token):
391 |             tokenized_input.append(float(token))
392 |         elif token.isdigit():
393 |             tokenized_input.append(int(token))
394 |         elif token:
395 |             tokenized_input.append(token)
396 |     return tokenized_input
397 | 
398 | 
399 | def trim_empty_lines(text):
400 |     """
401 |     Trim leading and trailing empty lines from the given text.
402 | 
403 |     :param text: The text to trim (a string).
404 |     :returns: The trimmed text (a string).
405 |     """
406 |     lines = text.splitlines(True)
407 |     while lines and is_empty_line(lines[0]):
408 |         lines.pop(0)
409 |     while lines and is_empty_line(lines[-1]):
410 |         lines.pop(-1)
411 |     return ''.join(lines)
412 | 


--------------------------------------------------------------------------------
/humanfriendly/usage.py:
--------------------------------------------------------------------------------
  1 | # Human friendly input/output in Python.
  2 | #
  3 | # Author: Peter Odding <peter@peterodding.com>
  4 | # Last Change: June 24, 2017
  5 | # URL: https://humanfriendly.readthedocs.io
  6 | 
  7 | """
  8 | Parsing and reformatting of usage messages.
  9 | 
 10 | The :mod:`~humanfriendly.usage` module parses and reformats usage messages:
 11 | 
 12 | - The :func:`format_usage()` function takes a usage message and inserts ANSI
 13 |   escape sequences that highlight items of special significance like command
 14 |   line options, meta variables, etc. The resulting usage message is (intended
 15 |   to be) easier to read on a terminal.
 16 | 
 17 | - The :func:`render_usage()` function takes a usage message and rewrites it to
 18 |   reStructuredText_ suitable for inclusion in the documentation of a Python
 19 |   package. This provides a DRY solution to keeping a single authoritative
 20 |   definition of the usage message while making it easily available in
 21 |   documentation. As a cherry on the cake it's not just a pre-formatted dump of
 22 |   the usage message but a nicely formatted reStructuredText_ fragment.
 23 | 
 24 | - The remaining functions in this module support the two functions above.
 25 | 
 26 | Usage messages in general are free format of course, however the functions in
 27 | this module assume a certain structure from usage messages in order to
 28 | successfully parse and reformat them, refer to :func:`parse_usage()` for
 29 | details.
 30 | 
 31 | .. _DRY: https://en.wikipedia.org/wiki/Don%27t_repeat_yourself
 32 | .. _reStructuredText: https://en.wikipedia.org/wiki/ReStructuredText
 33 | """
 34 | 
 35 | # Standard library modules.
 36 | import csv
 37 | import functools
 38 | import logging
 39 | import re
 40 | 
 41 | # Standard library module or external dependency (see setup.py).
 42 | from importlib import import_module
 43 | 
 44 | # Modules included in our package.
 45 | from humanfriendly.compat import StringIO
 46 | from humanfriendly.text import dedent, split_paragraphs, trim_empty_lines
 47 | 
 48 | # Public identifiers that require documentation.
 49 | __all__ = (
 50 |     'find_meta_variables',
 51 |     'format_usage',
 52 |     'import_module',  # previously exported (backwards compatibility)
 53 |     'inject_usage',
 54 |     'parse_usage',
 55 |     'render_usage',
 56 |     'USAGE_MARKER',
 57 | )
 58 | 
 59 | USAGE_MARKER = "Usage:"
 60 | """The string that starts the first line of a usage message."""
 61 | 
 62 | START_OF_OPTIONS_MARKER = "Supported options:"
 63 | """The string that marks the start of the documented command line options."""
 64 | 
 65 | # Compiled regular expression used to tokenize usage messages.
 66 | USAGE_PATTERN = re.compile(r'''
 67 |     # Make sure whatever we're matching isn't preceded by a non-whitespace
 68 |     # character.
 69 |     (?<!\S)
 70 |     (
 71 |         # A short command line option or a long command line option
 72 |         # (possibly including a meta variable for a value).
 73 |         (-\w|--\w+(-\w+)*(=\S+)?)
 74 |         # Or ...
 75 |         |
 76 |         # An environment variable.
 77 |         \$[A-Za-z_][A-Za-z0-9_]*
 78 |         # Or ...
 79 |         |
 80 |         # Might be a meta variable (usage() will figure it out).
 81 |         [A-Z][A-Z0-9_]+
 82 |     )
 83 | ''', re.VERBOSE)
 84 | 
 85 | # Compiled regular expression used to recognize options.
 86 | OPTION_PATTERN = re.compile(r'^(-\w|--\w+(-\w+)*(=\S+)?)$')
 87 | 
 88 | # Initialize a logger for this module.
 89 | logger = logging.getLogger(__name__)
 90 | 
 91 | 
 92 | def format_usage(usage_text):
 93 |     """
 94 |     Highlight special items in a usage message.
 95 | 
 96 |     :param usage_text: The usage message to process (a string).
 97 |     :returns: The usage message with special items highlighted.
 98 | 
 99 |     This function highlights the following special items:
100 | 
101 |     - The initial line of the form "Usage: ..."
102 |     - Short and long command line options
103 |     - Environment variables
104 |     - Meta variables (see :func:`find_meta_variables()`)
105 | 
106 |     All items are highlighted in the color defined by
107 |     :data:`.HIGHLIGHT_COLOR`.
108 |     """
109 |     # Ugly workaround to avoid circular import errors due to interdependencies
110 |     # between the humanfriendly.terminal and humanfriendly.usage modules.
111 |     from humanfriendly.terminal import ansi_wrap, HIGHLIGHT_COLOR
112 |     formatted_lines = []
113 |     meta_variables = find_meta_variables(usage_text)
114 |     for line in usage_text.strip().splitlines(True):
115 |         if line.startswith(USAGE_MARKER):
116 |             # Highlight the "Usage: ..." line in bold font and color.
117 |             formatted_lines.append(ansi_wrap(line, color=HIGHLIGHT_COLOR))
118 |         else:
119 |             # Highlight options, meta variables and environment variables.
120 |             formatted_lines.append(replace_special_tokens(
121 |                 line, meta_variables,
122 |                 lambda token: ansi_wrap(token, color=HIGHLIGHT_COLOR),
123 |             ))
124 |     return ''.join(formatted_lines)
125 | 
126 | 
127 | def find_meta_variables(usage_text):
128 |     """
129 |     Find the meta variables in the given usage message.
130 | 
131 |     :param usage_text: The usage message to parse (a string).
132 |     :returns: A list of strings with any meta variables found in the usage
133 |               message.
134 | 
135 |     When a command line option requires an argument, the convention is to
136 |     format such options as ``--option=ARG``. The text ``ARG`` in this example
137 |     is the meta variable.
138 |     """
139 |     meta_variables = set()
140 |     for match in USAGE_PATTERN.finditer(usage_text):
141 |         token = match.group(0)
142 |         if token.startswith('-'):
143 |             option, _, value = token.partition('=')
144 |             if value:
145 |                 meta_variables.add(value)
146 |     return list(meta_variables)
147 | 
148 | 
149 | def parse_usage(text):
150 |     """
151 |     Parse a usage message by inferring its structure (and making some assumptions :-).
152 | 
153 |     :param text: The usage message to parse (a string).
154 |     :returns: A tuple of two lists:
155 | 
156 |               1. A list of strings with the paragraphs of the usage message's
157 |                  "introduction" (the paragraphs before the documentation of the
158 |                  supported command line options).
159 | 
160 |               2. A list of strings with pairs of command line options and their
161 |                  descriptions: Item zero is a line listing a supported command
162 |                  line option, item one is the description of that command line
163 |                  option, item two is a line listing another supported command
164 |                  line option, etc.
165 | 
166 |     Usage messages in general are free format of course, however
167 |     :func:`parse_usage()` assume a certain structure from usage messages in
168 |     order to successfully parse them:
169 | 
170 |     - The usage message starts with a line ``Usage: ...`` that shows a symbolic
171 |       representation of the way the program is to be invoked.
172 | 
173 |     - After some free form text a line ``Supported options:`` (surrounded by
174 |       empty lines) precedes the documentation of the supported command line
175 |       options.
176 | 
177 |     - The command line options are documented as follows::
178 | 
179 |         -v, --verbose
180 | 
181 |           Make more noise.
182 | 
183 |       So all of the variants of the command line option are shown together on a
184 |       separate line, followed by one or more paragraphs describing the option.
185 | 
186 |     - There are several other minor assumptions, but to be honest I'm not sure if
187 |       anyone other than me is ever going to use this functionality, so for now I
188 |       won't list every intricate detail :-).
189 | 
190 |       If you're curious anyway, refer to the usage message of the `humanfriendly`
191 |       package (defined in the :mod:`humanfriendly.cli` module) and compare it with
192 |       the usage message you see when you run ``humanfriendly --help`` and the
193 |       generated usage message embedded in the readme.
194 | 
195 |       Feel free to request more detailed documentation if you're interested in
196 |       using the :mod:`humanfriendly.usage` module outside of the little ecosystem
197 |       of Python packages that I have been building over the past years.
198 |     """
199 |     introduction = []
200 |     documented_options = []
201 |     # Split the raw usage message into paragraphs.
202 |     paragraphs = split_paragraphs(text)
203 |     # Get the paragraphs that are part of the introduction.
204 |     while paragraphs:
205 |         # Check whether we've found the end of the introduction.
206 |         end_of_intro = (paragraphs[0] == START_OF_OPTIONS_MARKER)
207 |         # Append the current paragraph to the introduction.
208 |         introduction.append(paragraphs.pop(0))
209 |         # Stop after we've processed the complete introduction.
210 |         if end_of_intro:
211 |             break
212 |     logger.debug("Parsed introduction: %s", introduction)
213 |     # Parse the paragraphs that document command line options.
214 |     while paragraphs:
215 |         documented_options.append(dedent(paragraphs.pop(0)))
216 |         description = []
217 |         while paragraphs:
218 |             # Check if the next paragraph starts the documentation of another
219 |             # command line option. We split on a comma followed by a space so
220 |             # that our parsing doesn't trip up when the label used for an
221 |             # option's value contains commas.
222 |             tokens = [t.strip() for t in re.split(r',\s', paragraphs[0]) if t and not t.isspace()]
223 |             if all(OPTION_PATTERN.match(t) for t in tokens):
224 |                 break
225 |             else:
226 |                 description.append(paragraphs.pop(0))
227 |         # Join the description's paragraphs back together so we can remove
228 |         # common leading indentation.
229 |         documented_options.append(dedent('\n\n'.join(description)))
230 |     logger.debug("Parsed options: %s", documented_options)
231 |     return introduction, documented_options
232 | 
233 | 
234 | def render_usage(text):
235 |     """
236 |     Reformat a command line program's usage message to reStructuredText_.
237 | 
238 |     :param text: The plain text usage message (a string).
239 |     :returns: The usage message rendered to reStructuredText_ (a string).
240 |     """
241 |     meta_variables = find_meta_variables(text)
242 |     introduction, options = parse_usage(text)
243 |     output = [render_paragraph(p, meta_variables) for p in introduction]
244 |     if options:
245 |         output.append('\n'.join([
246 |             '.. csv-table::',
247 |             '   :header: Option, Description',
248 |             '   :widths: 30, 70',
249 |             '',
250 |         ]))
251 |         csv_buffer = StringIO()
252 |         csv_writer = csv.writer(csv_buffer)
253 |         while options:
254 |             variants = options.pop(0)
255 |             description = options.pop(0)
256 |             csv_writer.writerow([
257 |                 render_paragraph(variants, meta_variables),
258 |                 ('\n\n'.join(render_paragraph(p, meta_variables) for p in split_paragraphs(description))).rstrip(),
259 |             ])
260 |         csv_lines = csv_buffer.getvalue().splitlines()
261 |         output.append('\n'.join('   %s' % l for l in csv_lines))
262 |     logger.debug("Rendered output: %s", output)
263 |     return '\n\n'.join(trim_empty_lines(o) for o in output)
264 | 
265 | 
266 | def inject_usage(module_name):
267 |     """
268 |     Use cog_ to inject a usage message into a reStructuredText_ file.
269 | 
270 |     :param module_name: The name of the module whose ``__doc__`` attribute is
271 |                         the source of the usage message (a string).
272 | 
273 |     This simple wrapper around :func:`render_usage()` makes it very easy to
274 |     inject a reformatted usage message into your documentation using cog_. To
275 |     use it you add a fragment like the following to your ``*.rst`` file::
276 | 
277 |        .. [[[cog
278 |        .. from humanfriendly.usage import inject_usage
279 |        .. inject_usage('humanfriendly.cli')
280 |        .. ]]]
281 |        .. [[[end]]]
282 | 
283 |     The lines in the fragment above are single line reStructuredText_ comments
284 |     that are not copied to the output. Their purpose is to instruct cog_ where
285 |     to inject the reformatted usage message. Once you've added these lines to
286 |     your ``*.rst`` file, updating the rendered usage message becomes really
287 |     simple thanks to cog_:
288 | 
289 |     .. code-block:: sh
290 | 
291 |        $ cog.py -r README.rst
292 | 
293 |     This will inject or replace the rendered usage message in your
294 |     ``README.rst`` file with an up to date copy.
295 | 
296 |     .. _cog: http://nedbatchelder.com/code/cog/
297 |     """
298 |     import cog
299 |     usage_text = import_module(module_name).__doc__
300 |     cog.out("\n" + render_usage(usage_text) + "\n\n")
301 | 
302 | 
303 | def render_paragraph(paragraph, meta_variables):
304 |     # Reformat the "Usage:" line to highlight "Usage:" in bold and show the
305 |     # remainder of the line as pre-formatted text.
306 |     if paragraph.startswith(USAGE_MARKER):
307 |         tokens = paragraph.split()
308 |         return "**%s** `%s`" % (tokens[0], ' '.join(tokens[1:]))
309 |     # Reformat the "Supported options:" line to highlight it in bold.
310 |     if paragraph == 'Supported options:':
311 |         return "**%s**" % paragraph
312 |     # Reformat shell transcripts into code blocks.
313 |     if re.match(r'^\s*\$\s+\S', paragraph):
314 |         # Split the paragraph into lines.
315 |         lines = paragraph.splitlines()
316 |         # Check if the paragraph is already indented.
317 |         if not paragraph[0].isspace():
318 |             # If the paragraph isn't already indented we'll indent it now.
319 |             lines = ['  %s' % line for line in lines]
320 |         lines.insert(0, '.. code-block:: sh')
321 |         lines.insert(1, '')
322 |         return "\n".join(lines)
323 |     # The following reformatting applies only to paragraphs which are not
324 |     # indented. Yes this is a hack - for now we assume that indented paragraphs
325 |     # are code blocks, even though this assumption can be wrong.
326 |     if not paragraph[0].isspace():
327 |         # Change UNIX style `quoting' so it doesn't trip up DocUtils.
328 |         paragraph = re.sub("`(.+?)'", r'"\1"', paragraph)
329 |         # Escape asterisks.
330 |         paragraph = paragraph.replace('*', r'\*')
331 |         # Reformat inline tokens.
332 |         paragraph = replace_special_tokens(
333 |             paragraph, meta_variables,
334 |             lambda token: '``%s``' % token,
335 |         )
336 |     return paragraph
337 | 
338 | 
339 | def replace_special_tokens(text, meta_variables, replace_fn):
340 |     return USAGE_PATTERN.sub(functools.partial(
341 |         replace_tokens_callback,
342 |         meta_variables=meta_variables,
343 |         replace_fn=replace_fn
344 |     ), text)
345 | 
346 | 
347 | def replace_tokens_callback(match, meta_variables, replace_fn):
348 |     token = match.group(0)
349 |     if not (re.match('^[A-Z][A-Z0-9_]+$', token) and token not in meta_variables):
350 |         token = replace_fn(token)
351 |     return token
352 | 


--------------------------------------------------------------------------------