├── .gitignore ├── LICENSE ├── README.md ├── bin ├── markpress ├── markpress.cmd └── markpress.py ├── images ├── circo.png ├── digraph.png ├── math1.gif └── math2.gif ├── lib ├── ascmini.py ├── config.py ├── markdown2.py ├── nextpress.py ├── render.py ├── utils.py ├── utime.py └── wordpress2.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # config 10 | *.log 11 | *.ini 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | MANIFEST 31 | 32 | # PyInstaller 33 | # Usually these files are written by a python script from a template 34 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 35 | *.manifest 36 | *.spec 37 | 38 | # Installer logs 39 | pip-log.txt 40 | pip-delete-this-directory.txt 41 | 42 | # Unit test / coverage reports 43 | htmlcov/ 44 | .tox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | 63 | # Flask stuff: 64 | instance/ 65 | .webassets-cache 66 | 67 | # Scrapy stuff: 68 | .scrapy 69 | 70 | # Sphinx documentation 71 | docs/_build/ 72 | 73 | # PyBuilder 74 | target/ 75 | 76 | # Jupyter Notebook 77 | .ipynb_checkpoints 78 | 79 | # pyenv 80 | .python-version 81 | 82 | # celery beat schedule file 83 | celerybeat-schedule 84 | 85 | # SageMath parsed files 86 | *.sage.py 87 | 88 | # Environments 89 | .env 90 | .venv 91 | env/ 92 | venv/ 93 | ENV/ 94 | env.bak/ 95 | venv.bak/ 96 | 97 | # Spyder project settings 98 | .spyderproject 99 | .spyproject 100 | 101 | # Rope project settings 102 | .ropeproject 103 | 104 | # mkdocs documentation 105 | /site 106 | 107 | # mypy 108 | .mypy_cache/ 109 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Linwei 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Preface 2 | 3 | MarkPress is a command line tool to create and publish markdown post for WordPress. 4 | 5 | ## Features 6 | 7 | - Syntax highlighting for fenced code block. 8 | - Meta header in markdown to describe title / categories / tags. 9 | - Supports GraphViz scripts to render graphics. 10 | - Supports Proxy (HTTP/SOCKS4/SOCKS5). 11 | 12 | ## Installation 13 | 14 | Clone the repository to some where you like: 15 | 16 | ```bash 17 | git clone https://github.com/skywind3000/markpress.git ~/.local/app/markpress 18 | ``` 19 | 20 | Add `bin` folder in your `$PATH`, put the line below in your `.bashrc` / `.zshrc`: 21 | 22 | ```bash 23 | export PATH="~/.local/app/markpress/bin:$PATH" 24 | ``` 25 | 26 | If you don't want to modify `$PATH`, you can create a symbol link for `markpress/bin/markpress` and put it in somewhere within your `$PATH`. 27 | 28 | Install requirements: 29 | 30 | ```bash 31 | sudo pip install python-wordpress-xmlrpc markdown beautifulsoup4 PySocks 32 | ``` 33 | 34 | Now, command `markpress` is ready to work. 35 | 36 | 37 | ## Quick Start 38 | 39 | First, create `config.ini` in `~/.config/markpress`: 40 | 41 | ```ini 42 | [default] 43 | tabsize=4 44 | 45 | [0] 46 | url=http://your-wordpress.com/ 47 | user=USERNAME 48 | passwd=PASSWORD 49 | ``` 50 | 51 | Multiple sites can be defined in different section, like `[0]`, `[1]` and `[2]`. Section `0` is the default site. 52 | 53 | After that, we can create a new document: 54 | 55 | ```bash 56 | markpress -n mypost.md 57 | ``` 58 | 59 | MarkPress will create a new markdown document with meta header: 60 | 61 | ``` 62 | --- 63 | uuid: 1234 64 | title: 65 | status: draft 66 | categories: 67 | tags: 68 | --- 69 | ``` 70 | 71 | WordPress server will allocate a unique `uuid` for each new post, and use it for post identification. Now you can edit `mypost.md` with your favorite editor and write something like: 72 | 73 | ``` 74 | --- 75 | uuid: 1234 76 | title: How to use asyncio in python ? 77 | status: publish 78 | categories: Development 79 | tags: python, server 80 | --- 81 | # Why you need asyncio ? 82 | 83 | - reason 1 84 | - reason 2 85 | - reason 3 86 | 87 | # Principle behind the asyncio 88 | 89 | ... 90 | ``` 91 | 92 | Don't forget to change `status` from `draft` to `publish`. At last use MarkPress to update your document to server: 93 | 94 | ```bash 95 | markpress -u mypost.md 96 | ``` 97 | 98 | You may see the output: 99 | 100 | ``` 101 | post uuid=1234 updated: mypost.md 102 | https://www.xxxx.com/blog/?p=1234 103 | ``` 104 | 105 | Now you can use the output url above to access your document. 106 | 107 | For Windows, use `-o` to open the url in your favorite browser: 108 | 109 | ```bash 110 | markpress -o mypost.md 111 | ``` 112 | 113 | There is also a `date` option in the meta header for publishing date: 114 | 115 | ``` 116 | --- 117 | ... 118 | date: 2019-05-11 18:48 119 | --- 120 | ``` 121 | 122 | You can use it to indicate the document's date in local time precisely. For UTC time, use the format like `2019-05-11T18:48Z`. 123 | 124 | That's all you need to know. 125 | 126 | 127 | 128 | ## Options 129 | 130 | ### Syntax Highlighting 131 | 132 | When you are using fenced code block like: 133 | 134 | ````` 135 | ```cpp 136 | int x = 10; 137 | int y = 20; 138 | ``` 139 | ````` 140 | 141 | MarkPress will translate it to: 142 | 143 | ```html 144 |
int x = 10;
145 | int y = 20;
146 | 
147 | ``` 148 | 149 | A wordpress plugin "WP Code Highlight.js" can color each `` tags by using [highlight.js](https://highlightjs.org/): 150 | 151 | ```cpp 152 | int x = 10; 153 | int y = 20; 154 | ``` 155 | 156 | It supports 185 languages with 89 styles and will definitely satisfy your need. 157 | 158 | 159 | You can change the code block styles and modify css in the setting page of "WP Code Highlight.js" in your wordpress dashboard. 160 | 161 | 162 | ### MathJax 163 | 164 | [MathJax](https://www.mathjax.org) is a JavaScript display engine for mathematics. The most easy way to use it in WordPress is using the "Simple MathJax" plugin. 165 | 166 | By default: 167 | 168 | - Expression within `$...$` will be rendered inline. 169 | - Expression within `$$...$$` will be rendered in block. 170 | 171 | ``` 172 | $\sum_{n=1}^{100} n$ 173 | ``` 174 | 175 | Will be rendered as: 176 | 177 | ![](images/math1.gif) 178 | 179 | and: 180 | 181 | ``` 182 | $$ 183 | AveP = \int_0^1 p(r) dr 184 | $$ 185 | ``` 186 | 187 | Will be rendered as: 188 | 189 | ![](images/math2.gif) 190 | 191 | Backslash can be used for escaping the `$`, if you have to input a \$ (dollar) sign in your document, escape it like `\$`. 192 | 193 | ## GraphViz 194 | 195 | MarkPress supports [GraphViz](https://www.graphviz.org/) to render diagrams: 196 | 197 | ````` 198 | ```viz-dot 199 | graph g { 200 | A -> B 201 | B -> C 202 | B -> D 203 | } 204 | ``` 205 | ````` 206 | 207 | Code block with `viz-{engine}` notation will be compiled into inline SVG xml in your post. and be rendered by browser like: 208 | 209 | ![](images/digraph.png) 210 | 211 | Engine `dot`, `circo`, `neato`, `osage`, or `twopi` are supported. 212 | 213 | `GraphViz` must be installed to enable this. MarkPress needs to find the GraphViz executables (like `dot`, `circo` ...). If they are inaccessible from environment variable `$PATH`, you can tell MarkPress how to find them in `config.ini`: 214 | 215 | ```ini 216 | [default] 217 | tabsize=4 218 | graphviz=d:/dev/tools/graphviz/bin 219 | ``` 220 | 221 | We can use another engine `circo` in a `viz-circo` block: 222 | 223 | ````` 224 | ```viz-circo 225 | digraph st2 { 226 | rankdir=TB; 227 | 228 | node [fontname = "Verdana", fontsize = 10, color="skyblue", shape="record"]; 229 | edge [fontname = "Verdana", fontsize = 10, color="crimson", style="solid"]; 230 | 231 | st_hash_type [label="{st_hash_type|(*compare)|(*hash)}"]; 232 | st_table_entry [label="{st_table_entry|hash|key|record|next}"]; 233 | st_table [label="{st_table|type|num_bins|num_entries|bins}"]; 234 | 235 | st_table:bins -> st_table_entry:head; 236 | st_table:type -> st_hash_type:head; 237 | st_table_entry:next -> st_table_entry:head [style="dashed", color="forestgreen"]; 238 | } 239 | ``` 240 | ````` 241 | 242 | Result: 243 | 244 | ![](images/circo.png) 245 | 246 | A lot of funny examples are available in [GraphViz Gallery](https://www.graphviz.org/gallery/). 247 | 248 | ### Python-markdown 249 | 250 | MarkPress ships with a light-weight markdown parser called [markdown2](https://github.com/trentm/python-markdown2). Most of time it works just fine. 251 | 252 | But you can still change it to a much powerful one: [python-markdown](https://github.com/Python-Markdown/markdown). To setup this, add `engine=markdown` option in the `default` section of your `config.ini`: 253 | 254 | ```ini 255 | [default] 256 | engine=markdown 257 | ... 258 | ``` 259 | 260 | The `engine` can be one of: 261 | 262 | - `native`: default builtin markdown2 parser. 263 | - `markdown`: python-markdown parser. 264 | - `pandoc`: as its name. 265 | 266 | The `python-markdown` has a lot of [official](https://python-markdown.github.io/extensions/) and [third-party](https://github.com/Python-Markdown/markdown/wiki/Third-Party-Extensions) extensions. 267 | 268 | Declare the extensions as a comma separated list in the `config.ini`: 269 | 270 | ```ini 271 | [default] 272 | engine=markdown 273 | extensions=pymdownx.emoji,pymdownx.details,pymdownx.magiclink,pymdownx.tilde 274 | ... 275 | ``` 276 | 277 | Remember to install them before that: 278 | 279 | ```bash 280 | pip install pymdown-extensions 281 | ``` 282 | 283 | Then we can use github emoji markup like `:sunglasses:` in our markdown document, and it will be rendered as: :sunglasses: 284 | 285 | And use `~~tilde~~` to represents ~~tilde~~. 286 | 287 | There can also be a `~/.config/markpress/extensions.py` file can be used to define a `extension_configs` dict object which will be passed to `python-markdown`. 288 | 289 | If you are using emoji you may need this to specify styles, see more [here](https://github.com/skywind3000/markpress/wiki/python-markdown-configuration). 290 | 291 | 292 | ### Proxy 293 | 294 | Proxy is specified in the site sections: 295 | 296 | ```ini 297 | [0] 298 | url=https://www.xxxx.com/ 299 | user=jim 300 | passwd=xxxxx 301 | proxy=socks5://localhost:1080 302 | ``` 303 | 304 | PySocks Supports three protocols: `http`, `socks4` and `socks5`. 305 | 306 | 307 | ## Visual Studio Code 308 | 309 | A VSCode plugin: [Markdown Preview Enhanced](https://github.com/shd101wyy/markdown-preview-enhanced) is recommended to work with MarkPress. 310 | 311 | It supports MathJax and GraphViz preview directly. The only thing we need to take care is MPE uses a different notion for GraphViz: 312 | 313 | ````` 314 | ```viz {engine="dot"} 315 | script 316 | ``` 317 | ````` 318 | 319 | We can config MPE to support our `viz-{engine}` notion, use `CTRL+SHIFT+P` and input: 320 | 321 | Markdown Preview Enhanced: Extend Parser 322 | 323 | Press `ENTER` and write a parser extender like: 324 | 325 | ```javascript 326 | module.exports = { 327 | onWillParseMarkdown: function(markdown) { 328 | return new Promise((resolve, reject)=> { 329 | var reg = new RegExp("^\\s*\\`\\`\\`viz-(\\S+)\\s*$", "g"); 330 | var parts = markdown.split("\n"); 331 | var output = new Array(); 332 | for (var j = 0; j < parts.length; j++) { 333 | var text = parts[j]; 334 | var n = text.match(reg); 335 | if (n != null) { 336 | var pos = text.indexOf("```viz-"); 337 | if (pos >= 0) { 338 | var name = text.substr(pos + 7).trim(); 339 | text = text.substr(0, pos) + "```viz {engine=\"" + name + "\"}"; 340 | } 341 | } 342 | output.push(text); 343 | } 344 | markdown = output.join("\n"); 345 | return resolve(markdown) 346 | }) 347 | }, 348 | onDidParseMarkdown: function(html) { 349 | return new Promise((resolve, reject)=> { 350 | return resolve(html) 351 | }) 352 | } 353 | } 354 | ``` 355 | 356 | It will translate `viz-xxx` to `viz {engine="xxx"}` before parsing. Then press `CTRL+S` to save the javascript. 357 | 358 | Now MPE is capable to recognize our `viz-xxx` code blocks. 359 | 360 | -------------------------------------------------------------------------------- /bin/markpress: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | MY_PATH="`dirname \"$0\"`" 4 | python3 "${MY_PATH}/markpress.py" "$@" 5 | 6 | -------------------------------------------------------------------------------- /bin/markpress.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | python "%~dp0markpress.py" %* 3 | 4 | -------------------------------------------------------------------------------- /bin/markpress.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/python3 2 | import sys 3 | import os 4 | 5 | PATH = os.path.abspath(os.path.dirname(__file__)) 6 | PATH = os.path.join(PATH, '../lib') 7 | sys.path.append(os.path.normpath(PATH)) 8 | 9 | if 1: 10 | import nextpress 11 | 12 | nextpress.main() 13 | 14 | -------------------------------------------------------------------------------- /images/circo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skywind3000/markpress/9ceec4189c6132b2efbbfd6c211dcce9c6c4a187/images/circo.png -------------------------------------------------------------------------------- /images/digraph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skywind3000/markpress/9ceec4189c6132b2efbbfd6c211dcce9c6c4a187/images/digraph.png -------------------------------------------------------------------------------- /images/math1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skywind3000/markpress/9ceec4189c6132b2efbbfd6c211dcce9c6c4a187/images/math1.gif -------------------------------------------------------------------------------- /images/math2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/skywind3000/markpress/9ceec4189c6132b2efbbfd6c211dcce9c6c4a187/images/math2.gif -------------------------------------------------------------------------------- /lib/ascmini.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # vim: set ts=4 sw=4 tw=0 et : 4 | #====================================================================== 5 | # 6 | # ascmini.py - mini library 7 | # 8 | # Created by skywind on 2017/03/24 9 | # Version: 8, Last Modified: 2022/10/18 23:22 10 | # 11 | #====================================================================== 12 | from __future__ import print_function, unicode_literals 13 | import sys 14 | import time 15 | import os 16 | import socket 17 | import collections 18 | import json 19 | 20 | 21 | #---------------------------------------------------------------------- 22 | # python 2/3 compatible 23 | #---------------------------------------------------------------------- 24 | if sys.version_info[0] >= 3: 25 | long = int 26 | unicode = str 27 | xrange = range 28 | 29 | UNIX = (sys.platform[:3] != 'win') and True or False 30 | 31 | 32 | #---------------------------------------------------------------------- 33 | # call program and returns output (combination of stdout and stderr) 34 | #---------------------------------------------------------------------- 35 | def execute(args, shell = False, capture = False): 36 | import sys, os 37 | parameters = [] 38 | cmd = None 39 | if not isinstance(args, list): 40 | import shlex 41 | cmd = args 42 | if sys.platform[:3] == 'win': 43 | ucs = False 44 | if sys.version_info[0] < 3: 45 | if not isinstance(cmd, str): 46 | cmd = cmd.encode('utf-8') 47 | ucs = True 48 | args = shlex.split(cmd.replace('\\', '\x00')) 49 | args = [ n.replace('\x00', '\\') for n in args ] 50 | if ucs: 51 | args = [ n.decode('utf-8') for n in args ] 52 | else: 53 | args = shlex.split(cmd) 54 | for n in args: 55 | if sys.platform[:3] != 'win': 56 | replace = { ' ':'\\ ', '\\':'\\\\', '\"':'\\\"', '\t':'\\t', 57 | '\n':'\\n', '\r':'\\r' } 58 | text = ''.join([ replace.get(ch, ch) for ch in n ]) 59 | parameters.append(text) 60 | else: 61 | if (' ' in n) or ('\t' in n) or ('"' in n): 62 | parameters.append('"%s"'%(n.replace('"', ' '))) 63 | else: 64 | parameters.append(n) 65 | if cmd is None: 66 | cmd = ' '.join(parameters) 67 | if sys.platform[:3] == 'win' and len(cmd) > 255: 68 | shell = False 69 | if shell and (not capture): 70 | os.system(cmd) 71 | return b'' 72 | elif (not shell) and (not capture): 73 | import subprocess 74 | if 'call' in subprocess.__dict__: 75 | subprocess.call(args) 76 | return b'' 77 | import subprocess 78 | if 'Popen' in subprocess.__dict__: 79 | p = subprocess.Popen(args, shell = shell, 80 | stdin = subprocess.PIPE, stdout = subprocess.PIPE, 81 | stderr = subprocess.STDOUT) 82 | stdin, stdouterr = (p.stdin, p.stdout) 83 | else: 84 | p = None 85 | stdin, stdouterr = os.popen4(cmd) 86 | stdin.close() 87 | text = stdouterr.read() 88 | stdouterr.close() 89 | if p: p.wait() 90 | if not capture: 91 | sys.stdout.write(text) 92 | sys.stdout.flush() 93 | return b'' 94 | return text 95 | 96 | 97 | #---------------------------------------------------------------------- 98 | # call subprocess and returns retcode, stdout, stderr 99 | #---------------------------------------------------------------------- 100 | def call(args, input_data = None, combine = False): 101 | import sys, os 102 | parameters = [] 103 | for n in args: 104 | if sys.platform[:3] != 'win': 105 | replace = { ' ':'\\ ', '\\':'\\\\', '\"':'\\\"', '\t':'\\t', 106 | '\n':'\\n', '\r':'\\r' } 107 | text = ''.join([ replace.get(ch, ch) for ch in n ]) 108 | parameters.append(text) 109 | else: 110 | if (' ' in n) or ('\t' in n) or ('"' in n): 111 | parameters.append('"%s"'%(n.replace('"', ' '))) 112 | else: 113 | parameters.append(n) 114 | cmd = ' '.join(parameters) 115 | import subprocess 116 | bufsize = 0x100000 117 | if input_data is not None: 118 | if not isinstance(input_data, bytes): 119 | if sys.stdin and sys.stdin.encoding: 120 | input_data = input_data.encode(sys.stdin.encoding, 'ignore') 121 | elif sys.stdout and sys.stdout.encoding: 122 | input_data = input_data.encode(sys.stdout.encoding, 'ignore') 123 | else: 124 | input_data = input_data.encode('utf-8', 'ignore') 125 | size = len(input_data) * 2 + 0x10000 126 | if size > bufsize: 127 | bufsize = size 128 | if 'Popen' in subprocess.__dict__: 129 | p = subprocess.Popen(args, shell = False, bufsize = bufsize, 130 | stdin = subprocess.PIPE, stdout = subprocess.PIPE, 131 | stderr = combine and subprocess.STDOUT or subprocess.PIPE) 132 | stdin, stdout, stderr = p.stdin, p.stdout, p.stderr 133 | if combine: stderr = None 134 | else: 135 | p = None 136 | if combine is False: 137 | stdin, stdout, stderr = os.popen3(cmd) 138 | else: 139 | stdin, stdout = os.popen4(cmd) 140 | stderr = None 141 | if input_data is not None: 142 | stdin.write(input_data) 143 | stdin.flush() 144 | stdin.close() 145 | exeout = stdout.read() 146 | if stderr: exeerr = stderr.read() 147 | else: exeerr = None 148 | stdout.close() 149 | if stderr: stderr.close() 150 | retcode = None 151 | if p: 152 | retcode = p.wait() 153 | return retcode, exeout, exeerr 154 | 155 | 156 | #---------------------------------------------------------------------- 157 | # redirect process output to reader(what, text) 158 | #---------------------------------------------------------------------- 159 | def redirect(args, reader, combine = True): 160 | import subprocess 161 | parameters = [] 162 | for n in args: 163 | if sys.platform[:3] != 'win': 164 | replace = { ' ':'\\ ', '\\':'\\\\', '\"':'\\\"', '\t':'\\t', 165 | '\n':'\\n', '\r':'\\r' } 166 | text = ''.join([ replace.get(ch, ch) for ch in n ]) 167 | parameters.append(text) 168 | else: 169 | if (' ' in n) or ('\t' in n) or ('"' in n): 170 | parameters.append('"%s"'%(n.replace('"', ' '))) 171 | else: 172 | parameters.append(n) 173 | cmd = ' '.join(parameters) 174 | if 'Popen' in subprocess.__dict__: 175 | p = subprocess.Popen(args, shell = False, 176 | stdin = subprocess.PIPE, stdout = subprocess.PIPE, 177 | stderr = combine and subprocess.STDOUT or subprocess.PIPE) 178 | stdin, stdout, stderr = p.stdin, p.stdout, p.stderr 179 | if combine: stderr = None 180 | else: 181 | p = None 182 | if combine is False: 183 | stdin, stdout, stderr = os.popen3(cmd) 184 | else: 185 | stdin, stdout = os.popen4(cmd) 186 | stderr = None 187 | stdin.close() 188 | while 1: 189 | text = stdout.readline() 190 | if text in (b'', ''): 191 | break 192 | reader('stdout', text) 193 | while stderr is not None: 194 | text = stderr.readline() 195 | if text in (b'', ''): 196 | break 197 | reader('stderr', text) 198 | stdout.close() 199 | if stderr: stderr.close() 200 | retcode = None 201 | if p: 202 | retcode = p.wait() 203 | return retcode 204 | 205 | 206 | #---------------------------------------------------------------------- 207 | # OBJECT:enchanced object 208 | #---------------------------------------------------------------------- 209 | class OBJECT (object): 210 | def __init__ (self, **argv): 211 | for x in argv: self.__dict__[x] = argv[x] 212 | def __getitem__ (self, x): 213 | return self.__dict__[x] 214 | def __setitem__ (self, x, y): 215 | self.__dict__[x] = y 216 | def __delitem__ (self, x): 217 | del self.__dict__[x] 218 | def __contains__ (self, x): 219 | return self.__dict__.__contains__(x) 220 | def __len__ (self): 221 | return self.__dict__.__len__() 222 | def __repr__ (self): 223 | line = [ '%s=%s'%(k, repr(v)) for k, v in self.__dict__.items() ] 224 | return 'OBJECT(' + ', '.join(line) + ')' 225 | def __str__ (self): 226 | return self.__repr__() 227 | def __iter__ (self): 228 | return self.__dict__.__iter__() 229 | 230 | 231 | #---------------------------------------------------------------------- 232 | # call stack 233 | #---------------------------------------------------------------------- 234 | def callstack (): 235 | import traceback 236 | if sys.version_info[0] < 3: 237 | import cStringIO 238 | sio = cStringIO.StringIO() 239 | else: 240 | import io 241 | sio = io.StringIO() 242 | traceback.print_exc(file = sio) 243 | return sio.getvalue() 244 | 245 | 246 | #---------------------------------------------------------------------- 247 | # Posix tools 248 | #---------------------------------------------------------------------- 249 | class PosixKit (object): 250 | 251 | def __init__ (self): 252 | self.unix = (sys.platform[:3] != 'win') 253 | 254 | # get short path name on windows 255 | def pathshort (self, path): 256 | if path is None: 257 | return None 258 | path = os.path.abspath(path) 259 | if sys.platform[:3] != 'win': 260 | return path 261 | kernel32 = None 262 | textdata = None 263 | GetShortPathName = None 264 | try: 265 | import ctypes 266 | kernel32 = ctypes.windll.LoadLibrary("kernel32.dll") 267 | textdata = ctypes.create_string_buffer(b'\000' * 1034) 268 | GetShortPathName = kernel32.GetShortPathNameA 269 | args = [ ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int ] 270 | GetShortPathName.argtypes = args 271 | GetShortPathName.restype = ctypes.c_uint32 272 | except: 273 | pass 274 | if not GetShortPathName: 275 | return path 276 | if not isinstance(path, bytes): 277 | path = path.encode(sys.stdout.encoding, 'ignore') 278 | retval = GetShortPathName(path, textdata, 1034) 279 | shortpath = textdata.value 280 | if retval <= 0: 281 | return '' 282 | if isinstance(path, bytes): 283 | if sys.stdout.encoding: 284 | shortpath = shortpath.decode(sys.stdout.encoding, 'ignore') 285 | return shortpath 286 | 287 | def mkdir (self, path): 288 | unix = sys.platform[:3] != 'win' and True or False 289 | path = os.path.abspath(path) 290 | if os.path.exists(path): 291 | return False 292 | name = '' 293 | part = os.path.abspath(path).replace('\\', '/').split('/') 294 | if unix: 295 | name = '/' 296 | if (not unix) and (path[1:2] == ':'): 297 | part[0] += '/' 298 | for n in part: 299 | name = os.path.abspath(os.path.join(name, n)) 300 | if not os.path.exists(name): 301 | os.mkdir(name) 302 | return True 303 | 304 | # remove tree 305 | def rmtree (self, path, ignore_error = False, onerror = None): 306 | import shutil 307 | shutil.rmtree(path, ignore_error, onerror) 308 | return True 309 | 310 | # absolute path 311 | def abspath (self, path, resolve = False): 312 | if path is None: 313 | return None 314 | if '~' in path: 315 | path = os.path.expanduser(path) 316 | path = os.path.abspath(path) 317 | if not UNIX: 318 | return path.lower().replace('\\', '/') 319 | if resolve: 320 | return os.path.abspath(os.path.realpath(path)) 321 | return path 322 | 323 | # find files 324 | def find (self, path, extnames = None): 325 | result = [] 326 | if extnames: 327 | if UNIX == 0: 328 | extnames = [ n.lower() for n in extnames ] 329 | extnames = tuple(extnames) 330 | for root, _, files in os.walk(path): 331 | for name in files: 332 | if extnames: 333 | ext = os.path.splitext(name)[-1] 334 | if UNIX == 0: 335 | ext = ext.lower() 336 | if ext not in extnames: 337 | continue 338 | result.append(os.path.abspath(os.path.join(root, name))) 339 | return result 340 | 341 | # which file 342 | def which (self, name, prefix = None, postfix = None): 343 | if not prefix: 344 | prefix = [] 345 | if not postfix: 346 | postfix = [] 347 | PATH = os.environ.get('PATH', '').split(UNIX and ':' or ';') 348 | search = prefix + PATH + postfix 349 | for path in search: 350 | fullname = os.path.join(path, name) 351 | if os.path.exists(fullname): 352 | return fullname 353 | return None 354 | 355 | # search executable 356 | def search_exe (self, exename, prefix = None, postfix = None): 357 | path = self.which(exename, prefix, postfix) 358 | if path is None: 359 | return None 360 | return self.pathshort(path) 361 | 362 | # executable 363 | def search_cmd (self, cmdname, prefix = None, postfix = None): 364 | if sys.platform[:3] == 'win': 365 | ext = os.path.splitext(cmdname)[-1].lower() 366 | if ext: 367 | return self.search_exe(cmdname, prefix, postfix) 368 | for ext in ('.cmd', '.bat', '.exe', '.vbs'): 369 | path = self.which(cmdname + ext, prefix, postfix) 370 | if path: 371 | return self.pathshort(path) 372 | return self.search_exe(cmdname) 373 | 374 | # load content 375 | def load_file_content (self, filename, mode = 'r'): 376 | if hasattr(filename, 'read'): 377 | try: content = filename.read() 378 | except: pass 379 | return content 380 | try: 381 | fp = open(filename, mode) 382 | content = fp.read() 383 | fp.close() 384 | except: 385 | content = None 386 | return content 387 | 388 | # save file content 389 | def save_file_content (self, filename, content, mode = 'w'): 390 | try: 391 | fp = open(filename, mode) 392 | fp.write(content) 393 | fp.close() 394 | except: 395 | return False 396 | return True 397 | 398 | # find file recursive 399 | def find_files (self, cwd, pattern = '*.*'): 400 | import fnmatch 401 | matches = [] 402 | for root, dirnames, filenames in os.walk(cwd): 403 | for filename in fnmatch.filter(filenames, pattern): 404 | matches.append(os.path.join(root, filename)) 405 | return matches 406 | 407 | # load file and guess encoding 408 | def load_file_text (self, filename, encoding = None): 409 | content = self.load_file_content(filename, 'rb') 410 | if content is None: 411 | return None 412 | if content[:3] == b'\xef\xbb\xbf': 413 | text = content[3:].decode('utf-8') 414 | elif encoding is not None: 415 | text = content.decode(encoding, 'ignore') 416 | else: 417 | text = None 418 | guess = [sys.getdefaultencoding(), 'utf-8'] 419 | if sys.stdout and sys.stdout.encoding: 420 | guess.append(sys.stdout.encoding) 421 | try: 422 | import locale 423 | guess.append(locale.getpreferredencoding()) 424 | except: 425 | pass 426 | visit = {} 427 | for name in guess + ['gbk', 'ascii', 'latin1']: 428 | if name in visit: 429 | continue 430 | visit[name] = 1 431 | try: 432 | text = content.decode(name) 433 | break 434 | except: 435 | pass 436 | if text is None: 437 | text = content.decode('utf-8', 'ignore') 438 | return text 439 | 440 | # save file text 441 | def save_file_text (self, filename, content, encoding = None): 442 | import codecs 443 | if encoding is None: 444 | encoding = 'utf-8' 445 | if (not isinstance(content, unicode)) and isinstance(content, bytes): 446 | return self.save_file_content(filename, content) 447 | with codecs.open(filename, 'w', 448 | encoding = encoding, 449 | errors = 'ignore') as fp: 450 | fp.write(content) 451 | return True 452 | 453 | # load ini without ConfigParser 454 | def load_ini (self, filename, encoding = None): 455 | text = self.load_file_text(filename, encoding) 456 | config = {} 457 | sect = 'default' 458 | if text is None: 459 | return None 460 | for line in text.split('\n'): 461 | line = line.strip('\r\n\t ') 462 | if not line: 463 | continue 464 | elif line[:1] in ('#', ';'): 465 | continue 466 | elif line.startswith('['): 467 | if line.endswith(']'): 468 | sect = line[1:-1].strip('\r\n\t ') 469 | if sect not in config: 470 | config[sect] = {} 471 | else: 472 | pos = line.find('=') 473 | if pos >= 0: 474 | key = line[:pos].rstrip('\r\n\t ') 475 | val = line[pos + 1:].lstrip('\r\n\t ') 476 | if sect not in config: 477 | config[sect] = {} 478 | config[sect][key] = val 479 | return config 480 | 481 | 482 | #---------------------------------------------------------------------- 483 | # instance 484 | #---------------------------------------------------------------------- 485 | posix = PosixKit() 486 | 487 | 488 | #---------------------------------------------------------------------- 489 | # file content load/save 490 | #---------------------------------------------------------------------- 491 | 492 | def load_config(path): 493 | import json 494 | try: 495 | text = posix.load_file_content(path, 'rb') 496 | if text is None: 497 | return None 498 | if sys.version_info[0] < 3: 499 | if text[:3] == '\xef\xbb\xbf': # remove BOM+ 500 | text = text[3:] 501 | return json.loads(text, encoding = "utf-8") 502 | else: 503 | if text[:3] == b'\xef\xbb\xbf': # remove BOM+ 504 | text = text[3:] 505 | text = text.decode('utf-8', 'ignore') 506 | return json.loads(text) 507 | except: 508 | return None 509 | return None 510 | 511 | def save_config(path, obj): 512 | import json 513 | if sys.version_info[0] < 3: 514 | text = json.dumps(obj, indent = 4, encoding = "utf-8") + '\n' 515 | else: 516 | text = json.dumps(obj, indent = 4) + '\n' 517 | text = text.encode('utf-8', 'ignore') 518 | if not posix.save_file_content(path, text, 'wb'): 519 | return False 520 | return True 521 | 522 | 523 | #---------------------------------------------------------------------- 524 | # http_request 525 | #---------------------------------------------------------------------- 526 | def http_request(url, timeout = 10, data = None, post = False, head = None): 527 | headers = [] 528 | import urllib 529 | import ssl 530 | status = -1 531 | if sys.version_info[0] >= 3: 532 | import urllib.parse 533 | import urllib.request 534 | import urllib.error 535 | if data is not None: 536 | if isinstance(data, dict): 537 | data = urllib.parse.urlencode(data) 538 | if not post: 539 | if data is None: 540 | req = urllib.request.Request(url) 541 | else: 542 | mark = '?' in url and '&' or '?' 543 | req = urllib.request.Request(url + mark + data) 544 | else: 545 | data = data is not None and data or '' 546 | if not isinstance(data, bytes): 547 | data = data.encode('utf-8', 'ignore') 548 | req = urllib.request.Request(url, data) 549 | if head: 550 | for k, v in head.items(): 551 | req.add_header(k, v) 552 | try: 553 | res = urllib.request.urlopen(req, timeout = timeout) 554 | headers = res.getheaders() 555 | except urllib.error.HTTPError as e: 556 | return e.code, str(e.message), None 557 | except urllib.error.URLError as e: 558 | return -1, str(e), None 559 | except socket.timeout: 560 | return -2, 'timeout', None 561 | except ssl.SSLError: 562 | return -2, 'timeout', None 563 | content = res.read() 564 | status = res.getcode() 565 | else: 566 | import urllib2 567 | if data is not None: 568 | if isinstance(data, dict): 569 | part = {} 570 | for key in data: 571 | val = data[key] 572 | if isinstance(key, unicode): 573 | key = key.encode('utf-8') 574 | if isinstance(val, unicode): 575 | val = val.encode('utf-8') 576 | part[key] = val 577 | data = urllib.urlencode(part) 578 | if not isinstance(data, bytes): 579 | data = data.encode('utf-8', 'ignore') 580 | if not post: 581 | if data is None: 582 | req = urllib2.Request(url) 583 | else: 584 | mark = '?' in url and '&' or '?' 585 | req = urllib2.Request(url + mark + data) 586 | else: 587 | req = urllib2.Request(url, data is not None and data or '') 588 | if head: 589 | for k, v in head.items(): 590 | req.add_header(k, v) 591 | try: 592 | res = urllib2.urlopen(req, timeout = timeout) 593 | content = res.read() 594 | status = res.getcode() 595 | if res.info().headers: 596 | for line in res.info().headers: 597 | line = line.rstrip('\r\n\t') 598 | pos = line.find(':') 599 | if pos < 0: 600 | continue 601 | key = line[:pos].rstrip('\t ') 602 | val = line[pos + 1:].lstrip('\t ') 603 | headers.append((key, val)) 604 | except urllib2.HTTPError as e: 605 | return e.code, str(e.message), None 606 | except urllib2.URLError as e: 607 | return -1, str(e), None 608 | except socket.timeout: 609 | return -2, 'timeout', None 610 | except ssl.SSLError: 611 | return -2, 'timeout', None 612 | return status, content, headers 613 | 614 | 615 | #---------------------------------------------------------------------- 616 | # request with retry 617 | #---------------------------------------------------------------------- 618 | def request_safe(url, timeout = 10, retry = 3, verbose = True, delay = 1): 619 | for i in xrange(retry): 620 | if verbose: 621 | print('%s: %s'%(i == 0 and 'request' or 'retry', url)) 622 | time.sleep(delay) 623 | code, content, _ = http_request(url, timeout) 624 | if code == 200: 625 | return content 626 | return None 627 | 628 | 629 | #---------------------------------------------------------------------- 630 | # request json rpc 631 | #---------------------------------------------------------------------- 632 | def json_rpc_post(url, message, timeout = 10): 633 | import json 634 | data = json.dumps(message) 635 | header = [('Content-Type', 'text/plain; charset=utf-8')] 636 | code, content, _ = http_request(url, timeout, data, True, header) 637 | if code == 200: 638 | content = json.loads(content) 639 | return code, content 640 | 641 | 642 | #---------------------------------------------------------------------- 643 | # timestamp 644 | #---------------------------------------------------------------------- 645 | def timestamp(ts = None, onlyday = False): 646 | import time 647 | if not ts: ts = time.time() 648 | if onlyday: 649 | time.strftime('%Y%m%d', time.localtime(ts)) 650 | return time.strftime('%Y%m%d%H%M%S', time.localtime(ts)) 651 | 652 | 653 | #---------------------------------------------------------------------- 654 | # timestamp 655 | #---------------------------------------------------------------------- 656 | def readts(ts, onlyday = False): 657 | if onlyday: ts += '000000' 658 | try: return time.mktime(time.strptime(ts, '%Y%m%d%H%M%S')) 659 | except: pass 660 | return 0 661 | 662 | 663 | #---------------------------------------------------------------------- 664 | # parse text 665 | #---------------------------------------------------------------------- 666 | def parse_conf_text(text, default = None): 667 | if text is None: 668 | return default 669 | if isinstance(default, str): 670 | return text 671 | elif isinstance(default, bool): 672 | text = text.lower() 673 | if not text: 674 | return default 675 | text = text.lower() 676 | if default: 677 | if text in ('false', 'f', 'no', 'n', '0'): 678 | return False 679 | else: 680 | if text in ('true', 'ok', 'yes', 't', 'y', '1'): 681 | return True 682 | if text.isdigit(): 683 | try: 684 | value = int(text) 685 | if value: 686 | return True 687 | except: 688 | pass 689 | return default 690 | elif isinstance(default, float): 691 | try: 692 | value = float(text) 693 | return value 694 | except: 695 | return default 696 | elif isinstance(default, int) or isinstance(default, long): 697 | multiply = 1 698 | text = text.strip('\r\n\t ') 699 | postfix1 = text[-1:].lower() 700 | postfix2 = text[-2:].lower() 701 | if postfix1 == 'k': 702 | multiply = 1024 703 | text = text[:-1] 704 | elif postfix1 == 'm': 705 | multiply = 1024 * 1024 706 | text = text[:-1] 707 | elif postfix2 == 'kb': 708 | multiply = 1024 709 | text = text[:-2] 710 | elif postfix2 == 'mb': 711 | multiply = 1024 * 1024 712 | text = text[:-2] 713 | try: text = int(text.strip('\r\n\t '), 0) 714 | except: text = default 715 | if multiply > 1: 716 | text *= multiply 717 | return text 718 | return text 719 | 720 | 721 | 722 | #---------------------------------------------------------------------- 723 | # ConfigReader 724 | #---------------------------------------------------------------------- 725 | class ConfigReader (object): 726 | 727 | def __init__ (self, ininame, codec = None): 728 | self.ininame = ininame 729 | self.reset() 730 | self.load(ininame, codec) 731 | 732 | def reset (self): 733 | self.config = {} 734 | self.sections = [] 735 | return True 736 | 737 | def load (self, ininame, codec = None): 738 | if not ininame: 739 | return False 740 | elif not os.path.exists(ininame): 741 | return False 742 | try: 743 | content = open(ininame, 'rb').read() 744 | except IOError: 745 | content = b'' 746 | if content[:3] == b'\xef\xbb\xbf': 747 | text = content[3:].decode('utf-8') 748 | elif codec is not None: 749 | text = content.decode(codec, 'ignore') 750 | else: 751 | codec = sys.getdefaultencoding() 752 | text = None 753 | for name in [codec, 'gbk', 'utf-8']: 754 | try: 755 | text = content.decode(name) 756 | break 757 | except: 758 | pass 759 | if text is None: 760 | text = content.decode('utf-8', 'ignore') 761 | if sys.version_info[0] < 3: 762 | import StringIO 763 | import ConfigParser 764 | sio = StringIO.StringIO(text) 765 | cp = ConfigParser.ConfigParser() 766 | cp.readfp(sio) 767 | else: 768 | import configparser 769 | cp = configparser.ConfigParser(interpolation = None) 770 | cp.read_string(text) 771 | for sect in cp.sections(): 772 | for key, val in cp.items(sect): 773 | lowsect, lowkey = sect.lower(), key.lower() 774 | self.config.setdefault(lowsect, {})[lowkey] = val 775 | if 'default' not in self.config: 776 | self.config['default'] = {} 777 | return True 778 | 779 | def option (self, section, item, default = None): 780 | sect = self.config.get(section, None) 781 | if not sect: 782 | return default 783 | text = sect.get(item, None) 784 | if text is None: 785 | return default 786 | return parse_conf_text(text, default) 787 | 788 | 789 | #---------------------------------------------------------------------- 790 | # Csv Read/Write 791 | #---------------------------------------------------------------------- 792 | 793 | def csv_load (filename, encoding = None): 794 | content = None 795 | text = None 796 | try: 797 | content = open(filename, 'rb').read() 798 | except: 799 | return None 800 | if content is None: 801 | return None 802 | if content[:3] == b'\xef\xbb\xbf': 803 | text = content[3:].decode('utf-8') 804 | elif encoding is not None: 805 | text = content.decode(encoding, 'ignore') 806 | else: 807 | codec = sys.getdefaultencoding() 808 | text = None 809 | for name in [codec, 'utf-8', 'gbk', 'ascii', 'latin1']: 810 | try: 811 | text = content.decode(name) 812 | break 813 | except: 814 | pass 815 | if text is None: 816 | text = content.decode('utf-8', 'ignore') 817 | if not text: 818 | return None 819 | import csv 820 | if sys.version_info[0] < 3: 821 | import cStringIO 822 | sio = cStringIO.StringIO(text.encode('utf-8', 'ignore')) 823 | else: 824 | import io 825 | sio = io.StringIO(text) 826 | reader = csv.reader(sio) 827 | output = [] 828 | if sys.version_info[0] < 3: 829 | for row in reader: 830 | output.append([ n.decode('utf-8', 'ignore') for n in row ]) 831 | else: 832 | for row in reader: 833 | output.append(row) 834 | return output 835 | 836 | 837 | def csv_save (filename, rows, encoding = 'utf-8'): 838 | import csv 839 | ispy2 = (sys.version_info[0] < 3) 840 | if not encoding: 841 | encoding = 'utf-8' 842 | if sys.version_info[0] < 3: 843 | fp = open(filename, 'wb') 844 | writer = csv.writer(fp) 845 | else: 846 | fp = open(filename, 'w', encoding = encoding, newline = '') 847 | writer = csv.writer(fp) 848 | for row in rows: 849 | newrow = [] 850 | for n in row: 851 | if isinstance(n, int) or isinstance(n, long): 852 | n = str(n) 853 | elif isinstance(n, float): 854 | n = str(n) 855 | elif not isinstance(n, bytes): 856 | if (n is not None) and ispy2: 857 | n = n.encode(encoding, 'ignore') 858 | newrow.append(n) 859 | writer.writerow(newrow) 860 | fp.close() 861 | return True 862 | 863 | 864 | #---------------------------------------------------------------------- 865 | # object pool 866 | #---------------------------------------------------------------------- 867 | class ObjectPool (object): 868 | 869 | def __init__ (self): 870 | import threading 871 | self._pools = {} 872 | self._lock = threading.Lock() 873 | 874 | def get (self, name): 875 | hr = None 876 | self._lock.acquire() 877 | pset = self._pools.get(name, None) 878 | if pset: 879 | hr = pset.pop() 880 | self._lock.release() 881 | return hr 882 | 883 | def put (self, name, obj): 884 | self._lock.acquire() 885 | pset = self._pools.get(name, None) 886 | if pset is None: 887 | pset = set() 888 | self._pools[name] = pset 889 | pset.add(obj) 890 | self._lock.release() 891 | return True 892 | 893 | 894 | #---------------------------------------------------------------------- 895 | # WebKit 896 | #---------------------------------------------------------------------- 897 | class WebKit (object): 898 | 899 | def __init__ (self): 900 | pass 901 | 902 | # Check IS FastCGI 903 | def IsFastCGI (self): 904 | import socket, errno 905 | if 'fromfd' not in socket.__dict__: 906 | return False 907 | try: 908 | s = socket.fromfd(sys.stdin.fileno(), socket.AF_INET, 909 | socket.SOCK_STREAM) 910 | s.getpeername() 911 | except socket.error as err: 912 | if err.errno != errno.ENOTCONN: 913 | return False 914 | return True 915 | 916 | def text2html (self, s): 917 | import cgi 918 | return cgi.escape(s, True).replace('\n', "
\n") 919 | 920 | def html2text (self, html): 921 | part = [] 922 | pos = 0 923 | while 1: 924 | f1 = html.find('<', pos) 925 | if f1 < 0: 926 | part.append((0, html[pos:])) 927 | break 928 | f2 = html.find('>', f1) 929 | if f2 < 0: 930 | part.append((0, html[pos:])) 931 | break 932 | text = html[pos:f1] 933 | flag = html[f1:f2 + 1] 934 | pos = f2 + 1 935 | if text: 936 | part.append((0, text)) 937 | if flag: 938 | part.append((1, flag)) 939 | output = '' 940 | for mode, text in part: 941 | if mode == 0: 942 | text = text.lstrip() 943 | text = text.replace(' ', ' ').replace('>', '>') 944 | text = text.replace('<', '<').replace('&', '&') 945 | output += text 946 | else: 947 | text = text.strip() 948 | tiny = text.replace(' ', '') 949 | if tiny in ('

', '

', '
', '
', '
'): 950 | output += '\n' 951 | elif tiny in ('', '', '', '', ''): 952 | output += '\n' 953 | elif tiny in ('', ''): 954 | output += ' ' 955 | elif tiny in ('',): 956 | output += '\n' 957 | return output 958 | 959 | def match_text (self, text, position, starts, ends): 960 | p1 = text.find(starts, position) 961 | if p1 < 0: 962 | return None, position 963 | p2 = text.find(ends, p1 + len(starts)) 964 | if p2 < 0: 965 | return None, position 966 | value = text[p1 + len(starts):p2] 967 | return value, p2 + len(ends) 968 | 969 | def replace_range (self, text, start, size, newtext): 970 | head = text[:start] 971 | tail = text[start + size:] 972 | return head + newtext + tail 973 | 974 | def url_parse (self, url): 975 | if sys.version_info[0] < 3: 976 | import urlparse 977 | return urlparse.urlparse(url) 978 | import urllib.parse 979 | return urllib.parse.urlparse(url) 980 | 981 | def url_unquote (self, text, plus = True): 982 | if sys.version_info[0] < 3: 983 | import urllib 984 | if plus: 985 | return urllib.unquote_plus(text) 986 | return urllib.unquote(text) 987 | import urllib.parse 988 | if plus: 989 | return urllib.parse.unquote_plus(text) 990 | return urllib.parse.unquote(text) 991 | 992 | def url_quote (self, text, plus = True): 993 | if sys.version_info[0] < 3: 994 | import urllib 995 | if plus: 996 | return urllib.quote_plus(text) 997 | return urlparse.quote(text) 998 | import urllib.parse 999 | if plus: 1000 | return urllib.parse.quote_plus(text) 1001 | return urllib.parse.quote(text) 1002 | 1003 | def url_parse_qs (self, text, keep_blank = 0): 1004 | if sys.version_info[0] < 3: 1005 | import urlparse 1006 | return urlparse.parse_qs(text, keep_blank) 1007 | import urllib.parse 1008 | return urllib.parse.parse_qs(text, keep_blank) 1009 | 1010 | def url_parse_qsl (self, text, keep_blank = 0): 1011 | if sys.version_info[0] < 3: 1012 | import urlparse 1013 | return urlparse.parse_qsl(text, keep_blank) 1014 | import urllib.parse 1015 | return urllib.parse.parse_qsl(text, keep_blank) 1016 | 1017 | 1018 | 1019 | #---------------------------------------------------------------------- 1020 | # instance 1021 | #---------------------------------------------------------------------- 1022 | web = WebKit() 1023 | 1024 | 1025 | #---------------------------------------------------------------------- 1026 | # LazyRequests 1027 | #---------------------------------------------------------------------- 1028 | class LazyRequests (object): 1029 | 1030 | def __init__ (self): 1031 | import threading 1032 | self._pools = {} 1033 | self._lock = threading.Lock() 1034 | self._options = {} 1035 | self._option = {} 1036 | 1037 | def __session_get (self, name): 1038 | hr = None 1039 | with self._lock: 1040 | pset = self._pools.get(name, None) 1041 | if pset: 1042 | hr = pset.pop() 1043 | return hr 1044 | 1045 | def __session_put (self, name, obj): 1046 | with self._lock: 1047 | pset = self._pools.get(name, None) 1048 | if pset is None: 1049 | pset = set() 1050 | self._pools[name] = pset 1051 | pset.add(obj) 1052 | return True 1053 | 1054 | def request (self, name, url, data = None, post = False, header = None): 1055 | import requests 1056 | import copy 1057 | s = self.__session_get(name) 1058 | if not s: 1059 | s = requests.Session() 1060 | r = None 1061 | option = self._options.get(name, {}) 1062 | argv = {} 1063 | timeout = self._option.get('timeout', None) 1064 | proxy = self._option.get('proxy', None) 1065 | agent = self._option.get('agent', None) 1066 | if 'timeout' in option: 1067 | timeout = option.get('timeout') 1068 | if 'proxy' in option: 1069 | proxy = option['proxy'] 1070 | if proxy and isinstance(proxy, str): 1071 | if proxy.startswith('socks5://'): 1072 | proxy = 'socks5h://' + proxy[9:] 1073 | proxy = {'http': proxy, 'https': proxy} 1074 | if 'agent' in option: 1075 | agent = option['agent'] 1076 | if timeout: 1077 | argv['timeout'] = timeout 1078 | if proxy: 1079 | argv['proxies'] = proxy 1080 | if header is None: 1081 | header = {} 1082 | else: 1083 | header = copy.deepcopy(header) 1084 | if agent: 1085 | header['User-Agent'] = agent 1086 | if header is not None: 1087 | argv['headers'] = header 1088 | if not post: 1089 | if data is not None: 1090 | argv['params'] = data 1091 | else: 1092 | if data is not None: 1093 | argv['data'] = data 1094 | try: 1095 | if not post: 1096 | r = s.get(url, **argv) 1097 | else: 1098 | r = s.post(url, **argv) 1099 | except requests.exceptions.ConnectionError: 1100 | r = None 1101 | except requests.exceptions.RetryError as e: 1102 | r = requests.Response() 1103 | r.status_code = -1 1104 | r.text = 'RetryError' 1105 | r.error = e 1106 | except requests.exceptions.BaseHTTPError as e: 1107 | r = requests.Response() 1108 | r.status_code = -2 1109 | r.text = 'BaseHTTPError' 1110 | r.error = e 1111 | except requests.exceptions.HTTPError as e: 1112 | r = requests.Response() 1113 | r.status_code = -3 1114 | r.text = 'HTTPError' 1115 | r.error = e 1116 | except requests.exceptions.RequestException as e: 1117 | r = requests.Response() 1118 | r.status_code = -4 1119 | r.error = e 1120 | self.__session_put(name, s) 1121 | return r 1122 | 1123 | def option (self, name, opt, value): 1124 | if name is None: 1125 | self._option[opt] = value 1126 | else: 1127 | if name not in self._options: 1128 | self._options[name] = {} 1129 | opts = self._options[name] 1130 | opts[opt] = value 1131 | return True 1132 | 1133 | def get (self, name, url, data = None, header = None): 1134 | return self.request(name, url, data, False, header) 1135 | 1136 | def post (self, name, url, data = None, header = None): 1137 | return self.request(name, url, data, True, header) 1138 | 1139 | def wget (self, name, url, data = None, post = False, header = None): 1140 | r = self.request(name, url, data, post, header) 1141 | if r is None: 1142 | return -1, None 1143 | if r.content: 1144 | text = r.content.decode('utf-8') 1145 | else: 1146 | text = r.text 1147 | return r.status_code, text 1148 | 1149 | 1150 | #---------------------------------------------------------------------- 1151 | # instance 1152 | #---------------------------------------------------------------------- 1153 | lazy = LazyRequests() 1154 | 1155 | 1156 | #---------------------------------------------------------------------- 1157 | # ShellUtils 1158 | #---------------------------------------------------------------------- 1159 | class ShellUtils (object): 1160 | 1161 | # compress into a zip file, srcnames must be a list of tuples: 1162 | # [ (filename_1, arcname_1), (filename_2, arcname_2), ... ] 1163 | def zip_compress (self, zipname, srcnames, mode = 'w'): 1164 | import zipfile 1165 | if isinstance(srcnames, dict): 1166 | names = [ (v and v or k, k) for k, v in srcnames.items() ] 1167 | else: 1168 | names = [] 1169 | for item in srcnames: 1170 | if isinstance(item, tuple) or isinstance(item, list): 1171 | srcname, arcname = item[0], item[1] 1172 | else: 1173 | srcname, arcname = item, None 1174 | names.append((arcname and arcname or srcname, srcname)) 1175 | names.sort() 1176 | zfp = zipfile.ZipFile(zipname, mode, zipfile.ZIP_DEFLATED) 1177 | for arcname, srcname in names: 1178 | zfp.write(srcname, arcname) 1179 | zfp.close() 1180 | zfp = None 1181 | return 0 1182 | 1183 | # find root 1184 | def find_root (self, path, markers = None, fallback = False): 1185 | if markers is None: 1186 | markers = ('.git', '.svn', '.hg', '.project', '.root') 1187 | if path is None: 1188 | path = os.getcwd() 1189 | path = os.path.abspath(path) 1190 | base = path 1191 | while True: 1192 | parent = os.path.normpath(os.path.join(base, '..')) 1193 | for marker in markers: 1194 | test = os.path.join(base, marker) 1195 | if os.path.exists(test): 1196 | return base 1197 | if os.path.normcase(parent) == os.path.normcase(base): 1198 | break 1199 | base = parent 1200 | if fallback: 1201 | return path 1202 | return None 1203 | 1204 | # project root 1205 | def project_root (self, path, markers = None): 1206 | return self.find_root(path, markers, True) 1207 | 1208 | # getopt: returns (options, args) 1209 | def getopt (self, argv): 1210 | args = [] 1211 | options = {} 1212 | if argv is None: 1213 | argv = sys.argv[1:] 1214 | index = 0 1215 | count = len(argv) 1216 | while index < count: 1217 | arg = argv[index] 1218 | if arg != '': 1219 | head = arg[:1] 1220 | if head != '-': 1221 | break 1222 | if arg == '-': 1223 | break 1224 | name = arg.lstrip('-') 1225 | key, _, val = name.partition('=') 1226 | options[key.strip()] = val.strip() 1227 | index += 1 1228 | while index < count: 1229 | args.append(argv[index]) 1230 | index += 1 1231 | return options, args 1232 | 1233 | # hexdump 1234 | def hexdump (self, data, char = False): 1235 | content = '' 1236 | charset = '' 1237 | lines = [] 1238 | if isinstance(data, str): 1239 | if sys.version_info[0] >= 3: 1240 | data = data.encode('utf-8', 'ignore') 1241 | if not isinstance(data, bytes): 1242 | raise ValueError('data must be bytes') 1243 | for i, _ in enumerate(data): 1244 | if sys.version_info[0] < 3: 1245 | ascii = ord(data[i]) 1246 | else: 1247 | ascii = data[i] 1248 | if i % 16 == 0: content += '%08X '%i 1249 | content += '%02X'%ascii 1250 | content += ((i & 15) == 7) and '-' or ' ' 1251 | if (ascii >= 0x20) and (ascii < 0x7f): charset += chr(ascii) 1252 | else: charset += '.' 1253 | if (i % 16 == 15): 1254 | lines.append(content + ' ' + charset) 1255 | content, charset = '', '' 1256 | if len(content) < 60: content += ' ' * (58 - len(content)) 1257 | lines.append(content + ' ' + charset) 1258 | limit = char and 104 or 58 1259 | return '\n'.join([ n[:limit] for n in lines ]) 1260 | 1261 | def print_binary (self, data, char = False): 1262 | print(self.hexdump(data, char)) 1263 | return True 1264 | 1265 | 1266 | utils = ShellUtils() 1267 | 1268 | 1269 | #---------------------------------------------------------------------- 1270 | # TraceOut 1271 | #---------------------------------------------------------------------- 1272 | class TraceOut (object): 1273 | 1274 | def __init__ (self, prefix = ''): 1275 | self._prefix = prefix 1276 | import threading 1277 | self._lock = threading.Lock() 1278 | self._logtime = None 1279 | self._logfile = None 1280 | self._channels = {'info':True, 'debug':True, 'error':True} 1281 | self._channels['warn'] = True 1282 | self._encoding = 'utf-8' 1283 | self._stdout = sys.__stdout__ 1284 | self._stderr = False 1285 | self._makedir = False 1286 | 1287 | def _writelog (self, *args): 1288 | now = time.strftime('%Y-%m-%d %H:%M:%S') 1289 | date = now.split(None, 1)[0].replace('-', '') 1290 | self._lock.acquire() 1291 | if date != self._logtime: 1292 | self._logtime = date 1293 | if self._logfile is not None: 1294 | try: 1295 | self._logfile.close() 1296 | except: 1297 | pass 1298 | self._logfile = None 1299 | if self._logfile is None: 1300 | import codecs 1301 | logname = '%s%s.log'%(self._prefix, date) 1302 | dirname = os.path.dirname(logname) 1303 | if self._makedir: 1304 | if not os.path.exists(dirname): 1305 | try: os.makedirs(dirname) 1306 | except: pass 1307 | self._logfile = codecs.open(logname, 'a', self._encoding) 1308 | part = [] 1309 | for text in args: 1310 | if isinstance(text, unicode) or isinstance(text, str): 1311 | if not isinstance(text, unicode): 1312 | text = text.decode(self._encoding) 1313 | else: 1314 | text = unicode(text) 1315 | part.append(text) 1316 | text = u' '.join(part) 1317 | self._logfile.write('[%s] %s\r\n'%(now, text)) 1318 | self._logfile.flush() 1319 | self._lock.release() 1320 | if self._stdout: 1321 | self._stdout.write('[%s] %s\n'%(now, text)) 1322 | self._stdout.flush() 1323 | if self._stderr: 1324 | self._stderr.write('[%s] %s\n'%(now, text)) 1325 | self._stderr.flush() 1326 | return True 1327 | 1328 | def change (self, prefix): 1329 | self._lock.acquire() 1330 | self._logtime = None 1331 | self._prefix = prefix 1332 | if self._logfile: 1333 | try: 1334 | self._logfile.close() 1335 | except: 1336 | pass 1337 | self._logfile = None 1338 | self._lock.release() 1339 | return True 1340 | 1341 | def out (self, channel, *args): 1342 | if not self._channels.get(channel, False): 1343 | return False 1344 | self._writelog('[%s]'%channel, *args) 1345 | return True 1346 | 1347 | def info (self, *args): 1348 | self.out('info', *args) 1349 | 1350 | def warn (self, *args): 1351 | self.out('warn', *args) 1352 | 1353 | def error (self, *args): 1354 | self.out('error', *args) 1355 | 1356 | def debug (self, *args): 1357 | self.out('debug', *args) 1358 | 1359 | 1360 | #---------------------------------------------------------------------- 1361 | # OutputHandler 1362 | #---------------------------------------------------------------------- 1363 | class OutputHandler (object): 1364 | def __init__(self, writer): 1365 | import threading 1366 | self.writer = writer 1367 | self.content = '' 1368 | self.lock = threading.Lock() 1369 | self.encoding = sys.__stdout__.encoding 1370 | def flush(self): 1371 | return True 1372 | def write(self, s): 1373 | self.lock.acquire() 1374 | self.content += s 1375 | while True: 1376 | pos = self.content.find('\n') 1377 | if pos < 0: break 1378 | self.writer(self.content[:pos]) 1379 | self.content = self.content[pos + 1:] 1380 | self.lock.release() 1381 | return True 1382 | def writelines(self, l): 1383 | map(self.write, l) 1384 | 1385 | 1386 | #---------------------------------------------------------------------- 1387 | # run until mainfunc returns false 1388 | #---------------------------------------------------------------------- 1389 | def safe_loop (mainfunc, trace = None, sleep = 2.0, dtor = None): 1390 | while True: 1391 | try: 1392 | hr = mainfunc() 1393 | if not hr: 1394 | break 1395 | except KeyboardInterrupt: 1396 | tb = callstack().split('\n') 1397 | if trace: 1398 | for line in tb: 1399 | trace.error(line) 1400 | else: 1401 | for line in tb: 1402 | sys.stderr.write(line + '\n') 1403 | break 1404 | except: 1405 | tb = callstack().split('\n') 1406 | if trace: 1407 | for line in tb: 1408 | trace.error(line) 1409 | else: 1410 | for line in tb: 1411 | sys.stderr.write(line + '\n') 1412 | if dtor: 1413 | if trace: 1414 | trace.error('clean up') 1415 | else: 1416 | sys.stderr.write('clean up\n') 1417 | try: 1418 | dtor() 1419 | except: 1420 | pass 1421 | if trace: 1422 | trace.error('') 1423 | trace.error('restarting in %s seconds'%sleep) 1424 | trace.error('') 1425 | else: 1426 | sys.stderr.write('\nready to restart\n') 1427 | time.sleep(sleep) 1428 | return True 1429 | 1430 | 1431 | #---------------------------------------------------------------------- 1432 | # tabulify: style = 0, 1, 2 1433 | #---------------------------------------------------------------------- 1434 | def tabulify (rows, style = 0): 1435 | colsize = {} 1436 | maxcol = 0 1437 | output = [] 1438 | if not rows: 1439 | return '' 1440 | for row in rows: 1441 | maxcol = max(len(row), maxcol) 1442 | for col, text in enumerate(row): 1443 | text = str(text) 1444 | size = len(text) 1445 | if col not in colsize: 1446 | colsize[col] = size 1447 | else: 1448 | colsize[col] = max(size, colsize[col]) 1449 | if maxcol <= 0: 1450 | return '' 1451 | def gettext(row, col): 1452 | csize = colsize[col] 1453 | if row >= len(rows): 1454 | return ' ' * (csize + 2) 1455 | row = rows[row] 1456 | if col >= len(row): 1457 | return ' ' * (csize + 2) 1458 | text = str(row[col]) 1459 | padding = 2 + csize - len(text) 1460 | pad1 = 1 1461 | pad2 = padding - pad1 1462 | return (' ' * pad1) + text + (' ' * pad2) 1463 | if style == 0: 1464 | for y, row in enumerate(rows): 1465 | line = ''.join([ gettext(y, x) for x in xrange(maxcol) ]) 1466 | output.append(line) 1467 | elif style == 1: 1468 | if rows: 1469 | newrows = rows[:1] 1470 | head = [ '-' * colsize[i] for i in xrange(maxcol) ] 1471 | newrows.append(head) 1472 | newrows.extend(rows[1:]) 1473 | rows = newrows 1474 | for y, row in enumerate(rows): 1475 | line = ''.join([ gettext(y, x) for x in xrange(maxcol) ]) 1476 | output.append(line) 1477 | elif style == 2: 1478 | sep = '+'.join([ '-' * (colsize[x] + 2) for x in xrange(maxcol) ]) 1479 | sep = '+' + sep + '+' 1480 | for y, row in enumerate(rows): 1481 | output.append(sep) 1482 | line = '|'.join([ gettext(y, x) for x in xrange(maxcol) ]) 1483 | output.append('|' + line + '|') 1484 | output.append(sep) 1485 | return '\n'.join(output) 1486 | 1487 | 1488 | #---------------------------------------------------------------------- 1489 | # compact dict: k1:v1,k2:v2,...,kn:vn 1490 | #---------------------------------------------------------------------- 1491 | def compact_dumps(data): 1492 | output = [] 1493 | for k, v in data.items(): 1494 | k = k.strip().replace(',', '').replace(':', '') 1495 | v = v.strip().replace(',', '').replace(':', '') 1496 | output.append(k + ':' + v) 1497 | return ','.join(output) 1498 | 1499 | def compact_loads(text): 1500 | data = {} 1501 | for pp in text.strip().split(','): 1502 | pp = pp.strip() 1503 | if not pp: 1504 | continue 1505 | ps = pp.split(':') 1506 | if len(ps) < 2: 1507 | continue 1508 | k = ps[0].strip() 1509 | v = ps[1].strip() 1510 | if k: 1511 | data[k] = v 1512 | return data 1513 | 1514 | 1515 | #---------------------------------------------------------------------- 1516 | # replace file atomicly 1517 | #---------------------------------------------------------------------- 1518 | def replace_file (srcname, dstname): 1519 | import sys, os 1520 | if sys.platform[:3] != 'win': 1521 | try: 1522 | os.rename(srcname, dstname) 1523 | except OSError: 1524 | return False 1525 | else: 1526 | import ctypes.wintypes 1527 | kernel32 = ctypes.windll.kernel32 1528 | wp, vp, cp = ctypes.c_wchar_p, ctypes.c_void_p, ctypes.c_char_p 1529 | DWORD, BOOL = ctypes.wintypes.DWORD, ctypes.wintypes.BOOL 1530 | kernel32.ReplaceFileA.argtypes = [ cp, cp, cp, DWORD, vp, vp ] 1531 | kernel32.ReplaceFileW.argtypes = [ wp, wp, wp, DWORD, vp, vp ] 1532 | kernel32.ReplaceFileA.restype = BOOL 1533 | kernel32.ReplaceFileW.restype = BOOL 1534 | kernel32.GetLastError.argtypes = [] 1535 | kernel32.GetLastError.restype = DWORD 1536 | success = False 1537 | try: 1538 | os.rename(srcname, dstname) 1539 | success = True 1540 | except OSError: 1541 | pass 1542 | if success: 1543 | return True 1544 | if sys.version_info[0] < 3 and isinstance(srcname, str): 1545 | hr = kernel32.ReplaceFileA(dstname, srcname, None, 2, None, None) 1546 | else: 1547 | hr = kernel32.ReplaceFileW(dstname, srcname, None, 2, None, None) 1548 | if not hr: 1549 | return False 1550 | return True 1551 | 1552 | 1553 | #---------------------------------------------------------------------- 1554 | # random temp 1555 | #---------------------------------------------------------------------- 1556 | def tmpname (filename, fill = 5): 1557 | import time, os, random 1558 | while 1: 1559 | name = '.' + str(int(time.time() * 1000000)) 1560 | for i in range(fill): 1561 | k = random.randint(0, 51) 1562 | name += (k < 26) and chr(ord('A') + k) or chr(ord('a') + k - 26) 1563 | test = filename + name + str(os.getpid()) 1564 | if not os.path.exists(test): 1565 | return test 1566 | return None 1567 | 1568 | 1569 | #---------------------------------------------------------------------- 1570 | # save json atomic 1571 | #---------------------------------------------------------------------- 1572 | def save_config_atomic(filename, obj): 1573 | temp = tmpname(filename) 1574 | save_config(temp, obj) 1575 | return replace_file(temp, filename) 1576 | 1577 | 1578 | #---------------------------------------------------------------------- 1579 | # Simple Timer 1580 | #---------------------------------------------------------------------- 1581 | class SimpleTimer (object): 1582 | 1583 | def __init__ (self, period): 1584 | self.__current = None 1585 | self.__timeslap = None 1586 | self.__period = period 1587 | 1588 | def run (self): 1589 | raise NotImplementedError('Method not implemented') 1590 | 1591 | def update (self, now): 1592 | self.__current = now 1593 | if self.__timeslap is None: 1594 | self.__timeslap = self.__current + self.__period 1595 | elif self.__current >= self.__timeslap: 1596 | self.__timeslap = self.__current + self.__period 1597 | self.run() 1598 | return True 1599 | 1600 | 1601 | #---------------------------------------------------------------------- 1602 | # Registry 1603 | #---------------------------------------------------------------------- 1604 | class Registry (object): 1605 | 1606 | def __init__ (self, filename = None): 1607 | self.registry = {} 1608 | if filename: 1609 | registry = load_config(filename) 1610 | if registry: 1611 | self.registry = registry 1612 | self.filename = filename 1613 | 1614 | def save (self, filename = None): 1615 | filename = (filename) and filename or self.filename 1616 | if filename is None: 1617 | raise IOError('Filename must not be None') 1618 | names = list(self.registry.keys()) 1619 | names.sort() 1620 | dump = collections.OrderedDict() 1621 | for name in names: 1622 | dump[name] = self.registry[name] 1623 | save_config_atomic(filename, dump) 1624 | 1625 | def get (self, key, default = None): 1626 | return self.registry.get(key, default) 1627 | 1628 | def set (self, key, value): 1629 | if (not isinstance(key, str)) and (not isinstance(key, int)): 1630 | raise ValueError('key must be int/string') 1631 | if (not isinstance(value, str)) and (not isinstance(value, int)): 1632 | if (not isinstance(value, float)) and (not isinstance(value, bool)): 1633 | if value is not None: 1634 | raise ValueError('value must be int/string/float') 1635 | self.registry[key] = value 1636 | return True 1637 | 1638 | def __contains__ (self, key): 1639 | return (key in self.registry) 1640 | 1641 | def __len__ (self): 1642 | return len(self.registry) 1643 | 1644 | def __getitem__ (self, key): 1645 | return self.registry[key] 1646 | 1647 | def __setitem__ (self, key, value): 1648 | self.set(key, value) 1649 | 1650 | def __iter__ (self): 1651 | return self.registry.__iter__() 1652 | 1653 | def keys (self): 1654 | return self.registry.keys() 1655 | 1656 | 1657 | #---------------------------------------------------------------------- 1658 | # json decode: safe for python 3.5 1659 | #---------------------------------------------------------------------- 1660 | def json_loads(text): 1661 | if sys.version_info[0] == 3 and sys.version_info[1] < 7: 1662 | if isinstance(text, bytes): 1663 | text = text.decode('utf-8') 1664 | return json.loads(text) 1665 | 1666 | 1667 | #---------------------------------------------------------------------- 1668 | # misc functions 1669 | #---------------------------------------------------------------------- 1670 | 1671 | # calling fzf 1672 | def fzf_execute(input, args = None, fzf = None): 1673 | import tempfile 1674 | code = 0 1675 | output = None 1676 | args = args is not None and args or '' 1677 | fzf = fzf is not None and fzf or 'fzf' 1678 | with tempfile.TemporaryDirectory(prefix = 'fzf.') as dirname: 1679 | outname = os.path.join(dirname, 'output.txt') 1680 | if isinstance(input, list): 1681 | inname = os.path.join(dirname, 'input.txt') 1682 | with open(inname, 'wb') as fp: 1683 | content = '\n'.join([ str(n) for n in input ]) 1684 | fp.write(content.encode('utf-8')) 1685 | cmd = '%s %s < "%s" > "%s"'%(fzf, args, inname, outname) 1686 | elif isinstance(input, str): 1687 | cmd = '%s | %s %s > "%s"'%(input, fzf, args, outname) 1688 | code = os.system(cmd) 1689 | if os.path.exists(outname): 1690 | with open(outname, 'rb') as fp: 1691 | output = fp.read() 1692 | if output is not None: 1693 | output = output.decode('utf-8') 1694 | if code != 0: 1695 | return None 1696 | return output 1697 | 1698 | 1699 | #---------------------------------------------------------------------- 1700 | # write application level log 1701 | #---------------------------------------------------------------------- 1702 | def mlog(*args): 1703 | import sys, codecs, os, time 1704 | now = time.strftime('%Y-%m-%d %H:%M:%S') 1705 | part = [ str(n) for n in args ] 1706 | text = u' '.join(part) 1707 | logfile = sys.modules[__name__].__dict__.get('_mlog_file', None) 1708 | encoding = sys.modules[__name__].__dict__.get('_mlog_encoding', 'utf-8') 1709 | stdout = sys.modules[__name__].__dict__.get('_mlog_stdout', True) 1710 | if logfile is None: 1711 | name = os.path.abspath(sys.argv[0]) 1712 | name = os.path.splitext(name)[0] + '.log' 1713 | logfile = codecs.open(name, 'a', encoding = encoding, errors = 'ignore') 1714 | sys.modules[__name__]._mlog_file = logfile 1715 | content = '[%s] %s'%(now, text) 1716 | if logfile: 1717 | logfile.write(content + '\r\n') 1718 | logfile.flush() 1719 | sys.stdout.write(content + '\n') 1720 | return 0 1721 | 1722 | 1723 | #---------------------------------------------------------------------- 1724 | # testing case 1725 | #---------------------------------------------------------------------- 1726 | if __name__ == '__main__': 1727 | def test1(): 1728 | code, data, headers = http_request('http://www.baidu.com') 1729 | for k, v in headers: 1730 | print('%s: %s'%(k, v)) 1731 | print(web.IsFastCGI()) 1732 | print(code) 1733 | return 0 1734 | def test2(): 1735 | config = ConfigReader('e:/lab/casuald/conf/echoserver.ini') 1736 | print(config.option('transmod', 'portu')) 1737 | return 0 1738 | def test3(): 1739 | trace = TraceOut('m') 1740 | trace.info('haha', 'mama') 1741 | return 0 1742 | def test4(): 1743 | log = TraceOut('m') 1744 | def loop(): 1745 | time.sleep(2) 1746 | log.info('loop') 1747 | return False 1748 | safe_loop(loop, log) 1749 | return 0 1750 | def test5(): 1751 | reg = Registry('output.json') 1752 | import pprint 1753 | pprint.pprint(reg.registry) 1754 | reg.set('home.default', 1234) 1755 | reg.set('target.abc', 'asdfasdf') 1756 | reg.set('home.haha', 'hiahia') 1757 | reg.set('target.pi', 3.1415926) 1758 | # reg.save() 1759 | return 0 1760 | def test6(): 1761 | print(utils.find_root(__file__)) 1762 | print(utils.project_root('/')) 1763 | utils.print_binary('Hello, World !! Ni Hao !!', True) 1764 | print(utils.getopt(['-t', '--name=123', '--out', '-', 'abc', 'def', 'ghi'])) 1765 | print(utils.getopt([])) 1766 | print(web.replace_range('Hello, World', 4, 2, 'fuck')) 1767 | url = 'socks5://test:pass@localhost/tt?123=45' 1768 | res = web.url_parse(url) 1769 | print(res) 1770 | print(res.hostname) 1771 | print(res.port) 1772 | print(res.username) 1773 | print(res.password) 1774 | return 0 1775 | test6() 1776 | 1777 | 1778 | 1779 | -------------------------------------------------------------------------------- /lib/config.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | #====================================================================== 4 | # 5 | # config.py - 6 | # 7 | # Created by skywind on 2019/05/04 8 | # Last Modified: 2019/05/04 23:07:13 9 | # 10 | #====================================================================== 11 | from __future__ import print_function, unicode_literals 12 | import sys 13 | import os 14 | import ascmini 15 | 16 | 17 | #---------------------------------------------------------------------- 18 | # loading 19 | #---------------------------------------------------------------------- 20 | ININAME = '~/.config/markpress/config.ini' 21 | ININAME = os.path.abspath(os.path.expanduser(ININAME)) 22 | PRESENT = os.path.exists(ININAME) 23 | MARKPRESS = os.environ.get('MARKPRESS', '').strip() 24 | 25 | cfg = ascmini.ConfigReader(ININAME) 26 | options = {} 27 | 28 | 29 | #---------------------------------------------------------------------- 30 | # default config 31 | #---------------------------------------------------------------------- 32 | options['engine'] = cfg.option('default', 'engine', '').strip() 33 | options['tabsize'] = cfg.option('default', 'tabsize', 4) 34 | options['encoding'] = cfg.option('default', 'encoding', '').strip() 35 | options['graphviz'] = cfg.option('default', 'graphviz', '').strip() 36 | options['path'] = cfg.option('default', 'path', '').strip() 37 | options['extensions'] = cfg.option('default', 'extensions', '').strip() 38 | options['extras'] = cfg.option('default', 'extras', '').strip() 39 | options['proxy'] = None 40 | 41 | 42 | #---------------------------------------------------------------------- 43 | # select 44 | #---------------------------------------------------------------------- 45 | def select(section): 46 | if not PRESENT: 47 | raise FileNotFoundError("missing: " + ININAME) 48 | if section not in cfg.config: 49 | raise ValueError("config section missing: " + section) 50 | options['url'] = cfg.option(section, 'url', '').strip() 51 | options['user'] = cfg.option(section, 'user', '').strip() 52 | options['passwd'] = cfg.option(section, 'passwd', '').strip() 53 | options['blog'] = cfg.option(section, 'blog', '').strip() 54 | options['proxy'] = cfg.option(section, 'proxy', '').strip() 55 | if not options['url']: 56 | raise ValueError('config error: empty url') 57 | if not options['user']: 58 | raise ValueError('config error: empty user') 59 | return True 60 | 61 | try: 62 | select('0') 63 | except: 64 | pass 65 | 66 | if MARKPRESS: 67 | select(MARKPRESS) 68 | 69 | 70 | #---------------------------------------------------------------------- 71 | # fatal 72 | #---------------------------------------------------------------------- 73 | def fatal(message, code = 1): 74 | message = message.rstrip('\n') 75 | sys.stderr.write('Fatal: ' + message + '\n') 76 | sys.stderr.flush() 77 | sys.exit(code) 78 | return 0 79 | 80 | 81 | #---------------------------------------------------------------------- 82 | # output error 83 | #---------------------------------------------------------------------- 84 | def perror(fname, line, text): 85 | sys.stderr.write('%s:%d: error: %s\n'%(fname, line, text)) 86 | sys.stderr.flush() 87 | return 0 88 | 89 | 90 | #---------------------------------------------------------------------- 91 | # template 92 | #---------------------------------------------------------------------- 93 | template = {} 94 | 95 | def _load_template(): 96 | names = ['style.css', 'header.html', 'footer.html'] 97 | home = os.path.expanduser('~/.config/markpress') 98 | for name in names: 99 | fn = os.path.join(home, name) 100 | text = ascmini.posix.load_file_text(fn) 101 | if text: 102 | text = '\n'.join([ t.rstrip('\r\n') for t in text.split('\n') ]) 103 | template[name] = text 104 | # print('name', len(template[name] and template[name] or '')) 105 | return True 106 | 107 | _load_template() 108 | 109 | 110 | #---------------------------------------------------------------------- 111 | # use proxy 112 | #---------------------------------------------------------------------- 113 | def proxy(url): 114 | url = url.strip() 115 | import socket 116 | if '_socket_' not in socket.__dict__: 117 | socket._socket_ = socket.socket 118 | if (not url) or (url in ('raw', 'tcp', '', 'native')): 119 | socket.socket = socket._socket_ 120 | return True 121 | try: 122 | import socks 123 | except ImportError: 124 | fatal('PySocks module is required') 125 | return False 126 | res = ascmini.web.url_parse(url) 127 | protocol = socks.HTTP 128 | if res.scheme == 'socks4': 129 | protocol = socks.SOCKS4 130 | elif res.scheme in ('socks5', 'socks'): 131 | protocol = socks.SOCKS5 132 | port = res.port 133 | if not port: 134 | port = (protocol == socks.HTTP) and 80 or 1080 135 | args = [protocol, res.hostname, port, True, res.username, res.password] 136 | socks.set_default_proxy(*args) 137 | socket.socket = socks.socksocket 138 | return 0 139 | 140 | 141 | 142 | #---------------------------------------------------------------------- 143 | # create wp 144 | #---------------------------------------------------------------------- 145 | def wp_client(): 146 | import wordpress2 147 | url = options['url'] 148 | wp = wordpress2.WordPress(url, options['user'], options['passwd']) 149 | return wp 150 | 151 | 152 | #---------------------------------------------------------------------- 153 | # testing suit 154 | #---------------------------------------------------------------------- 155 | if __name__ == '__main__': 156 | def test1(): 157 | print(ININAME) 158 | print(options) 159 | return 0 160 | def test2(): 161 | proxy('socks5://localhost/') 162 | proxy('http://localhost/') 163 | proxy('socks5://linwei:1234@localhost/') 164 | proxy('socks5://linwei:1234:12@localhost/') 165 | fatal("test") 166 | return 0 167 | def test3(): 168 | # proxy('socks5://localhost:1080') 169 | url = 'https://www.google.com' 170 | print(ascmini.http_request(url)) 171 | test2() 172 | 173 | 174 | 175 | -------------------------------------------------------------------------------- /lib/nextpress.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | #====================================================================== 4 | # 5 | # nextpress.py - 6 | # 7 | # Created by skywind on 2019/05/05 8 | # Last Modified: 2019/05/05 16:50:15 9 | # 10 | #====================================================================== 11 | from __future__ import print_function, unicode_literals 12 | import sys 13 | import os 14 | import time 15 | import config 16 | import ascmini 17 | import utils 18 | 19 | 20 | #---------------------------------------------------------------------- 21 | # markdown render 22 | #---------------------------------------------------------------------- 23 | def markpress_render(doc): 24 | html = doc.convert('config') 25 | try: 26 | import render 27 | except ImportError: 28 | return html 29 | hr = render.HtmlRender(html) 30 | hr.process_viz() 31 | return hr.render() 32 | 33 | 34 | #---------------------------------------------------------------------- 35 | # load markdown 36 | #---------------------------------------------------------------------- 37 | def markpress_load(filename): 38 | if not os.path.exists(filename): 39 | config.fatal('file not find: ' + filename) 40 | doc = utils.MarkdownDoc(filename) 41 | if not doc._uuid: 42 | config.perror(filename, 1, 'uuid not find in the markdown meta-header') 43 | return None 44 | uuid = doc._uuid.strip() 45 | if not uuid.isdigit(): 46 | config.perror(filename, 1, 'invalid uuid %s'%uuid) 47 | return None 48 | return doc 49 | 50 | 51 | #---------------------------------------------------------------------- 52 | # page maker 53 | #---------------------------------------------------------------------- 54 | def markpress_page_make(html, title): 55 | output = '' 56 | output += '\n\n' 57 | output += '\n' 58 | if title: 59 | text = ascmini.web.text2html(title) 60 | output += '%s\n'%text 61 | css = config.template['style.css'] 62 | if css: 63 | output += '\n' 66 | header = config.template['header.html'] 67 | if header: 68 | output += header 69 | output += '\n' 70 | output += '\n\n\n\n' 71 | output += '\n' 72 | output += html 73 | output += '\n\n' 74 | output += '\n\n' 75 | footer = config.template['footer.html'] 76 | if footer: 77 | output += footer 78 | output += '\n' 79 | output += '\n\n\n\n' 80 | return output 81 | 82 | 83 | #---------------------------------------------------------------------- 84 | # update file 85 | #---------------------------------------------------------------------- 86 | def markpress_update(filename): 87 | doc = markpress_load(filename) 88 | if not doc: 89 | return -1 90 | uuid = doc._uuid 91 | post = {} 92 | post['id'] = uuid 93 | post['title'] = doc._title 94 | post['content'] = markpress_render(doc) 95 | status = doc._status 96 | if status not in ('', 'draft', 'private', 'publish'): 97 | config.perror(filename, 1, 'invalid status %s'%status) 98 | return -3 99 | post['status'] = status and status or 'draft' 100 | if doc._cats: 101 | post['category'] = doc._cats 102 | if doc._tags: 103 | post['tag'] = doc._tags 104 | if doc._slug: 105 | post['slug'] = doc._slug 106 | if doc._date: 107 | post['date'] = utils.utc_datetime(doc._date) 108 | wp = config.wp_client() 109 | wp.post_edit(post) 110 | pp = wp.post_get(uuid) 111 | print('post uuid=%s updated: %s'%(uuid, filename)) 112 | print('%s'%pp.link) 113 | return 0 114 | 115 | 116 | #---------------------------------------------------------------------- 117 | # fetch info 118 | #---------------------------------------------------------------------- 119 | def markpress_info(filename): 120 | doc = markpress_load(filename) 121 | if not doc: 122 | return -1 123 | uuid = doc._uuid 124 | wp = config.wp_client() 125 | pp = wp.post_get(uuid) 126 | print('uuid: %s'%uuid) 127 | print('title: %s'%doc._title) 128 | print('link: %s'%pp.link) 129 | return 0 130 | 131 | 132 | #---------------------------------------------------------------------- 133 | # convert 134 | #---------------------------------------------------------------------- 135 | def markpress_compile(filename, outname): 136 | doc = markpress_load(filename) 137 | if not doc: 138 | return -1 139 | doc._html = markpress_render(doc) 140 | content = markpress_page_make(doc._html, doc._title) 141 | if (not outname) or (outname == '-'): 142 | fp = sys.stdout 143 | else: 144 | import codecs 145 | fp = codecs.open(outname, 'w', encoding = 'utf-8') 146 | fp.write(content) 147 | if fp != sys.stdout: 148 | fp.close() 149 | return 0 150 | 151 | 152 | #---------------------------------------------------------------------- 153 | # open in browser 154 | #---------------------------------------------------------------------- 155 | def markpress_open(filename, preview): 156 | doc = markpress_load(filename) 157 | if not doc: 158 | return -1 159 | url = doc.link() 160 | if preview: 161 | url = url + '&preview=true' 162 | import subprocess 163 | subprocess.call(['cmd.exe', '/C', 'start', url]) 164 | return 0 165 | 166 | 167 | #---------------------------------------------------------------------- 168 | # 169 | #---------------------------------------------------------------------- 170 | def markpress_preview(filename): 171 | markpress_open(filename, True) 172 | return 0 173 | 174 | 175 | #---------------------------------------------------------------------- 176 | # main 177 | #---------------------------------------------------------------------- 178 | def main(argv = None): 179 | if argv is None: 180 | argv = sys.argv 181 | options, args = ascmini.utils.getopt(argv[1:]) 182 | if not config.PRESENT: 183 | config.fatal('missing config: ' + config.ININAME) 184 | if 'site' in options: 185 | site = options['site'].strip() 186 | if site: 187 | if site not in config.cfg.config: 188 | config.fatal('config section mission: ' + site) 189 | try: 190 | config.select(site) 191 | except ValueError as e: 192 | config.fatal(str(e)) 193 | if config.options['proxy']: 194 | config.proxy(config.options['proxy']) 195 | if 'h' in options or 'help' in options: 196 | if 'n' in options or 'new' in options: 197 | print('usage: markpress {-n --new} [--site=SITE] ') 198 | print('Create a new post and save it to file. Dump to stdout') 199 | print('if filename is a hyphen (-).') 200 | elif 'u' in options or 'update' in options: 201 | print('usage: markpress {-u --update} [--site=SITE] ') 202 | print('Update file to wordpress server') 203 | elif 'i' in options or 'info' in options: 204 | print('usage: markpress {-i --info} [--site=SITE] ') 205 | print('Get post info') 206 | elif 'c' in options or 'compile' in options: 207 | print('usage: markpress {-c --compile} [--site=SITE] [outname]') 208 | print('Compile markdown to html') 209 | elif 'o' in options or 'open' in options: 210 | print('usage: markpress {-o --open} ') 211 | print('Open post in browser') 212 | elif 'p' in options or 'preview' in options: 213 | print('usage: markpress {-p --preview} [--site=SITE] ') 214 | print('Preview markdown') 215 | else: 216 | config.fatal('what help do you need ?') 217 | elif 'n' in options or 'new' in options: 218 | if not args: 219 | config.fatal('missing file name') 220 | name = args[0] 221 | if name == '-': 222 | fp = sys.stdout 223 | elif os.path.exists(name): 224 | if 'f' not in options: 225 | config.fatal('file already exists: ' + name) 226 | wp = config.wp_client() 227 | pid = wp.post_new() 228 | pp = wp.post_get(pid) 229 | if name != '-': 230 | import codecs 231 | fp = codecs.open(name, 'w', encoding = 'utf-8') 232 | fp.write('---\n') 233 | fp.write('uuid: ' + str(pid) + '\n') 234 | fp.write('title: \n') 235 | fp.write('status: draft\n') 236 | fp.write('categories: \n') 237 | fp.write('tags: \n') 238 | fp.write('slug: \n') 239 | fp.write('---\n\n') 240 | if name != '-': 241 | print('new post uuid=%s saved in %s'%(pid, name)) 242 | print(pp.link) 243 | elif 'u' in options or 'update' in options: 244 | if not args: 245 | config.fatal('missing file name') 246 | markpress_update(args[0]) 247 | elif 'i' in options or 'info' in options: 248 | if not args: 249 | config.fatal('missing file name') 250 | markpress_info(args[0]) 251 | elif 'c' in options or 'compile' in options: 252 | if not args: 253 | config.fatal('missing file name') 254 | if len(args) >= 2: 255 | outname = args[1] 256 | else: 257 | outname = os.path.splitext(args[0])[0] + '.html' 258 | markpress_compile(args[0], outname) 259 | elif 'o' in options or 'open' in options: 260 | if not args: 261 | config.fatal('missing file name') 262 | markpress_open(args[0], False) 263 | elif 'p' in options or 'preview' in options: 264 | if not args: 265 | config.fatal('missing file name') 266 | markpress_preview(args[0]) 267 | else: 268 | print('usage: markpress [...]') 269 | print('operations:') 270 | print(' markpress {-n --new} ') 271 | print(' markpress {-u --update} ') 272 | print(' markpress {-i --info} ') 273 | print(' markpress {-c --compile} [outname]') 274 | print(' markpress {-o --open} ') 275 | print(' markpress {-p --preview} ') 276 | print() 277 | print("use 'markpress {-h --help}' with an operation for detail") 278 | return 0 279 | 280 | 281 | #---------------------------------------------------------------------- 282 | # testing suit 283 | #---------------------------------------------------------------------- 284 | if __name__ == '__main__': 285 | def test1(): 286 | args = ['', '-n', '-f', '../content/1.md'] 287 | args = ['', '-u', '../content/1.md'] 288 | main(args) 289 | return 0 290 | def test2(): 291 | args = ['', '-i', '../content/1.md'] 292 | args = ['', '-c', '../content/1.md'] 293 | # args = ['', '-c', '../content/1.md', '-'] 294 | main(args) 295 | return 0 296 | def test9(): 297 | args = ['', '-n', '-'] 298 | args = [] 299 | args = ['', '-h', '-n'] 300 | main(args) 301 | return 0 302 | test1() 303 | # main() 304 | 305 | 306 | 307 | -------------------------------------------------------------------------------- /lib/render.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | #====================================================================== 4 | # 5 | # render.py - 6 | # 7 | # Created by skywind on 2019/05/06 8 | # Last Modified: 2019/05/06 23:10:23 9 | # 10 | #====================================================================== 11 | from __future__ import print_function, unicode_literals 12 | import sys 13 | import os 14 | import bs4 15 | import ascmini 16 | import config 17 | 18 | 19 | #---------------------------------------------------------------------- 20 | # 2/3 compatible 21 | #---------------------------------------------------------------------- 22 | if sys.version_info[0] >= 3: 23 | unicode = str 24 | 25 | 26 | #---------------------------------------------------------------------- 27 | # INTERNAL 28 | #---------------------------------------------------------------------- 29 | ENGINES = ['dot', 'circo', 'neato', 'osage', 'twopi'] 30 | VIZPATH = [] 31 | 32 | SCRIPT_ENABLE = True 33 | SCRIPT_ENCODING = None 34 | SCRIPT_PATH = [] 35 | 36 | 37 | #---------------------------------------------------------------------- 38 | # GraphViz 39 | #---------------------------------------------------------------------- 40 | def graphviz(engine, text): 41 | if engine not in ENGINES: 42 | raise ValueError('Invalid Engine: %s'%engine) 43 | path = [ n for n in VIZPATH ] 44 | if config.options['graphviz']: 45 | viz = config.options['graphviz'] 46 | if os.path.isdir(viz): 47 | path.append(viz) 48 | exe = ascmini.posix.search_cmd(engine, path) 49 | if not exe: 50 | raise FileNotFoundError('Missing GraphViz executable: ' + engine) 51 | args = [exe, '-Tsvg'] 52 | code, stdout, stderr = ascmini.call(args, text) 53 | if stdout: 54 | stdout = stdout.decode('utf-8', 'ignore') 55 | if stderr: 56 | stderr = stderr.decode('utf-8', 'ignore') 57 | if code != 0: 58 | raise ChildProcessError('error: %s: %s'%(engine, stderr)) 59 | return stdout 60 | 61 | 62 | #---------------------------------------------------------------------- 63 | # External 64 | #---------------------------------------------------------------------- 65 | def script_eval(script, text, binary = False): 66 | path = [ n for n in SCRIPT_PATH ] 67 | if config.options['path']: 68 | cmd = config.options['path'] 69 | if os.path.isdir(cmd): 70 | path.append(cmd) 71 | exe = ascmini.posix.search_cmd(script, path) 72 | if not exe: 73 | raise FileNotFoundError('Missing executable: ' + script) 74 | encoding = config.options['encoding'] 75 | if not encoding: 76 | encoding = SCRIPT_ENCODING 77 | if not encoding: 78 | try: 79 | import locale 80 | encoding = locale.getdefaultlocale()[1] 81 | except: 82 | encoding = sys.getdefaultencoding() 83 | if not isinstance(text, bytes): 84 | text = text.encode(encoding, 'ignore') 85 | code, stdout, stderr = ascmini.call([exe], text) 86 | if not binary: 87 | if stdout: 88 | stdout = stdout.decode(encoding, 'ignore') 89 | if stderr: 90 | stderr = stderr.decode(encoding, 'ignore') 91 | if code != 0: 92 | raise ChildProcessError('error: %s: %s'%(script, stderr)) 93 | return stdout 94 | 95 | 96 | #---------------------------------------------------------------------- 97 | # render html 98 | #---------------------------------------------------------------------- 99 | class HtmlRender (object): 100 | 101 | def __init__ (self, html): 102 | self._origin_html = html 103 | self._soup = bs4.BeautifulSoup(html, 'html.parser') 104 | 105 | def process_viz (self): 106 | soup = self._soup 107 | for pre in soup.find_all('pre'): 108 | code = pre.code 109 | if code is None: 110 | continue 111 | if 'class' not in code.attrs: 112 | continue 113 | if not code['class']: 114 | continue 115 | engine = None 116 | for cls in code['class']: 117 | if cls.startswith('viz-'): 118 | name = cls[4:] 119 | if name in ENGINES: 120 | engine = name 121 | break 122 | elif cls.startswith('cmd-') and SCRIPT_ENABLE: 123 | name = str(cls).strip() 124 | mode, _, _ = name[4:].partition('-') 125 | if mode.strip() in ('text', 'html', 'png', 'jpeg'): 126 | engine = name 127 | break 128 | if engine is None: 129 | continue 130 | if engine.startswith('cmd-'): 131 | tag = self._cmd_replace(engine, code.text) 132 | else: 133 | tag = self._viz_replace(engine, code.text) 134 | pre.insert_before(tag) 135 | pre.decompose() 136 | return 0 137 | 138 | def _cmd_replace (self, engine, text): 139 | mode, _, script = engine[4:].strip().partition('-') 140 | mode = mode.strip() 141 | binary = (mode in ('png', 'jpeg')) 142 | try: 143 | output = script_eval(script.strip(), text, binary) 144 | if mode == 'text': 145 | p = self._soup.new_tag('pre') 146 | code = self._soup.new_tag('code') 147 | p.insert(0, code) 148 | code.string = output 149 | elif mode == 'html': 150 | soup = bs4.BeautifulSoup(output, 'html.parser') 151 | p = self._soup.new_tag('p') 152 | p.insert(0, soup) 153 | else: 154 | import base64 155 | html = '%s\n'%script 164 | soup = bs4.BeautifulSoup(html, 'html.parser') 165 | p = self._soup.new_tag('p') 166 | p.insert(0, soup) 167 | except: 168 | text = ascmini.callstack() 169 | pre = self._soup.new_tag('pre') 170 | code = self._soup.new_tag('code') 171 | pre.insert(0, code) 172 | code.string = text 173 | return pre 174 | return p 175 | 176 | def _viz_replace (self, engine, text): 177 | try: 178 | output = graphviz(engine, text) 179 | soup = bs4.BeautifulSoup(output, 'html.parser') 180 | # soup = bs4.BeautifulSoup(output, 'lxml') 181 | svg = soup.svg.extract() 182 | p = self._soup.new_tag('pre') 183 | p.insert(0, svg) 184 | p['style'] = 'background:none; border:0px;' 185 | # print(svg) 186 | except: 187 | text = ascmini.callstack() 188 | pre = self._soup.new_tag('pre') 189 | code = self._soup.new_tag('code') 190 | pre.insert(0, code) 191 | code.string = text 192 | return pre 193 | return p 194 | 195 | def render (self): 196 | # return self._soup.prettify() 197 | return unicode(self._soup) 198 | 199 | 200 | 201 | #---------------------------------------------------------------------- 202 | # testing suit 203 | #---------------------------------------------------------------------- 204 | if __name__ == '__main__': 205 | def test1(): 206 | import utils 207 | VIZPATH.append('d:/dev/tools/graphviz/bin') 208 | doc = utils.MarkdownDoc('../content/test2.md') 209 | html = doc.convert('') 210 | hr = HtmlRender(html) 211 | hr.process_viz() 212 | print(hr.render()) 213 | # print(unicode(soup)) 214 | return 0 215 | def test2(): 216 | print(ascmini.posix.search_cmd('gcc', ['d:/dev/tools/graphviz/bin'])) 217 | VIZPATH.append('d:/dev/tools/graphviz/bin') 218 | text = 'digraph G {\n A -> B\nB -> C\nB -> D\n}' 219 | output = graphviz('dot', text) 220 | print(output) 221 | return 0 222 | def test3(): 223 | t = script_eval('d:/filter2', 'test', False) 224 | print('-----') 225 | print(t) 226 | return 0 227 | def test4(): 228 | import utils 229 | SCRIPT_PATH.append('e:/site/markpress/test/filters') 230 | global SCRIPT_ENCODING 231 | # SCRIPT_ENCODING = 'utf-8' 232 | doc = utils.MarkdownDoc('../test/1.md') 233 | html = doc.convert('markdown') 234 | # html = doc.convert('') 235 | hr = HtmlRender(html) 236 | hr.process_viz() 237 | print(hr.render()) 238 | return 0 239 | test4() 240 | 241 | 242 | 243 | -------------------------------------------------------------------------------- /lib/utils.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | #====================================================================== 4 | # 5 | # utils.py - 6 | # 7 | # Created by skywind on 2019/05/04 8 | # Last Modified: 2019/05/04 22:58:10 9 | # 10 | #====================================================================== 11 | from __future__ import print_function, unicode_literals 12 | import sys 13 | import os 14 | import time 15 | import config 16 | import ascmini 17 | import utime 18 | 19 | 20 | #---------------------------------------------------------------------- 21 | # extras 22 | #---------------------------------------------------------------------- 23 | MD_EXTRAS = ['metadata', 'fenced-code-blocks', 'cuddled-list', 24 | 'tables', 'footnotes', 'highlightjs-lang', 'target-blank-links', 25 | 'use-file-vars', 'code-friendly'] 26 | 27 | PANDOC_FLAGS = ['--no-highlight'] 28 | 29 | PANDOC_EXTENSION = ['fancy_lists', 'fenced_code_blocks', 30 | 'fenced_code_attributes'] 31 | 32 | PYMD_EXTENSION = [ 'fenced_code', 'footnotes', 'tables', 'meta' ] 33 | 34 | 35 | #---------------------------------------------------------------------- 36 | # Error 37 | #---------------------------------------------------------------------- 38 | class ConvertError (ValueError): 39 | pass 40 | 41 | 42 | #---------------------------------------------------------------------- 43 | # MarkdownDoc 44 | #---------------------------------------------------------------------- 45 | class MarkdownDoc (object): 46 | 47 | def __init__ (self, filename): 48 | self._filename = os.path.abspath(filename) 49 | self._content = ascmini.posix.load_file_text(filename) 50 | self._html = None 51 | self._meta = {} 52 | self._cats = [] 53 | self._tags = [] 54 | self._uuid = None 55 | self._title = None 56 | self._error = None 57 | self._status = None 58 | self.__parse() 59 | 60 | def __parse_list (self, text): 61 | if not text: 62 | text = '' 63 | if isinstance(text, bytes): 64 | text = text.decode('utf-8', 'ignore') 65 | if '\uff0c' in text: 66 | text = text.replace('\uff0c', ',') 67 | parts = [] 68 | for part in text.split(','): 69 | part = part.strip() 70 | if part: 71 | parts.append(part) 72 | return parts 73 | 74 | def __parse_meta (self, content): 75 | state = 0 76 | meta = {} 77 | size = len(content) 78 | pos = 0 79 | while pos < size: 80 | end = content.find('\n', pos) 81 | if end < 0: 82 | end = size 83 | line = content[pos:end] 84 | pos = end + 1 85 | line = line.rstrip('\r\n\t ') 86 | if not line: 87 | continue 88 | if state == 0: 89 | if line == ('-' * len(line)) and len(line) >= 3: 90 | state = 1 91 | else: 92 | break 93 | elif state == 1: 94 | if line == ('-' * len(line)) and len(line) >= 3: 95 | state = 2 96 | elif ':' in line: 97 | key, _, value = line.partition(':') 98 | key = key.strip() 99 | if key: 100 | meta[key] = value.strip() 101 | else: 102 | break 103 | return meta 104 | 105 | def __parse (self): 106 | self._meta = self.__parse_meta(self._content) 107 | self._uuid = self._meta.get('uuid', None) 108 | self._title = self._meta.get('title', None) 109 | self._cats = self.__parse_list(self._meta.get('categories')) 110 | self._tags = self.__parse_list(self._meta.get('tags')) 111 | self._slug = self._meta.get('slug', None) 112 | self._date = self._meta.get('date', None) 113 | self._status = self._meta.get('status', 'draft') 114 | if not self._uuid: 115 | self._uuid = None 116 | if not self._title: 117 | self._title = None 118 | if not self._cats: 119 | self._cats = None 120 | if not self._tags: 121 | self._tags = None 122 | return True 123 | 124 | def _fenced_code_block (self, content, tabsize = 4): 125 | import re 126 | output = [] 127 | source = [] 128 | state = 0 129 | mark = '' 130 | lang = None 131 | p1 = re.compile(r'^\s{0,3}```*') 132 | p2 = re.compile(r'^\s{0,3}~~~*') 133 | for line in content.split('\n'): 134 | line = line.rstrip('\r\n\t ') 135 | if state == 0: 136 | m1 = re.match(p1, line) 137 | m2 = re.match(p2, line) 138 | mm = m1 and m1 or m2 139 | if not mm: 140 | output.append(line) 141 | continue 142 | span = mm.span() 143 | mark = line[:span[1]] 144 | lang = line[span[1]:].strip() 145 | state = 1 146 | source = [] 147 | elif state == 1: 148 | if not line.startswith(mark): 149 | source.append(line.expandtabs(tabsize)) 150 | continue 151 | src = '\n'.join(source).strip('\n') 152 | head = '

'
153 |                 if lang:
154 |                     head = '
'%lang
155 |                 replacements = [
156 |                     ("&", "&"),
157 |                     ("<", "<"),
158 |                     (">", ">"),
159 |                     ("`", "`"),
160 |                 ]
161 |                 for new, old in replacements:
162 |                     src = src.replace(old, new)
163 |                 output.append(head + src)
164 |                 output.append('
') 165 | state = 0 166 | return '\n'.join(output) 167 | 168 | def _convert_default (self, content): 169 | import markdown2 170 | tabsize = config.options['tabsize'] 171 | content = self._fenced_code_block(content, tabsize) 172 | extras = [ n for n in MD_EXTRAS ] 173 | if config.options['extras']: 174 | for n in config.options['extras'].split(','): 175 | extras.append(n.strip()) 176 | md = markdown2.Markdown(extras = extras, tab_width = tabsize) 177 | html = md.convert(content) 178 | if sys.version_info[0] >= 3: 179 | unicode = str 180 | text = unicode(html) 181 | return text 182 | 183 | # require: https://github.com/Python-Markdown/markdown/ 184 | def _convert_markdown (self, content): 185 | import markdown 186 | exts = [ n for n in PYMD_EXTENSION ] 187 | if config.options['extensions']: 188 | for n in config.options['extensions'].split(','): 189 | exts.append(n.strip()) 190 | path = os.path.expanduser('~/.config/markpress') 191 | name = os.path.join(path, 'extensions.py') 192 | tabsize = config.options['tabsize'] 193 | sys.path.insert(0, path) 194 | argv = {} 195 | argv['extensions'] = exts 196 | if os.path.exists(name): 197 | import extensions 198 | if 'extensions' in extensions.__dict__: 199 | exts.extend(extensions.extensions) 200 | if 'extension_configs' in extensions.__dict__: 201 | argv['extension_configs'] = extensions.extension_configs 202 | argv['tab_length'] = int(tabsize) 203 | html = markdown.markdown(content, **argv) 204 | return html 205 | 206 | def _convert_pandoc (self, content): 207 | tabsize = config.options['tabsize'] 208 | content = self._fenced_code_block(content, tabsize) 209 | input = content.encode('utf-8', 'ignore') 210 | args = ['pandoc', '-f', 'markdown', '-t', 'html'] 211 | args.extend(PANDOC_FLAGS) 212 | for exts in PANDOC_EXTENSION: 213 | if exts[:1] not in ('+', '-'): 214 | exts = '+' + exts 215 | args[2] += exts 216 | code, stdout, stderr = ascmini.call(args, input) 217 | self._error = None 218 | if code != 0: 219 | stderr = stderr.decode('utf-8', 'ignore') 220 | error = ConvertError("pandoc exits with code %d: %s" % ( 221 | code, stderr)) 222 | error.stdout = stderr 223 | raise error 224 | return stdout.decode('utf-8', 'ignore') 225 | 226 | # engine: native, markdown, pandoc, auto, config 227 | def convert (self, engine): 228 | if engine is None: 229 | engine = '' 230 | engine = engine.strip().lower() 231 | if engine in ('markdown2', 'default', '0', '', 'native', 0): 232 | engine = '' 233 | if engine == 'config': 234 | engine = config.options['engine'] 235 | if engine == 'auto': 236 | try: 237 | import markdown 238 | engine = 'markdown' 239 | except ImportError: 240 | engine = 'default' 241 | content = self._content 242 | if engine == 'markdown': 243 | return self._convert_markdown(content) 244 | elif engine == 'pandoc': 245 | return self._convert_pandoc(content) 246 | return self._convert_default(content) 247 | 248 | def link (self): 249 | url = config.options['url'] 250 | if not url.endswith('/'): 251 | url += '/' 252 | return url + '?' + self._uuid 253 | 254 | 255 | #---------------------------------------------------------------------- 256 | # utc datetime 257 | #---------------------------------------------------------------------- 258 | def utc_datetime(text): 259 | import datetime 260 | ts = utime.read_timestamp(text) 261 | return datetime.datetime.utcfromtimestamp(ts) 262 | 263 | 264 | #---------------------------------------------------------------------- 265 | # testing suit 266 | #---------------------------------------------------------------------- 267 | if __name__ == '__main__': 268 | def test1(): 269 | doc = MarkdownDoc('../content/test.md') 270 | print(doc._meta) 271 | print(doc._cats) 272 | print(doc._tags) 273 | print(doc.convert('')) 274 | return 0 275 | def test2(): 276 | text = ascmini.execute(['cmd', '/c', 'dir'], capture = True) 277 | print('---') 278 | print(text.decode('gbk')) 279 | return 0 280 | def test3(): 281 | import markdown2 282 | extras = MD_EXTRAS 283 | html = markdown2.markdown('\n`````text\n```cpp\ntext\n```\n`````\n', extras = extras) 284 | print(html) 285 | return 0 286 | def test4(): 287 | doc = MarkdownDoc('../test/3.md') 288 | html = doc.convert('') 289 | # html = doc.convert('markdown') 290 | html = doc.convert('pandoc') 291 | print(html) 292 | return 0 293 | test4() 294 | 295 | 296 | -------------------------------------------------------------------------------- /lib/utime.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | #====================================================================== 4 | # 5 | # utime.py - time related functions 6 | # 7 | # Created by skywind on 2018/08/02 8 | # Last Modified: 2018/08/02 15:55:48 9 | # 10 | #====================================================================== 11 | from __future__ import print_function 12 | import sys 13 | import time 14 | import datetime 15 | 16 | 17 | #---------------------------------------------------------------------- 18 | # python 2/3 compatible 19 | #---------------------------------------------------------------------- 20 | if sys.version_info[0] >= 3: 21 | long = int 22 | unicode = str 23 | xrange = range 24 | 25 | 26 | #---------------------------------------------------------------------- 27 | # timezone 28 | #---------------------------------------------------------------------- 29 | class timezone(datetime.tzinfo): 30 | """Backport of datetime.timezone. 31 | Notes 32 | ----- 33 | Backport of datetime.timezone for Python 2.7, from Python 3.6 34 | documentation (https://tinyurl.com/z4cegu9), copyright Python Software 35 | Foundation (https://docs.python.org/3/license.html) 36 | """ 37 | __slots__ = '_offset', '_name' 38 | 39 | # Sentinel value to disallow None 40 | _Omitted = object() 41 | 42 | def __new__(cls, offset, name=_Omitted): 43 | if not isinstance(offset, datetime.timedelta): 44 | raise TypeError("offset must be a timedelta") 45 | if name is cls._Omitted: 46 | if not offset: 47 | return cls.utc 48 | name = None 49 | elif not isinstance(name, str): 50 | raise TypeError("name must be a string") 51 | if not cls._minoffset <= offset <= cls._maxoffset: 52 | raise ValueError("offset must be a timedelta " 53 | "strictly between -timedelta(hours=24) and " 54 | "timedelta(hours=24).") 55 | if (offset.microseconds != 0 or offset.seconds % 60 != 0): 56 | raise ValueError("offset must be a timedelta " 57 | "representing a whole number of minutes") 58 | return cls._create(offset, name) 59 | 60 | @classmethod 61 | def _create(cls, offset, name=None): 62 | self = datetime.tzinfo.__new__(cls) 63 | self._offset = offset 64 | self._name = name 65 | return self 66 | 67 | def __getinitargs__(self): 68 | """pickle support""" 69 | if self._name is None: 70 | return (self._offset,) 71 | return (self._offset, self._name) 72 | 73 | def __eq__(self, other): 74 | if not isinstance(other, timezone): 75 | return False 76 | return self._offset == other._offset 77 | 78 | def __lt__(self, other): 79 | raise TypeError("'<' not supported between instances of" 80 | " 'datetime.timezone' and 'datetime.timezone'") 81 | 82 | def __hash__(self): 83 | return hash(self._offset) 84 | 85 | def __repr__(self): 86 | if self is self.utc: 87 | return '%s.%s.utc' % (self.__class__.__module__, 88 | self.__class__.__name__) 89 | if self._name is None: 90 | return "%s.%s(%r)" % (self.__class__.__module__, 91 | self.__class__.__name__, 92 | self._offset) 93 | return "%s.%s(%r, %r)" % (self.__class__.__module__, 94 | self.__class__.__name__, 95 | self._offset, self._name) 96 | 97 | def __str__(self): 98 | return self.tzname(None) 99 | 100 | def utcoffset(self, dt): 101 | if isinstance(dt, datetime.datetime) or dt is None: 102 | return self._offset 103 | raise TypeError("utcoffset() argument must be a datetime instance" 104 | " or None") 105 | 106 | def tzname(self, dt): 107 | if isinstance(dt, datetime.datetime) or dt is None: 108 | if self._name is None: 109 | return self._name_from_offset(self._offset) 110 | return self._name 111 | raise TypeError("tzname() argument must be a datetime instance" 112 | " or None") 113 | 114 | def dst(self, dt): 115 | if isinstance(dt, datetime.datetime) or dt is None: 116 | return None 117 | raise TypeError("dst() argument must be a datetime instance" 118 | " or None") 119 | 120 | def fromutc(self, dt): 121 | if isinstance(dt, datetime.datetime): 122 | if dt.tzinfo is not self: 123 | raise ValueError("fromutc: dt.tzinfo " 124 | "is not self") 125 | return dt + self._offset 126 | raise TypeError("fromutc() argument must be a datetime instance" 127 | " or None") 128 | 129 | _maxoffset = datetime.timedelta(hours=23, minutes=59) 130 | _minoffset = -_maxoffset 131 | 132 | @staticmethod 133 | def _name_from_offset(delta): 134 | if not delta: 135 | return 'UTC' 136 | if delta < datetime.timedelta(0): 137 | sign = '-' 138 | delta = -delta 139 | else: 140 | sign = '+' 141 | hours, rest = divmod(delta.total_seconds(), 3600) 142 | hours = int(hours) 143 | minutes = rest // datetime.timedelta(minutes=1).total_seconds() 144 | minutes = int(minutes) 145 | return 'UTC{}{:02d}:{:02d}'.format(sign, hours, minutes) 146 | 147 | 148 | timezone.utc = timezone._create(datetime.timedelta(0)) 149 | timezone.min = timezone._create(timezone._minoffset) 150 | timezone.max = timezone._create(timezone._maxoffset) 151 | timezone.cst = timezone._create(datetime.timedelta(hours = 8)) 152 | 153 | _EPOCH = datetime.datetime(1970, 1, 1, tzinfo=timezone.utc) 154 | 155 | 156 | if sys.version_info[0] < 3: 157 | datetime.timezone = timezone 158 | 159 | 160 | 161 | #---------------------------------------------------------------------- 162 | # Tools 163 | #---------------------------------------------------------------------- 164 | DATETIME_FMT = '%Y-%m-%d %H:%M:%S' 165 | 166 | 167 | #---------------------------------------------------------------------- 168 | # 1551884065 to datetime.dateime(2019, 3, 6, 22, 54, 25) 169 | #---------------------------------------------------------------------- 170 | def timestamp_to_datetime (ts, tz = None): 171 | return datetime.datetime.fromtimestamp(ts, tz) 172 | 173 | 174 | #---------------------------------------------------------------------- 175 | # datetime.datetime(2019, 3, 6, 22, 54, 25) -> 1551884065 176 | #---------------------------------------------------------------------- 177 | def datetime_to_timestamp (dt): 178 | if hasattr(dt, 'timestamp'): 179 | return dt.timestamp() 180 | epoch = datetime.datetime.fromtimestamp(0, dt.tzinfo) 181 | return (dt - epoch).total_seconds() 182 | 183 | 184 | #---------------------------------------------------------------------- 185 | # datetime.datetime(2019, 3, 6, 22, 54, 25) -> '2019-03-06 22:54:25' 186 | #---------------------------------------------------------------------- 187 | def datetime_to_string (dt, fmt = None): 188 | return dt.strftime(fmt and fmt or DATETIME_FMT) 189 | 190 | 191 | #---------------------------------------------------------------------- 192 | # '2019-03-06 22:54:25' -> datetime.datetime(2019, 3, 6, 22, 54, 25) 193 | #---------------------------------------------------------------------- 194 | def string_to_datetime (text, tz = None, fmt = None): 195 | dt = datetime.datetime.strptime(text, fmt and fmt or DATETIME_FMT) 196 | if tz: 197 | if hasattr(tz, 'localize'): 198 | # in case, we have pytz 199 | dt = tz.localize(dt) 200 | else: 201 | dt = dt.replace(tzinfo = tz) 202 | return dt 203 | 204 | 205 | #---------------------------------------------------------------------- 206 | # 1551884065 -> '2019-03-06 22:54:25' 207 | #---------------------------------------------------------------------- 208 | def timestamp_to_string (ts, tz = None): 209 | return datetime_to_string(timestamp_to_datetime(ts, tz)) 210 | 211 | 212 | #---------------------------------------------------------------------- 213 | # '2019-03-06 22:54:25' -> 1551884065 214 | #---------------------------------------------------------------------- 215 | def string_to_timestamp (text, tz = None): 216 | return datetime_to_timestamp(string_to_datetime(text, tz)) 217 | 218 | 219 | #---------------------------------------------------------------------- 220 | # 1551884065 -> '2019-03-06T22:54:25.000Z' 221 | #---------------------------------------------------------------------- 222 | def timestamp_to_iso (ts): 223 | dt = timestamp_to_datetime(ts, timezone.utc) 224 | return dt.strftime('%Y-%m-%dT%H:%M:%S.%fZ') 225 | 226 | 227 | #---------------------------------------------------------------------- 228 | # '2019-03-06T22:54:25.000Z' -> 1551884065 229 | #---------------------------------------------------------------------- 230 | def iso_to_timestamp (iso): 231 | iso = iso.strip() 232 | if iso[-1:].upper() != 'Z': 233 | raise ValueError('require an ISO 8601 UTC format') 234 | if iso[10:11].upper() != 'T': 235 | raise ValueError('require an ISO 8601 UTC format') 236 | if '.' in iso: 237 | if len(iso) >= 28: 238 | if iso[19:20] == '.': 239 | iso = iso[:26] + 'Z' 240 | else: 241 | raise ValueError('bad iso 8601 format') 242 | dt = datetime.datetime.strptime(iso[:-1], '%Y-%m-%dT%H:%M:%S.%f') 243 | else: 244 | if len(iso) == 17: 245 | iso = iso[:16] + ':00Z' 246 | elif len(iso) == 12: 247 | iso = iso[:11] + '00:00:00Z' 248 | dt = datetime.datetime.strptime(iso[:-1], '%Y-%m-%dT%H:%M:%S') 249 | dt = dt.replace(tzinfo = timezone.utc) 250 | return datetime_to_timestamp(dt) 251 | 252 | 253 | #---------------------------------------------------------------------- 254 | # read timestamp from various format 255 | #---------------------------------------------------------------------- 256 | def read_timestamp (timestamp, tz = None): 257 | if isinstance(timestamp, datetime.datetime): 258 | return datetime_to_timestamp(timestamp) 259 | elif isinstance(timestamp, int) or isinstance(timestamp, long): 260 | return timestamp 261 | elif isinstance(timestamp, float): 262 | return timestamp 263 | elif len(timestamp) == 10 and timestamp.isdigit(): 264 | return int(timestamp) 265 | elif len(timestamp) == 13 and timestamp.isdigit(): 266 | return float(timestamp) * 0.001 267 | elif timestamp.find('.') == 10 and timestamp[:10].isdigit(): 268 | return float(timestamp) 269 | elif timestamp[:1].isdigit() and timestamp[4:5] == '-': 270 | if len(timestamp) == 19: 271 | return string_to_timestamp(timestamp, tz) 272 | elif len(timestamp) == 16: 273 | return string_to_timestamp(timestamp + ':00', tz) 274 | elif len(timestamp) == 10: 275 | return string_to_timestamp(timestamp + ' 00:00:00', tz) 276 | if timestamp[:1].isdigit() and timestamp[-1:] == 'Z': 277 | return iso_to_timestamp(timestamp) 278 | return timestamp 279 | 280 | 281 | 282 | #---------------------------------------------------------------------- 283 | # compact string: YYYYMMDDHHMM 284 | #---------------------------------------------------------------------- 285 | def compact_from_timestamp(ts, tz = None, seconds = False): 286 | fmt1 = '%Y%m%d%H%M' 287 | fmt2 = '%Y%m%d%H%M%S' 288 | dt = timestamp_to_datetime(ts, tz) 289 | return dt.strftime(seconds and fmt2 or fmt1) 290 | 291 | 292 | #---------------------------------------------------------------------- 293 | # utc to local without tz 294 | #---------------------------------------------------------------------- 295 | def utc_to_local(utc): 296 | epoch = time.mktime(utc.timetuple()) 297 | offset = datetime.datetime.fromtimestamp(epoch) - \ 298 | datetime.datetime.utcfromtimestamp(epoch) 299 | return utc + offset 300 | 301 | 302 | #---------------------------------------------------------------------- 303 | # local to utc 304 | #---------------------------------------------------------------------- 305 | def local_to_utc(local): 306 | epoch = time.mktime(local.timetuple()) 307 | offset = datetime.datetime.fromtimestamp(epoch) - \ 308 | datetime.datetime.utcfromtimestamp(epoch) 309 | return local - offset 310 | 311 | 312 | #---------------------------------------------------------------------- 313 | # testing case 314 | #---------------------------------------------------------------------- 315 | if __name__ == '__main__': 316 | def test1(): 317 | ts = time.time() 318 | uc = datetime.datetime.utcfromtimestamp(ts) 319 | uc = timestamp_to_datetime(ts, timezone.utc) 320 | print(ts) 321 | print(uc.tzinfo) 322 | print(datetime_to_timestamp(uc)) 323 | return 0 324 | def test2(): 325 | ts = time.time() 326 | print(ts) 327 | text = timestamp_to_iso(ts) 328 | print(timestamp_to_iso(ts)) 329 | print(iso_to_timestamp(text)) 330 | tt = '2018-08-31T09:45:56.0000000Z' 331 | print(len(tt)) 332 | ts = iso_to_timestamp(tt) 333 | dt = timestamp_to_datetime(ts, timezone.utc) 334 | print(dt) 335 | print(read_timestamp('2018-01-01 12:00:32')) 336 | def test3(): 337 | dt = datetime.datetime.fromtimestamp(time.time()) 338 | print(dt) 339 | utc = local_to_utc(dt) 340 | print(utc) 341 | print(utc_to_local(utc)) 342 | return 0 343 | test3() 344 | 345 | 346 | 347 | -------------------------------------------------------------------------------- /lib/wordpress2.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | #====================================================================== 4 | # 5 | # wordpress2.py - wordpress api 6 | # 7 | # Created by skywind on 2019/05/04 8 | # Last Modified: 2019/05/04 23:24:18 9 | # 10 | #====================================================================== 11 | from __future__ import print_function, unicode_literals 12 | import sys 13 | import wordpress_xmlrpc 14 | 15 | 16 | #---------------------------------------------------------------------- 17 | # WordPress 18 | #---------------------------------------------------------------------- 19 | class WordPress (object): 20 | 21 | def __init__ (self, url, username, password): 22 | self._url = self.__url_normalize(url) 23 | self._rpc = self.__parse_url(self._url) 24 | self._username = username 25 | self._password = password 26 | self._client = wordpress_xmlrpc.Client(self._rpc, 27 | self._username, self._password) 28 | 29 | def __url_normalize (self, url): 30 | if not url.startswith('https://'): 31 | if not url.startswith('http://'): 32 | url = 'http://' + url 33 | return url 34 | 35 | def __parse_url (self, url): 36 | if url.endswith('/xmlrpc.php'): 37 | return url 38 | if not url.endswith('/'): 39 | url = url + '/' 40 | return url + 'xmlrpc.php' 41 | 42 | # post['id']: integer uid of the post 43 | # post['content']: string content 44 | # post['title']: string title 45 | # post['status']: string of draft, private, publish 46 | # post['category']: list 47 | # post['tag']: list 48 | # post['comment']: open/closed 49 | # post['date']: datetime object in UTC 50 | def __convert_post (self, post): 51 | newpost = wordpress_xmlrpc.WordPressPost() 52 | if post: 53 | if 'id' in post: 54 | newpost.id = post['id'] 55 | if 'title' in post: 56 | newpost.title = post['title'] 57 | newpost.content = post.get('content', '') 58 | newpost.post_status = 'draft' 59 | if post.get('status'): 60 | newpost.post_status = post['status'] 61 | cats = post.get('category') 62 | tags = post.get('tag') 63 | if cats or tags: 64 | newpost.terms_names = {} 65 | if cats: 66 | newpost.terms_names['category'] = cats 67 | if tags: 68 | newpost.terms_names['post_tag'] = tags 69 | if post.get('comment'): 70 | newpost.comment_status = post['comment'] 71 | if post.get('date'): 72 | newpost.date = post['date'] 73 | if post.get('date_modifed'): 74 | newpost.date_modified = post['date_modified'] 75 | if post.get('slug'): 76 | newpost.slug = post['slug'] 77 | else: 78 | newpost.post_status = 'draft' 79 | newpost.comment_status = 'open' 80 | newpost.content = '' 81 | return newpost 82 | 83 | def post_new (self, post = None): 84 | newpost = self.__convert_post(post) 85 | action = wordpress_xmlrpc.methods.posts.NewPost(newpost) 86 | pid = self._client.call(action) 87 | return pid 88 | 89 | def post_edit (self, post): 90 | if 'id' not in post: 91 | raise ValueError('missing id in post') 92 | newpost = self.__convert_post(post) 93 | pid = post['id'] 94 | action = wordpress_xmlrpc.methods.posts.EditPost(pid, newpost) 95 | return self._client.call(action) 96 | 97 | def post_get (self, pid): 98 | action = wordpress_xmlrpc.methods.posts.GetPost(pid) 99 | return self._client.call(action) 100 | 101 | # keys of query: number, offset, orderby, order(ASC/DESC), post_type 102 | # and post_status 103 | # returns list of WordPressPost instances 104 | def post_list (self, query): 105 | action = wordpress_xmlrpc.methods.posts.GetPosts(query) 106 | return self._client.call(action) 107 | 108 | # returns { 'id':xx, 'file':xx, 'url':xx, 'type':xx } 109 | def media_upload (self, source, name, mime = None): 110 | if isinstance(source, str): 111 | content = open(source, 'rb').read() 112 | if hasattr(source, 'read'): 113 | content = source.read() 114 | else: 115 | content = source 116 | if mime is None: 117 | if content[:3] == b'\xff\xd8\xff': 118 | mime = 'image/jpeg' 119 | elif content[:5] == b'\x89PNG\x0d': 120 | mime = 'image/png' 121 | elif content[:2] == b'\x42\x4d': 122 | mime = 'image/x-bmp' 123 | elif content[:5] == b'GIF89': 124 | mime = 'image/gif' 125 | elif content[:2] == b'\x50\x4b': 126 | mime = 'application/zip' 127 | elif content[:3] == b'\x37\x7a\xbc': 128 | mime = 'application/x-7z-compressed' 129 | data = {} 130 | data['name'] = name 131 | data['bits'] = wordpress_xmlrpc.compat.xmlrpc_client.Binary(content) 132 | if mime: 133 | data['type'] = mime 134 | action = wordpress_xmlrpc.methods.media.UploadFile(data) 135 | response = self._client.call(action) 136 | return response 137 | 138 | def media_get (self, attachment_id): 139 | action = wordpress_xmlrpc.methods.media.GetMediaItem(attachment_id) 140 | return self._client.call(action) 141 | 142 | # keys of query: number, offset, parent_id, mime_type 143 | # returns list of WordPressMedia instances 144 | def media_list (self, query): 145 | action = wordpress_xmlrpc.methods.media.GetMediaLibrary(query) 146 | return self._client.call(action) 147 | 148 | 149 | 150 | 151 | 152 | #---------------------------------------------------------------------- 153 | # testing suit 154 | #---------------------------------------------------------------------- 155 | if __name__ == '__main__': 156 | def test1(): 157 | wp = WordPress('localhost/web/blog', 'skywind', '678900') 158 | print(wp._rpc) 159 | return 0 160 | def test2(): 161 | wp = WordPress('localhost/web/blog', 'skywind', '678900') 162 | pid = wp.post_new() 163 | post = {} 164 | import ascmini 165 | post['id'] = pid 166 | post['content'] = 'Now is: ' + ascmini.timestamp() 167 | post['title'] = 'robot' 168 | post['status'] = 'publish' 169 | post['category'] = ['life'] 170 | post['tag'] = ['ai', 'game'] 171 | print(wp.post_edit(post)) 172 | return 0 173 | def test3(): 174 | wp = WordPress('localhost/web/blog', 'skywind', '678900') 175 | post = wp.post_get(40) 176 | print(post) 177 | print(post.link) 178 | print(post.content) 179 | print(post.id) 180 | return 0 181 | def test4(): 182 | wp = WordPress('localhost/web/blog', 'skywind', '678900') 183 | query = {} 184 | query['offset'] = 0 185 | for post in wp.post_list(query): 186 | print(post) 187 | print(post.link) 188 | print(post.date) 189 | print(type(post.date), post.date.timestamp()) 190 | return 0 191 | test4() 192 | 193 | 194 | 195 | 196 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | python-wordpress-xmlrpc 2 | markdown 3 | beautifulsoup4 4 | PySocks 5 | 6 | 7 | 8 | --------------------------------------------------------------------------------