├── .gitignore ├── Auto ├── LICENSE ├── README.md ├── common ├── __init__.py ├── httpd.py ├── httpd_test.py ├── spy.py ├── spy_test.py └── util.py ├── deps.sh ├── doc.sh ├── doc ├── html.jsont ├── latch-help.txt ├── plugins.md ├── screencast.md ├── screenshot.jpg ├── shell.md ├── simple-html.jsont ├── webpipe.md ├── wp-help-advanced.txt └── wp-help.txt ├── install.sh ├── latch.sh ├── latch ├── NOTES ├── latch.js ├── latch.py ├── latch_test.py └── templates.py ├── live-plugins └── demo.sh ├── plugins ├── DEFAULT │ └── render ├── R │ ├── render │ └── testdata │ │ └── tiny.R ├── README ├── Rplot.png │ └── render ├── _bin │ └── render-tar.sh ├── ansi │ ├── README.md │ ├── render │ └── testdata │ │ └── typescript ├── csv │ ├── render │ ├── render-dev.sh │ ├── render.py │ ├── static │ │ └── bar.txt │ └── testdata │ │ └── tiny.csv ├── dot │ ├── render │ └── testdata │ │ └── cluster.dot ├── html │ ├── render │ └── testdata │ │ ├── example.html │ │ └── tiny.html ├── markdown │ ├── gallery.md │ ├── render │ └── testdata │ │ └── tiny.markdown ├── tar.bz2 │ └── render ├── tar.gz │ └── render ├── tar.xz │ └── render ├── txt │ ├── render │ └── testdata │ │ ├── example.txt │ │ └── tiny.txt ├── webpipe-lib-test.sh ├── webpipe-lib.sh └── zip │ ├── render │ └── testdata │ └── tiny.zip ├── run.sh ├── static └── webpipe.css ├── testdata ├── file with spaces.txt └── file.unknown ├── third_party ├── README ├── json │ ├── README │ ├── gallery.md │ ├── render │ ├── static │ │ ├── jsontree.css │ │ └── jsontree.js │ └── testdata │ │ └── tiny.json └── treemap │ ├── COPYING │ ├── README │ ├── render │ ├── static │ ├── webtreemap.css │ └── webtreemap.js │ ├── testdata │ └── tiny.treemap │ └── webtreemap.html ├── webpipe-test.sh ├── webpipe.R ├── webpipe ├── __init__.py ├── handlers.py ├── handlers_test.py ├── index.html ├── publish.py ├── recv.py ├── serve.py ├── serve_test.py ├── xrender.py └── xrender_test.py ├── wp-dev.sh ├── wp-stub.sh ├── wp-test.sh └── wp.sh /.gitignore: -------------------------------------------------------------------------------- 1 | _tmp 2 | *.pyc 3 | *.swp 4 | -------------------------------------------------------------------------------- /Auto: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o nounset 4 | 5 | # Setup: 6 | # - Single source repo for webpipe/latch, for convenience, code sharing, etc. 7 | # - Two different basis packages? Or one. 8 | # - A single package can expose two executables: webpipe and latch. 9 | # - And the webpipe.R library. 10 | # - A single set of documentation? If it is versioned, that doesn't really 11 | # make that much sense. But for now we won't bother versioning it. 12 | 13 | readonly THIS_DIR=$(readlink -f $(dirname $0)) 14 | 15 | export PYTHONPATH=$THIS_DIR:~/hg/tnet/python:~/hg/json-template/python 16 | 17 | # Now do the data directory, but make sure to only get the stuff we want. Not 18 | # examples, testdata, .swp files, etc. 19 | plugins-manifest() { 20 | # sh: including _bin/render-tar.sh now 21 | find plugins \ 22 | -name render -o \ 23 | -name \*.sh -o \ 24 | -name \*.js -o \ 25 | -name \*.css -o \ 26 | -name \*.html 27 | } 28 | 29 | manifest() { 30 | # TODO: create a versioning scheme. We are just using this for the build 31 | # stamp (timestamp, host) 32 | local version=prerelease 33 | basisc echo-stamp webpipe $version > _tmp/Package.stamp 34 | echo _tmp/Package.stamp Package.stamp 35 | 36 | # TODO: Use deps in _tmp 37 | # TODO: build basis package 38 | # TODO: add publish.py 39 | py-deps webpipe.serve webpipe.xrender 40 | ls webpipe/*.html webpipe.R wp.sh 41 | 42 | # docs are used for help 43 | ls doc/wp-help*.txt 44 | 45 | plugins-manifest 46 | } 47 | 48 | Build() { 49 | manifest | multi tar _tmp/webpipe.tar.gz 50 | } 51 | 52 | 53 | run-py-tests() { 54 | local dir=$1 55 | ls $dir/*_test.py | awk '{ print "./" $0 }' | sh -x -e 56 | } 57 | 58 | Test() { 59 | set -o errexit 60 | 61 | run-py-tests webpipe 62 | run-py-tests common 63 | run-py-tests latch 64 | 65 | echo PASS 66 | } 67 | 68 | # TODO: 69 | # - usage-address.txt should be set 70 | 71 | Deploy() { 72 | # copy basis package somewhere? 73 | echo 74 | } 75 | 76 | "$@" 77 | 78 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2014, Google Inc. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Google Inc. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | webpipe 2 | ======= 3 | 4 | webpipe is server and a set of tools which bridge the Unix shell and the web. 5 | You create files in a terminal (using [R][], a shell script, etc.), and they 6 | will be rendered immediately in your browser. 7 | 8 | It gets rid of the `Alt-Tab F5` dance when creating content. 9 | 10 | [R]: http://r-project.org/ 11 | 12 | End Users 13 | --------- 14 | 15 | See [doc/webpipe.html]() for instructions on how to use it. 16 | 17 | 18 | Developing 19 | ---------- 20 | 21 | Use the `wp-dev.sh` wrapper for `wp.sh`: 22 | 23 | $ ./wp-dev.sh run 24 | 25 | This script relies on a couple dependencies existing in hard-coded paths. 26 | Fetch them as follows: 27 | 28 | Make a `~/hg` dir. 29 | 30 | $ hg clone https://code.google.com/p/json-template/ 31 | 32 | $ hg clone https://code.google.com/p/tnet/ 33 | 34 | [JSON Template](https://code.google.com/p/json-template/) is the template 35 | language used, and [TNET](https://code.google.com/p/tnet/) is the serialization 36 | format. 37 | 38 | Portability 39 | ----------- 40 | 41 | There are multiple components to `webpipe`, each with different portability 42 | goals. 43 | 44 | It's somewhat confusing because the webpipe client often runs on server 45 | machines, and the webpipe server may run on your client machine (i.e. 46 | "localhost"). 47 | 48 | "Server" machines are some kind of Linux, or perhaps BSD. Client machines 49 | include those, but also add Mac. 50 | 51 | From roughly least to most portable: 52 | 53 | - R client: this generally runs on a Linux box. It currently depends on "nc", 54 | which should be on most Linux boxes. netcat is very un-portable, but we are 55 | just using the simple invocation `nc localhost $port`, which is hopefully 56 | portable. 57 | 58 | - server and renderer: should run on Linux/Mac 59 | 60 | - shell client: this is the MOST portable one. Probably won't run on Mac. But 61 | right now it runs on machines without Python or bash! It just uses plain 62 | shell. And "nc" client only. 63 | 64 | TODO: Get rid of nc servers. Not portable. socat servers also won't run well 65 | on Mac, and are not installed by default on Linux. So do it in Python. 66 | 67 | Notes 68 | ----- 69 | 70 | Disclaimer: This is not an official Google project. 71 | 72 | -------------------------------------------------------------------------------- /common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andychu/webpipe/78fe629829af0aa3df1854501d29df68c8c010df/common/__init__.py -------------------------------------------------------------------------------- /common/httpd.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | """ 8 | httpd.py 9 | """ 10 | 11 | import BaseHTTPServer 12 | import os 13 | import posixpath 14 | import SimpleHTTPServer 15 | import SocketServer 16 | import urllib 17 | 18 | 19 | class ThreadedHTTPServer(SocketServer.ThreadingMixIn, 20 | BaseHTTPServer.HTTPServer): 21 | """ 22 | The main reason we inherit from HTTPServer instead of SocketServer is that it 23 | sets allow_reuse_address, which prevents the issue where we can't bind the 24 | same port for a period of time after restarting. 25 | 26 | NOTE: This doesn't use a thread pool or anything. It will just start a new 27 | thread for each request. 28 | 29 | For webpipe, since every thread will block waiting for the next part of the 30 | scroll, you can create a huge number of threads just by having a huge number 31 | of clients. But since this is mainly a single-user server, it doesn't 32 | matter. 33 | 34 | TODO: There's a still a Ctrl-C bug here, because I think the request threads 35 | get blocked on the threading.Event(). Need to setDaemon() all threads, 36 | including the ones that the web server makes. 37 | """ 38 | # override class variable in ThreadingMixIn. This makes it so that Ctrl-C works. 39 | daemon_threads = True 40 | 41 | 42 | class BaseRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): 43 | """ 44 | NOTE: The structure of Python's SimpleHTTPServer / BaseHTTPServer is quite 45 | bad. But we are reusing it for now, since it is built in to the standard 46 | library, and it gives "Apache-like" static serving semantics. 47 | 48 | If we end up having to hack this up too much, it might be worth it to write 49 | our own (or at least copy and modify that code, rather than this fragile 50 | inheritance. 51 | """ 52 | server_version = None 53 | root_dir = None 54 | 55 | def url_to_fs_path(self, url): 56 | """Translate a URL to a local file system path. 57 | 58 | By default, we just treat URLs as paths relative to self.root_dir. 59 | 60 | If it returns None, then a 404 is generated, without looking at disk. 61 | 62 | Called from send_head() (see SimpleHTTPServer). 63 | 64 | NOTE: This is adapted from Python stdlib SimpleHTTPServer.py. I just 65 | changed os.getcwd() to self.root_dir. 66 | """ 67 | words = [p for p in url.split('/') if p] 68 | 69 | path = self.root_dir # note: class variable 70 | 71 | # TODO: This can be cleaned up. Should just be os.path.join. 72 | for word in words: 73 | drive, word = os.path.splitdrive(word) 74 | head, word = os.path.split(word) 75 | if word in (os.curdir, os.pardir): # . .. 76 | continue 77 | path = os.path.join(path, word) 78 | return path 79 | 80 | # Copied from stdlib SimpleHTTPServer.py. The code isn't really extensible 81 | # so we have to copy it. 82 | def send_head(self): 83 | """Common code for GET and HEAD commands. 84 | 85 | This sends the response code and MIME headers. 86 | 87 | Return value is either a file object (which has to be copied 88 | to the outputfile by the caller unless the command was HEAD, 89 | and must be closed by the caller under all circumstances), or 90 | None, in which case the caller has nothing further to do. 91 | 92 | """ 93 | path = self.path 94 | # Query params aren't relevant to looking up a path. 95 | # 96 | # NOTE: Fragment should never be sent by the browser. Python stdlib 97 | # originally had this. 98 | path = path.split('?',1)[0] 99 | path = path.split('#',1)[0] 100 | # eliminates double slashes, etc. 101 | path = posixpath.normpath(urllib.unquote(path)) 102 | 103 | path = self.url_to_fs_path(path) 104 | if path is None: 105 | self.send_error(404, "File not found") 106 | return None 107 | 108 | f = None 109 | if os.path.isdir(path): 110 | if not self.path.endswith('/'): 111 | # redirect browser - doing basically what apache does 112 | self.send_response(301) 113 | self.send_header("Location", self.path + "/") 114 | self.end_headers() 115 | return None 116 | for index in "index.html", "index.htm": 117 | index = os.path.join(path, index) 118 | if os.path.exists(index): 119 | path = index 120 | break 121 | else: 122 | return self.list_directory(path) 123 | ctype = self.guess_type(path) 124 | try: 125 | # Always read in binary mode. Opening files in text mode may cause 126 | # newline translations, making the actual size of the content 127 | # transmitted *less* than the content-length! 128 | f = open(path, 'rb') 129 | except IOError: 130 | self.send_error(404, "File not found") 131 | return None 132 | self.send_response(200) 133 | self.send_header("Content-type", ctype) 134 | fs = os.fstat(f.fileno()) 135 | self.send_header("Content-Length", str(fs[6])) 136 | self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) 137 | self.end_headers() 138 | return f 139 | 140 | -------------------------------------------------------------------------------- /common/httpd_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python -S 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | """ 8 | httpd_test.py: Tests for httpd.py 9 | """ 10 | 11 | import unittest 12 | 13 | import httpd # module under test 14 | 15 | 16 | class HandlerTest(unittest.TestCase): 17 | def setUp(self): 18 | pass 19 | 20 | def tearDown(self): 21 | pass 22 | 23 | def testHandler(self): 24 | # Can't really instantiate this. req should be a socket? 25 | return 26 | req = None 27 | client_address = None 28 | server = None 29 | handler = httpd.BaseRequestHandler(req, client_address, server) 30 | print handler.translate_path('/') 31 | 32 | 33 | if __name__ == '__main__': 34 | unittest.main() 35 | -------------------------------------------------------------------------------- /common/spy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | """ 8 | spy.py 9 | 10 | Client for usage/error reporting. 11 | """ 12 | 13 | import json 14 | import os 15 | import socket 16 | import sys 17 | import time 18 | 19 | 20 | class Error(Exception): 21 | pass 22 | 23 | 24 | class UsageReporter(object): 25 | 26 | def __init__(self, host_port): 27 | self.host_port = host_port 28 | # Internet / UDP socket 29 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 30 | 31 | # These values are constant throughout the life of the process, and can be 32 | # used to match messages. We assume we have just one usage reporter per 33 | # process, and that it is instantiated early in the process. That way 34 | # start-ts is reasonably accurate. 35 | 36 | self.id_data = { 37 | # TODO: would getfqdn() be appropriate? How does thiat work. 38 | 'hostname': socket.gethostname(), 39 | 'pid': os.getpid(), 40 | } 41 | 42 | def Send(self, msg): 43 | """Send a raw UDP packet.""" 44 | #log('msg %r', msg) 45 | self.sock.sendto(msg, self.host_port) 46 | 47 | def SendDict(self, d): 48 | """Send a JSON-encoded dictionary of data.""" 49 | # TODO: 50 | # - accept function objects as values. 51 | # - And wrap them in a try/catch, so that bad error reporting can't 52 | # affect the program. 53 | # - also need a "extended" process ID? 54 | # - hostname / PID / process start time? that might be good enough. 55 | # - can you encode this efficiently? 56 | # - homer/3434/13232 57 | # - does time.time have too much resolution? 58 | 59 | msg = json.dumps(d) 60 | self.Send(msg) 61 | 62 | def SendRecord(self, event, d): 63 | """Send a JSON-encoded record with some standard data. 64 | 65 | Hostname and PID are send to uniquely identify the process. local time is 66 | calculated and sent. 67 | """ 68 | rec = dict(self.id_data) 69 | if d: 70 | rec.update(d) 71 | rec['ev'] = event # type of event 72 | rec['ts'] = time.time() 73 | self.SendDict(rec) 74 | 75 | # TODO: 76 | # SendStart() ? Most programs use this at the beginning. 77 | # SendSummary() ? 78 | # maybe there should be a protocol where you can send an accumulator object. 79 | 80 | 81 | class NullUsageReporter(object): 82 | def Send(self, msg): 83 | pass 84 | def SendDict(self, d): 85 | pass 86 | def SendRecord(self, event, d): 87 | pass 88 | 89 | 90 | def _GetUsageConfig(): 91 | """Allow the user to turn off usage reporting with an environment variable.""" 92 | 93 | # TODO: maybe use a level number? default level 9? 94 | # You probably want to distinguish between: 95 | # - report bugs 96 | # - report usage 97 | # maybe name them. 98 | # 99 | # SPY_REPORT_LEVEL= # nothing 100 | # SPY_REPORT_LEVEL=bugs (only bugs) 101 | # SPY_REPORT_LEVEL=usage (usage and bugs. usage normally includes start, and 102 | # a summary at the end. Should involve perf) 103 | # SPY_REPORT_LEVEL=perf? Is this a separate one? I think the app should 104 | # collect this and send it as usage. 105 | 106 | # SPY_REPORT_LEVEL=usage-details (argv, env, etc. Sutff that could be 107 | # private) 108 | 109 | # default is 'usage'. 110 | 111 | var = os.getenv('SPY_REPORT_USAGE') 112 | do_report = True 113 | if var is not None: 114 | var = var.strip() 115 | # set it to 0 or empty to turn off 116 | if var in ('', '0'): 117 | do_report = False 118 | log('Not reporting usage because SPY_REPORT_USAGE was set') 119 | return do_report 120 | 121 | 122 | def _ReadAddressFile(address_file=None): 123 | # TODO: readlink? or does basis take care of it? 124 | 125 | if not address_file: 126 | d = os.path.dirname(sys.argv[0]) 127 | address_file = os.path.join(d, 'usage-address.txt') 128 | 129 | try: 130 | with open(address_file) as f: 131 | contents = f.read() 132 | host, port = contents.split(':') 133 | port = int(port) 134 | result = (host, port) 135 | except IOError, e: 136 | #log('address Not found') 137 | result = None 138 | 139 | return result 140 | 141 | 142 | def GetClientFromConfig(address_file=None): 143 | """ 144 | Get a client, which may be the null client. 145 | 146 | Right now we lookat 147 | """ 148 | # server can be configured with a usage file 149 | do_report = _GetUsageConfig() 150 | 151 | address = _ReadAddressFile(address_file=address_file) 152 | if do_report and address: 153 | reporter = UsageReporter(address) 154 | else: 155 | reporter = NullUsageReporter() 156 | return reporter 157 | 158 | -------------------------------------------------------------------------------- /common/spy_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python -S 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | """ 8 | spy_test.py: Tests for spy.py 9 | """ 10 | 11 | import unittest 12 | 13 | import spy # module under test 14 | 15 | 16 | class FooTest(unittest.TestCase): 17 | def setUp(self): 18 | pass 19 | 20 | def tearDown(self): 21 | pass 22 | 23 | def testFoo(self): 24 | print 'Hello from spy_test.py' 25 | 26 | 27 | def testUsage(self): 28 | u = spy.UsageReporter(('localhost', 8988)) 29 | u.Send('unit test') 30 | u.SendDict({'method': 'SendDict'}) 31 | u.SendRecord('start', {'method': 'SendRecord'}) 32 | 33 | # TODO: 34 | # - send function values which raise errors 35 | # - make sure KeyboardInterrupt isn't swallowed 36 | 37 | if __name__ == '__main__': 38 | unittest.main() 39 | -------------------------------------------------------------------------------- /common/util.py: -------------------------------------------------------------------------------- 1 | """ 2 | util.py 3 | """ 4 | 5 | import os 6 | import sys 7 | 8 | ANSI_BLUE = '\033[36m' 9 | ANSI_GREEN = '\033[32m' 10 | 11 | ANSI_RESET = '\033[0;0m' 12 | 13 | class Logger(object): 14 | def __init__(self, color): 15 | self.color = color 16 | 17 | basename = os.path.basename(sys.argv[0]) 18 | name, _ = os.path.splitext(basename) 19 | 20 | self.prefix = name + ':' 21 | if sys.stderr.isatty(): 22 | self.prefix = self.color + self.prefix + ANSI_RESET 23 | 24 | def __call__(self, msg, *args): 25 | if args: 26 | msg = msg % args 27 | print >>sys.stderr, self.prefix, msg 28 | 29 | 30 | def GetPackageDir(): 31 | this_dir = os.path.dirname(sys.argv[0]) # webpipe subdir 32 | return os.path.dirname(this_dir) # root of package 33 | 34 | 35 | def GetUserDir(): 36 | return os.path.expanduser('~/webpipe') 37 | 38 | -------------------------------------------------------------------------------- /deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | # 8 | # Scripts to handle plugin dependencies. This can be done in a more principled 9 | # way later. 10 | # 11 | # Usage: 12 | # ./deps.sh 13 | 14 | set -o nounset 15 | 16 | get-json-tree() { 17 | local out=_tmp/json-tree 18 | wget --directory $out https://github.com/lmenezes/json-tree/archive/master.zip 19 | cd $out 20 | unzip master.zip 21 | } 22 | 23 | install-user() { 24 | local plugin=plugins/json/ 25 | mkdir -p $plugin/static 26 | 27 | local src=_tmp/json-tree/json-tree-master/ 28 | cp -v $src/jsontree.js $src/css/jsontree.css $plugin/static 29 | cp -v $src/example.html $plugin 30 | tree $plugin 31 | } 32 | 33 | "$@" 34 | -------------------------------------------------------------------------------- /doc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | # 8 | # Usage: 9 | # ./doc.sh 10 | 11 | # Add the HTML shell. A data dictionary should be readable stdin. Does NOT 12 | # depend on $PWD, so callers can cd. 13 | 14 | set -o nounset 15 | set -o errexit 16 | 17 | to-html() { 18 | local out=$1 19 | jsont doc/html.jsont > $out 20 | } 21 | 22 | # TODO: Get rid of bad dependencies: 23 | # 24 | # webpipe docs -> pin -> docopt 25 | make-dict() { 26 | local body_filename=$1 27 | pin --title-tag=h1 $body_filename 28 | } 29 | 30 | # Build main docs. Called by ./run.sh latch-demo, which calls './latch.sh 31 | # rebuild'. 32 | build() { 33 | local in=$1 34 | local out=$2 35 | 36 | local base_in=$(basename $in .md) # For now, get rid of subdirs 37 | local body=_tmp/$base_in-body.html 38 | 39 | mkdir -p --verbose $(dirname $out) 40 | echo "Building $in -> $body -> $out" 41 | 42 | markdown $in >$body 43 | 44 | ls -al $body 45 | 46 | local template 47 | case $base_in in 48 | # Use simpler template from the screencast (so it's not skinny) 49 | screencast) 50 | template=doc/simple-html.jsont 51 | ;; 52 | *) 53 | template=doc/html.jsont 54 | ;; 55 | esac 56 | 57 | make-dict $body | jsont $template > $out 58 | 59 | ls -al $out 60 | } 61 | 62 | build-all() { 63 | build doc/webpipe.md _tmp/doc/webpipe.html 64 | build doc/screencast.md _tmp/doc/screencast.html 65 | shrink-screenshot 66 | gallery 67 | 68 | # For the video 69 | ln -v -s -f \ 70 | ../../doc/screenshot.jpg \ 71 | ../../doc/screencast.ogv \ 72 | _tmp/doc 73 | 74 | tree _tmp/doc 75 | } 76 | 77 | shrink-screenshot() { 78 | convert doc/screenshot.jpg -scale '50%' _tmp/doc/screenshot_small.jpg 79 | } 80 | 81 | # TODO: Just list the plugins/ dir? 82 | # - need to show symlinks 83 | 84 | plugin-types() { 85 | echo dot html json markdown Rplot.png treemap tar.gz txt zip 86 | # TODO: 87 | # - csv has issue with importing JSON Template; not being hermetic 88 | # - R generates , it's not a real snippet 89 | # - typescript is also generates html 90 | } 91 | 92 | # Generate a named snippet for each plugin type. Then join them into an HTML 93 | # doc. 94 | 95 | gallery-snippets() { 96 | local plugin_types="$1" 97 | local out_dir=$2 98 | 99 | rm -rf $out_dir 100 | mkdir -p $out_dir 101 | 102 | for p in $plugin_types; do 103 | local plugin=$PWD/plugins/$p/render 104 | local input 105 | case $p in 106 | dot) 107 | input=$PWD/plugins/dot/testdata/cluster.dot 108 | ;; 109 | typescript) 110 | # disabled 111 | input=$PWD/plugins/typescript/testdata/typescript 112 | ;; 113 | *) 114 | input=$PWD/plugins/$p/testdata/tiny.$p 115 | ;; 116 | esac 117 | 118 | # NOTE: we are not providing a number 119 | local output=$p 120 | 121 | # plugins are written to be in the output dir. 122 | pushd $out_dir 123 | $plugin $input $output 124 | popd >/dev/null 125 | done 126 | } 127 | 128 | print-gallery() { 129 | local plugin_types="$1" 130 | local base_dir=$2 131 | 132 | cat <webpipe Gallery 134 | 135 |

Here is a list of file types and example documents. There is a JavaScript 136 | visualization behind some links.

137 | EOF 138 | 139 | # Generate TOC. 140 | for p in $plugin_types; do 141 | echo "$p
" 142 | done 143 | 144 | # Generate snippets. TODO: should be use a
or something? 145 | 146 | for p in $plugin_types; do 147 | echo '
' 148 | echo "

$p

" 149 | 150 | # TODO: alternative text? 151 | local path=plugins/$p/gallery.md 152 | if test -f $path; then 153 | # display on stdout 154 | markdown $path 155 | fi 156 | 157 | # snippet inline 158 | cat $base_dir/$p.html 159 | done 160 | } 161 | 162 | # TODO: 163 | # - how to get the .js to work? 164 | # It goes ../../... 165 | # Maybe it should be a variable, like $WEBPIPE_PLUGIN_BASE_URL. 166 | # 167 | # could publish it to: 168 | # there will be: 169 | # webpipe/plugins/ 170 | # webpipe/s/2014-04-04/1.html 171 | # webpipe/doc/gallery/index.html 172 | # png testdata 173 | # - just make a little R plot 174 | 175 | makelink() { 176 | local src=$1 177 | local dest=$2 178 | mkdir -p $(dirname $dest) 179 | ln -v -s -f $src --no-target-directory $dest 180 | } 181 | 182 | # The gallery needs to reference static assets. list the ones with static. 183 | # TODO: automate this more? 184 | 185 | link-static() { 186 | makelink $PWD/plugins/treemap/static/ _tmp/plugins/treemap/static 187 | makelink $PWD/plugins/json/static/ _tmp/plugins/json/static 188 | tree _tmp/doc 189 | } 190 | 191 | gallery() { 192 | local plugin_types="$(plugin-types)" 193 | local base_dir=$PWD/_tmp/doc/gallery # absolute path 194 | 195 | gallery-snippets "$plugin_types" $base_dir 196 | 197 | local body=$base_dir/body.html 198 | local out=$base_dir/index.html 199 | print-gallery "$plugin_types" $base_dir > $body 200 | 201 | # NOTE: It's interesting that #anchors don't work without ? At least 202 | # in Chrome. 203 | make-dict $body | to-html $out 204 | 205 | # The gallery needs to link to static assets. TODO: Should be _tmp/doc? 206 | link-static 207 | 208 | ls -al $base_dir 209 | } 210 | 211 | check() { 212 | # TODO: put this in a test? 213 | tidy -errors _tmp/gallery/out/index.html 214 | } 215 | 216 | # NOTE: gallery has to be 3 levels deep to access ../../../plugins/*/static 217 | deploy() { 218 | set -o errexit 219 | # create a file in this dir with the base dir, e.g. user@host.com:mydir 220 | local base=$(cat ssh-base.txt) 221 | echo $base 222 | # Have to get all the generated dirs. NOTE: Don't need the individual 223 | # snippets. Maybe remove. 224 | scp -r _tmp/doc/* $base/webpipe/doc 225 | scp -r _tmp/plugins $base/webpipe 226 | } 227 | 228 | # 229 | # Screencast 230 | # 231 | 232 | # Use the method here. "Record my desktop", then convert using mplayer then Image Magick. 233 | # 234 | # http://askubuntu.com/questions/107726/how-to-create-animated-gif-images-of-a-screencast 235 | 236 | # 57s for a 2.7 M file. 237 | frames() { 238 | local video_in=$1 239 | local frames_out=${2:-} 240 | if test -z "$frames_out"; then 241 | local out_dir=_tmp/frames 242 | mkdir -p $out_dir 243 | frames_out="$out_dir/$(basename $video_in)" 244 | fi 245 | 246 | # TODO: use png or gif? 247 | time mplayer -ao null $video_in -vo jpeg:outdir=$frames_out 248 | } 249 | 250 | # Took 4m 37s for a 1.4 MB ogv? Resulted in 50 MB animated gif. 251 | animated-gif() { 252 | local frames_in=$1 253 | local gif_out=${2:-_tmp/gif/screencast.gif} 254 | mkdir -p $(dirname $gif_out) 255 | time convert $frames_in/* $gif_out 256 | } 257 | 258 | # This method reduces gif from 50MB to 1MB. 259 | # 260 | # But on 142MB gif, this doesn't just crash, but hangs the machine! 261 | 262 | optimize-gif() { 263 | local gif_in=$1 264 | local gif_out=$(echo $gif_in | sed 's/\.gif/_opt.gif/') 265 | set -x 266 | export MAGICK_THREAD_LIMIT=1 267 | time convert $gif_in -fuzz 10% -layers Optimize $gif_out 268 | } 269 | 270 | "$@" 271 | -------------------------------------------------------------------------------- /doc/html.jsont: -------------------------------------------------------------------------------- 1 | meta: [] 2 | default-formatter: raw 3 | 4 | 5 | 6 | [# A template for HTML docs with text] 7 | 8 | 9 | 10 | 11 | 12 | 13 | [title] 14 | 32 | 33 | 34 | 35 | 36 | 37 | [# NOTE: id="content" is needed for the toc.js script] 38 | 39 | 40 |

Waiting for latch...

41 | 42 | [body] 43 | 44 | [.section source-modtime] 45 |
46 |

Last modified: [@|html]

47 | [.end] 48 | 49 | [.section google-analytics-id] 50 | 54 | 59 | [.end] 60 | 61 | 62 | 63 | -------------------------------------------------------------------------------- /doc/latch-help.txt: -------------------------------------------------------------------------------- 1 | Usage: latch [ rebuild | serve ] 2 | 3 | NOTE: This interface is subject to change. 4 | 5 | latch rebuild 6 | Whenever one of the files change, runs the command to rebuild them. 7 | 8 | latch serve 9 | Start latch server. Your static file should have the string 10 | '' in it. This will be replaced by JavaScript which 11 | does a hanging GET on a rebuild event. 12 | 13 | latch help 14 | Show this help. 15 | 16 | -------------------------------------------------------------------------------- /doc/plugins.md: -------------------------------------------------------------------------------- 1 | Webpipe Plugin Interface 2 | ======================== 3 | 4 | TODO 5 | 6 | -------------------------------------------------------------------------------- /doc/screencast.md: -------------------------------------------------------------------------------- 1 | Back to [webpipe](webpipe.html) 2 | 3 | webpipe Screencast 4 | ================== 5 | 6 | NOTE: This requires HTML5 and ogg video support in your browser. (There is no 7 | audio.) 8 | 9 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /doc/screenshot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andychu/webpipe/78fe629829af0aa3df1854501d29df68c8c010df/doc/screenshot.jpg -------------------------------------------------------------------------------- /doc/shell.md: -------------------------------------------------------------------------------- 1 | Using webpipe with the shell 2 | ============================ 3 | 4 | Or "Bringing back graphical computing for headless servers" 5 | 6 | The original idea behind webpipe was to use it with an R client. But using it 7 | at the shell opens up a lot of possibilities, e.g. for system administration 8 | and performance analysis. 9 | 10 | # run the server that listens on port 8988 11 | 12 | $ wp run 13 | 14 | # hm I don't want to run another server... 15 | 16 | # Run EITHER 17 | 18 | $ wp run-recv 19 | # listen on port 8988 for local filenames 20 | # also listen on 8987 for remote 21 | 22 | # Run a socat server on 8987, which pipes to recv, then writes the file, then 23 | # sends filename to 9899. 24 | 25 | 26 | # Now you can send stuff locally to 8987. "wp-stub show" ? Or send? 27 | 28 | # copy 29 | $ wp scp-stub example.com 30 | 31 | $ wp ssh example.com # Local 8987 tunnel 32 | 33 | # TODO: Also need to test two tunnels. Can we use the same port? What about 34 | # with big files? 35 | 36 | example.com$ wp-stub show foo.png 37 | 38 | Now foo.png appears in the browser on localhost:8989. 39 | 40 | Even further generalization: 41 | 42 | You want not just the xrender endpoint, but the raw server endpoint? Hm. 43 | Allow interposition at any point... 44 | 45 | The use case is where you run the render pipelines? Running render pipelines 46 | remotely, but server locally. If you want to be secure, and don't want to 47 | expose an HTTP server. 48 | 49 | 50 | Pipeline implementation options: 51 | 52 | - threads and Queue() 53 | - coroutines 54 | - it's synchronous dataflow, so you don't need queues really. 55 | - BUT: if each stage has input from a real socket, and the previous stage, 56 | can you do that with Python coroutines? can you do fan in? 57 | I think so. Call stage.send() from two places. 58 | Multiple network ports need a select() loop. 59 | 60 | Add --listen for every stage? So you can listen on recv input, xrender input, 61 | or server input port (and listen on HTTP serving port too) 62 | 63 | What's the command line syntax? You can run: 64 | 65 | - serve 66 | - xrender and serve 67 | - recv xrender and serve 68 | 69 | And you can run other parts on the remote machine. 70 | 71 | - send 72 | - xrender then send 73 | 74 | So pipelines look like this, where == is a TCP socket separating machines. | 75 | is a pipe, or possibly in-process Queue/channel. 76 | 77 | xrender | serve (local machine only) 78 | 79 | send == recv | xrender | serve 80 | 81 | xrender | send == recv | serve 82 | 83 | NOTE: send does not currently handle directories, or the combo of file + 84 | directory that webpipe uses. I guess we write the file last for this reason. 85 | 86 | 87 | What does the client look like? Difference between show and send? I think you 88 | want to use show everywhere. Both local and remote. Ports are the same. 89 | "show" goes to port 8988? "send" goes to port 8987. 90 | 91 | Ports: 92 | - 8987 for recv input 93 | - 8988 for xrender input 94 | - TODO: change is to it's 8980 8981 8982? Three consecutive ports 95 | WEBPIPE_RECV_PORT 96 | WEBPIPE_XRENDER_PORT 97 | WEBPIPE_HTML_PORT 98 | Need this because those ports might be used on the machine. 99 | 100 | - 8989 is webpipe HTTP server 101 | - 8900 is for latch HTTP 102 | 103 | 104 | More advanced idea 105 | ------------------ 106 | 107 | Along the lines of "bringing back graphical computing". 108 | 109 | - What if you want a "live top" or something. Or a "live pstree". A visualization. 110 | 111 | Then you aren't just sending files back. You want a client program that prints 112 | a textual protocol to stdout for updates. Like it will print CPU usage lines 113 | or something. 114 | 115 | And then you want to pipe that directly to JS and visualize it? 116 | 117 | The easiest demo is some kind of time series. You scp a little shell/Python 118 | script to a server. And then it will output a time series. 119 | 120 | 121 | 122 | -------------------------------------------------------------------------------- /doc/simple-html.jsont: -------------------------------------------------------------------------------- 1 | meta: [] 2 | default-formatter: raw 3 | 4 | 5 | 6 | [# A simple template for HTML docs] 7 | 8 | 9 | 10 | 11 | 12 | 13 | [title] 14 | 20 | 21 | 22 | 23 | 24 | 25 | [# NOTE: id="content" is needed for the toc.js script] 26 | 27 | 28 |

Waiting for latch...

29 | 30 | [body] 31 | 32 | [.section source-modtime] 33 |
34 |

Last modified: [@|html]

35 | [.end] 36 | 37 | [.section google-analytics-id] 38 | 42 | 47 | [.end] 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /doc/webpipe.md: -------------------------------------------------------------------------------- 1 | webpipe 2 | ======= 3 | 4 | webpipe is server and a set of tools which bridge the Unix shell and the web. 5 | You create files in a terminal (using [R][], bash, etc.), and they will be 6 | rendered immediately in your browser. 7 | 8 | It gets rid of the `Alt-Tab F5` dance when creating content. 9 | 10 | [R]: http://r-project.org/ 11 | 12 | 13 | webpipe screenshot 14 | 15 | 16 | Setup 17 | ----- 18 | 19 | Put `wp` in your `PATH`. For example: 20 | 21 | $ ln -s /path/to/webpipe/wp.sh ~/bin/wp 22 | 23 | Do one-time initialization of your `~/webpipe` dir: 24 | 25 | $ wp init 26 | 27 | Create a convenience symlink for the R library: 28 | 29 | $ ln -s /path/to/webpipe/webpipe.R ~/webpipe 30 | 31 | 32 | R Plots 33 | ------- 34 | 35 | The initial motivation for webpipe was to show R plots in a browser, avoiding 36 | the remote X11 protocol in favor of HTTP. 37 | 38 | First start the renderer and server: 39 | 40 | $ wp run 41 | 42 | This creates a new "session", e.g. `2014-04-03`. Files can be put in the 43 | `~/webpipe/input` directory, and then they are rendered to HTML in the 44 | `~/webpipe/s/2014-02-17` directory. 45 | 46 | Visit http://localhost:8989/ in your browser. 47 | 48 | Then in R, make a plot using the the wrapper functions in `webpipe.R`: 49 | 50 | $ R 51 | ... 52 | > source('~/webpipe/webpipe.R') 53 | > 54 | > web.plot(1:10) 55 | > web.hist(rnorm(10)) 56 | 57 | Instead of opening up an desktop window, the plot will be pushed to your 58 | browser via AJAX. 59 | 60 | ggplot works easily as well: 61 | 62 | > library(ggplot2) 63 | > p = ggplot(mtcars, aes(wt, mpg)) + geom_point() 64 | > web.plot(p) 65 | 66 | R client options 67 | ---------------- 68 | 69 | The `webpipe.png.args` option is used to determine additional arguments to the 70 | `png` device. Example: 71 | 72 | options(webpipe.png.args=list(width=800, height=600)) 73 | 74 | This can be put in your `~/.Rprofile`, if desired. 75 | 76 | Shell Usage 77 | ----------- 78 | 79 | You can also display files from the shell: 80 | 81 | $ wp show mydata.csv 82 | 83 | With no file, `show` reads from stdin. 84 | 85 | $ ls -l | wp show 86 | 87 | Use `wp help` to see more actions. 88 | 89 | 90 | File Types 91 | ---------- 92 | 93 | The renderer process is called `xrender`, which shells out to various plugins. 94 | It understands `.png` files, plain text, HTML, and CSV files. (TODO: document 95 | the full list) 96 | 97 | More Docs 98 | --------- 99 | 100 | * [Gallery](gallery/) of plugin types (under construction) 101 | 102 | Publishing 103 | ---------- 104 | 105 | You can publish entries to "shared hosting", so you don't have to keep your 106 | server up. 107 | 108 | TODO ... 109 | 110 | 111 | Advanced Usage 112 | -------------- 113 | 114 | TODO ... 115 | 116 | 117 | 124 | 125 | 126 | 127 | 177 | 178 | Known Issues 179 | ------------ 180 | 181 | - Plugins should have more consistent style (CSS, etc.) 182 | - There is too much debug spew on the terminal. 183 | 184 | Feedback 185 | -------- 186 | 187 | Contact `__EMAIL_ADDRESS__`. 188 | 189 | 190 | 191 | 225 | 226 | 227 | -------------------------------------------------------------------------------- /doc/wp-help-advanced.txt: -------------------------------------------------------------------------------- 1 | Advanced 'wp' actions: 2 | 3 | REMOTE ACCESS 4 | 5 | scp-stub 6 | scp the wp-stub.sh file to a host. The stub can be used to send files. 7 | Example: wp scp-stub user@example.com:/tmp 8 | 9 | ssh 10 | ssh to a host, opening a reverse tunnel. For copying files. 11 | Example: wp ssh user@example.com 12 | 13 | (NOTE: These features are experimental, and the documentation is incomplete.) 14 | 15 | 16 | COMPONENTS 17 | 18 | xrender 19 | Run the rendering process (which processes plugins) on its own. 20 | 21 | serve 22 | Run the server on its own. (TODO: this has actions too) 23 | -------------------------------------------------------------------------------- /doc/wp-help.txt: -------------------------------------------------------------------------------- 1 | Usage: wp [ init | run | show | sink | publish | help | version ] 2 | 3 | wp init 4 | Initialize the ~/webpipe directory. Run before using 5 | 6 | wp run [session] 7 | Run the terminal -> browser pipeline. There will be a rendering process and 8 | a server process. The default session is derived from today's date. 9 | 10 | wp show [file]... 11 | Show files in the browser. If no file is given, show stdin as text (type 12 | 'txt'). 13 | 14 | wp show-as [file]... 15 | Show files in the browser, with the given file type. If no file is given, 16 | show stdin. 17 | Alias: wp as 18 | 19 | wp publish 20 | Publish an entry in a scroll. is the name of a publishing plugin. 21 | 22 | wp help 23 | Show this help. More help at: 'wp help advanced' 24 | 25 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./install.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | _link() { 11 | ln -s -f -v "$@" 12 | } 13 | 14 | bin() { 15 | _link $PWD/latch.sh ~/bin/latch 16 | _link $PWD/wp.sh ~/bin/wp 17 | } 18 | 19 | 20 | "$@" 21 | -------------------------------------------------------------------------------- /latch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | # 8 | # latch.sh 9 | # 10 | # Usage: 11 | # ./latch.sh 12 | # 13 | # To reload edit docs quickly, do: 14 | # 15 | # ./latch.sh rebuild ... 16 | # 17 | # This rebuilds files in a loop. 18 | # 19 | # Then. 20 | # 21 | # ./latch.sh serve 22 | # 23 | # TODO: There should be a 'latch run' command that does 'watch and 'serve' 24 | # together. This is like 'webpipe run'. 25 | 26 | 27 | set -o nounset 28 | 29 | readonly THIS_DIR=$(dirname $(readlink -f $0)) 30 | 31 | log() { 32 | echo 1>&2 "$@" 33 | } 34 | 35 | die() { 36 | log "$@" 37 | exit 1 38 | } 39 | 40 | 41 | print-files() { 42 | set -x 43 | #inotifywait --monitor --event close_write "$@" 44 | inotifywait --monitor --event modify "$@" 45 | #inotifywait --monitor --quiet --event close_write "$@" 46 | } 47 | 48 | check-tools() { 49 | which inotifywait >/dev/null \ 50 | || die "inotifywait must be installed (sudo apt-get install inotify-tools)." 51 | 52 | which which >/dev/null \ 53 | || die "curl must be installed (sudo apt-get install curl)." 54 | } 55 | 56 | # Protocol we want: just print out the filename (which is the same as the latch 57 | # name). This unfortunately seems to require editor-specific info. 58 | 59 | wait-default() { 60 | ### Seems to work with Kate GUI editor, vim, VScode 61 | 62 | # With --event move_self, it doesn't print the right path to stdout. 63 | # Not sure if I even need that for Vim and vscodium. 64 | 65 | log "wait-other: Watching $@" 66 | inotifywait --quiet --format '%w' "$@" 67 | } 68 | 69 | # TODO: Delete wait-vscodium and wait-vim? 70 | wait-vscodium() { 71 | log "wait-vscodium: Watching $@" 72 | inotifywait --quiet --format '%w' --event modify "$@" 73 | } 74 | 75 | wait-vim() { 76 | log "wait-vim: Watching $@" 77 | inotifywait --quiet --format '%w' --event move_self "$@" 78 | } 79 | 80 | # NOTE: This does NOT work, because the watch is set on a file, and then it 81 | # BECOMES a different file. wait-vim in a loop is what we want. 82 | 83 | _monitor-vim() { 84 | inotifywait --monitor --format '%w' --event move_self "$@" 85 | } 86 | 87 | # Rebuild in a loop. 88 | # 89 | # Should there be another loop to watch _tmp and notify the latch? Or can it 90 | # be the same loop? 91 | # 92 | # some files are for rebuilding, some are for serving? 93 | 94 | # if there is no build process, then just use "true" ? or maybe cp? 95 | 96 | 97 | readonly LATCH_HOST=localhost:8990 98 | 99 | # Usage: 100 | # ./latch.sh rebuild ... 101 | 102 | rebuild() { 103 | local build_cmd=$1 104 | local wait_cmd=${2:-wait-default} 105 | shift 2 106 | 107 | log "build_cmd: $build_cmd" 108 | 109 | check-tools 110 | 111 | while true; do 112 | # Wait for a changed file 113 | local changed=$($wait_cmd "$@") 114 | log "changed file: $changed" 115 | 116 | # We need to know the output name here relative to _tmp to notify the 117 | # server. 118 | local rel_output="$(dirname $changed)/$(basename $changed .md).html" 119 | # Hacky normalization to remove /./ , since that isn't valid in a URL 120 | rel_output=$(echo $rel_output | sed 's|/./|/|g') 121 | log "rel_output: $rel_output" 122 | 123 | # TODO: Don't hard-code _site! 124 | local output="_site/$rel_output" 125 | 126 | # HACK to sleep 100ms before building. Otherwise we get: 127 | # 128 | # Couldn't watch doc/tutorial.md: No such file or directory 129 | # changed 130 | # ./build.sh: line 68: doc/tutorial.md: No such file or directory 131 | # 132 | # What is happening is that: 133 | # - we get notification of a changed inode 134 | # - but vim hasn't actually saved the new file yet 135 | # - we can't build without the new file 136 | # - TODO: should we listen for another event in wait-vim? move_self vs 137 | # modify? 138 | 139 | sleep 0.1 140 | 141 | # Rebuild 142 | $build_cmd $changed $output 143 | 144 | log "notify $rel_output" 145 | 146 | # Release latch so that the page is refreshed. 147 | notify $rel_output 148 | done 149 | } 150 | 151 | # For Oil posts with presenter notes 152 | one-rebuild-loop() { 153 | local in=$1 154 | local rel_output=$2 # URL 155 | local build_cmd=$3 156 | 157 | while true; do 158 | wait-vim $in 159 | 160 | $build_cmd 161 | notify $rel_output 162 | done 163 | } 164 | 165 | watch() { 166 | which inotifywait \ 167 | || die "inotifywait must be installed (sudo apt-get install inotifytools)." 168 | 169 | local tool=$1 170 | shift 171 | 172 | while true; do 173 | echo Watching "$@" 174 | local exit_code=$? 175 | if test $? -ne 0; then 176 | die "inotifywait failed with code $exit_code" 177 | fi 178 | 179 | # Add latch automatically. 180 | #PULP_latch=1 poly build . 181 | 182 | #curl -d X http://localhost:1212/HOST/latch/default 183 | echo 184 | done 185 | } 186 | 187 | serve() { 188 | export PYTHONPATH=$THIS_DIR:~/git/json-template/python 189 | $THIS_DIR/latch/latch.py "$@" 190 | } 191 | 192 | notify() { 193 | local name=$1 194 | curl --request POST http://$LATCH_HOST/-/latch/$name 195 | } 196 | 197 | help() { 198 | cat $THIS_DIR/doc/latch-help.txt 199 | } 200 | 201 | if test $# -eq 0; then 202 | help 203 | exit 0 204 | fi 205 | 206 | "$@" 207 | -------------------------------------------------------------------------------- /latch/NOTES: -------------------------------------------------------------------------------- 1 | TODO 2 | 3 | - Copy latch server from Poly, and use the same static server pattern as 4 | webpipe. I think latch should be a separate server for now. If people care 5 | about auth, they could make an App file. But we also want to be compatible 6 | with other servers. 7 | 8 | - have a latch for each filename. 9 | 10 | Usage: 11 | 12 | $ latch _tmp/doc 13 | 14 | # starts the server 15 | 16 | - The server should take comments like this: 17 | 18 | 19 | ... 20 | 21 | ... 22 | 23 | 24 | 25 |
26 | 27 | 28 | The JSON Template should have this comment in the section, so that all 29 | docs have it. 30 | 31 | 32 | - Write a shell script like file-latch.sh 33 | 34 | It should take a hook, like 35 | 36 | file-latch.sh doc.sh *.txt 37 | 38 | Any time a .txt file is changed, then it will do the following: 39 | 40 | 1) call out to 41 | 42 | $ doc.sh index.txt 43 | 44 | Which can all make if it wants, but often a shell script will suffice. 45 | 46 | 2) unhook the latch 47 | - should I curl it, or write to a pipe, reading from stdin? 48 | stdin is more 49 | 50 | 51 | - Test the cases of: 52 | 53 | 1) directly editing HTML 54 | 2) editing .txt, having it built, and writng .html to _tmp 55 | 56 | I think you need two inotifywait processes. One for txt, and one for HTML. 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /latch/latch.js: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Google Inc. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be found 3 | // in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 4 | 5 | // Continuously get from a latch. Depends on jQuery. 6 | 7 | function waitForLatch(name) { 8 | //$('#latch-status').text("Waiting for change"); 9 | 10 | var url = '/-/latch/' + name; 11 | 12 | // TODO: use raw XHR to get rid of jQuery dependency. 13 | $.ajax({ 14 | url: url, 15 | type: 'GET', 16 | success: function(data){ 17 | $('#latch-status').text("response: " + data); 18 | location.reload(); 19 | }, 20 | error: function(jqXhr, textStatus, errorThrown) { 21 | // Show error from the server. 22 | $('#latch-status').text( 23 | "error contacting " + url + ": " + jqXhr.responseText); 24 | } 25 | }); 26 | } 27 | 28 | function getLatchName(path) { 29 | // /README.html -> README.html 30 | // TODO: what about index.html? 31 | return path.substring(1); 32 | } 33 | 34 | // Each document has its own latch. 35 | var latchName = getLatchName(window.location.pathname); 36 | waitForLatch(latchName); 37 | 38 | -------------------------------------------------------------------------------- /latch/latch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | """ 8 | latch.py 9 | 10 | Latch server (based on code from polyweb repo). 11 | """ 12 | 13 | import optparse 14 | import re 15 | import os 16 | import sys 17 | import threading 18 | 19 | import jsontemplate 20 | 21 | from common import httpd 22 | from common import util 23 | from common import spy 24 | 25 | import templates 26 | 27 | log = util.Logger(util.ANSI_BLUE) 28 | 29 | 30 | class Error(Exception): 31 | pass 32 | 33 | 34 | class LatchApp(object): 35 | """Get and set latches.""" 36 | 37 | def __init__(self, num_slots=3): 38 | """ 39 | Args: 40 | num_slots: Maximum number of simultaneous waiters. We don't want to take 41 | up all the threads in the server, so this is limited. 42 | """ 43 | self.slots = threading.Semaphore(num_slots) 44 | # threading.Condition()? or Queue.Queue()? 45 | # Simply dictionary 46 | # when you get a GET, just do .get(). Block forever. 47 | # when you get a POST, just do .put(). 48 | self.latches = {'default': threading.Event()} 49 | 50 | def HandleRequest(self, request): 51 | # Is there a better way to do this? 52 | route = request['__META_INTERNAL']['route'] # hack to get route 53 | if route == 'index': 54 | data = {'latches': self.latches.keys()} 55 | elif route == 'wait': 56 | name = request['latch_name'] 57 | 58 | ok = self.slots.acquire(False) 59 | if not ok: 60 | return util.TextResponse(503, 'All slots taken') 61 | 62 | event = self.latches.get(name) 63 | if not event: 64 | return util.TextResponse(404, 'Unknown latch %r' % name) 65 | 66 | start = time.time() 67 | event.wait() 68 | elapsed = time.time() - start 69 | 70 | self.slots.release() 71 | 72 | return util.TextResponse(200, 73 | 'Waited %.2f seconds for latch %r.' % (elapsed, name)) 74 | 75 | elif route == 'notify': 76 | name = request['latch_name'] 77 | event = self.latches.get(name) 78 | if not event: 79 | return util.TextResponse(404, 'Unknown latch %r' % name) 80 | event.set() 81 | 82 | # Reset the flag so we can wait again. 83 | event.clear() 84 | return util.TextResponse(200, 'Notified all waiters on latch %r.' % name) 85 | 86 | else: 87 | # App should have prevented this 88 | raise AssertionError("Invalid route %r" % route) 89 | return {'body_data': data} 90 | 91 | 92 | class Latches(object): 93 | 94 | def __init__(self, num_slots=5): 95 | """ 96 | Args: 97 | num_slots: Maximum number of simultaneous waiters. We don't want to take 98 | up all the threads in the server, so this is limited. 99 | """ 100 | self.slots = threading.Semaphore(num_slots) 101 | # threading.Condition()? or Queue.Queue()? 102 | # Simply dictionary 103 | # when you get a GET, just do .get(). Block forever. 104 | # when you get a POST, just do .put(). 105 | self.latches = {} 106 | self.lock = threading.Lock() # protect self.latches 107 | 108 | def Wait(self, name): 109 | assert isinstance(name, str) and len(name) > 0, name 110 | 111 | with self.lock: # don't want a race between checking and setting 112 | event = self.latches.get(name) 113 | if event is None: 114 | event = threading.Event() 115 | self.latches[name] = event 116 | 117 | log('waiting on %r', name) 118 | event.wait() 119 | 120 | def Notify(self, name): 121 | """Returns whether the named latch was successfully notified.""" 122 | event = self.latches.get(name) 123 | if event: 124 | event.set() 125 | # Reset the flag so we can wait again. 126 | event.clear() 127 | return True 128 | else: 129 | return False 130 | 131 | 132 | def CreateOptionsParser(): 133 | parser = optparse.OptionParser('webpipe_main [options]') 134 | 135 | parser.add_option( 136 | '-v', '--verbose', dest='verbose', default=False, action='store_true', 137 | help='Write more log messages') 138 | parser.add_option( 139 | '--port', dest='port', type='int', default=8990, 140 | help='Port to serve on') 141 | parser.add_option( 142 | '--num-threads', dest='num_threads', type='int', default=5, 143 | help='Number of server threads, i.e. simultaneous connections.') 144 | 145 | parser.add_option( 146 | '--root-dir', dest='root_dir', type='str', 147 | default='_tmp', 148 | help='Directory to serve out of.') 149 | 150 | return parser 151 | 152 | 153 | HOME_PAGE = jsontemplate.Template("""\ 154 |

latch

155 | 156 | {.repeated section pages} 157 | {@}
158 | {.end} 159 | """, default_formatter='html') 160 | 161 | 162 | LATCH_PATH_RE = re.compile(r'/-/latch/(\S+)$') 163 | 164 | # TODO: Rewrite latch.js using raw XHR, and get rid of jQuery. This could 165 | # interfere with pages that have jQuery already. 166 | LATCH_HEAD = """\ 167 | 170 | 171 | 172 | """ 173 | 174 | LATCH_BODY = """\ 175 |

Waiting for latch...

176 | """ 177 | 178 | class LatchRequestHandler(httpd.BaseRequestHandler): 179 | """ 180 | Notify latches 181 | """ 182 | server_version = "Latch" 183 | latches = None 184 | latch_js = None 185 | 186 | def send_index(self): 187 | self.send_response(200) 188 | self.send_header('Content-Type', 'text/html') 189 | self.end_headers() 190 | 191 | pages = os.listdir(self.root_dir) 192 | pages.sort(reverse=True) 193 | html = HOME_PAGE.expand({'pages': pages}) 194 | self.wfile.write(html) 195 | 196 | def send_content(self, content_type, body): 197 | self.send_response(200) 198 | self.send_header('Content-Type', content_type) 199 | self.end_headers() 200 | 201 | self.wfile.write(body) 202 | 203 | def send_404(self, msg): 204 | self.send_response(404) 205 | self.send_header('Content-Type', 'text/plain') 206 | self.end_headers() 207 | 208 | self.wfile.write(msg + '\n') 209 | 210 | def do_GET(self): 211 | """Serve a GET request.""" 212 | 213 | # NOTE: 214 | # GET notifies? 215 | # POST notifies? 216 | # do_POST? 217 | 218 | if self.path == '/': 219 | self.send_index() 220 | return 221 | 222 | if self.path == '/-/latch.js': 223 | self.send_content('application/javascript', self.latch_js) 224 | return 225 | 226 | m = LATCH_PATH_RE.match(self.path) 227 | if m: 228 | name = m.group(1) 229 | log('GET LATCH %s', name) 230 | 231 | # wait on or create the latch 232 | self.latches.Wait(name) 233 | 234 | self.send_content('text/plain', 'ok') 235 | return 236 | 237 | # Serve static file. 238 | # TODO: if it ends with HTML, search for 239 | 240 | f = self.send_head() 241 | if f: 242 | for line in f: 243 | stripped = line.strip() 244 | if stripped == '': 245 | out = LATCH_HEAD 246 | log('replaced %r', stripped) 247 | elif stripped == '': 248 | out = LATCH_BODY 249 | log('replaced %r', stripped) 250 | else: 251 | out = line 252 | self.wfile.write(out) 253 | f.close() 254 | 255 | def do_POST(self): 256 | """Serve a POST request.""" 257 | m = LATCH_PATH_RE.match(self.path) 258 | if not m: 259 | self.send_404('invalid resource %r' % self.path) 260 | return 261 | 262 | name = m.group(1) 263 | log('POST LATCH %s', name) 264 | 265 | success = self.latches.Notify(name) 266 | log('success %s', success) 267 | 268 | if success: 269 | self.send_content('text/plain', 'notified %r\n' % name) 270 | else: 271 | self.send_404('no latch named %r' % name) 272 | 273 | 274 | def main(argv): 275 | """Returns an exit code.""" 276 | 277 | (opts, _) = CreateOptionsParser().parse_args(argv[1:]) 278 | 279 | # TODO: 280 | # pass request handler map 281 | # - index 282 | # - list self.latches ? 283 | # - if you click, does it wait? 284 | # /-/latch.js 285 | # /-/latch/README.html 286 | # /-/wait/ 287 | # /-/notify/ -- or do those come on stdin? or POST? 288 | 289 | # - latch 290 | # - static 291 | # - except this filters self.wfile 292 | # - 293 | 294 | latches = Latches() 295 | 296 | d = os.path.dirname(sys.argv[0]) 297 | path = os.path.join(d, 'latch.js') 298 | with open(path) as f: 299 | latch_js = f.read() 300 | 301 | handler_class = LatchRequestHandler 302 | handler_class.root_dir = opts.root_dir 303 | handler_class.latches = latches 304 | handler_class.latch_js = latch_js 305 | 306 | s = httpd.ThreadedHTTPServer(('', opts.port), handler_class) 307 | 308 | log("Serving dir %s on port %d... (Ctrl-C to quit)", 309 | opts.root_dir, opts.port) 310 | 311 | s.serve_forever() 312 | 313 | 314 | if __name__ == '__main__': 315 | try: 316 | sys.exit(main(sys.argv)) 317 | except Error, e: 318 | print >> sys.stderr, e.args[0] 319 | sys.exit(1) 320 | -------------------------------------------------------------------------------- /latch/latch_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python -S 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | """ 8 | latch_test.py: Tests for latch.py 9 | """ 10 | 11 | import unittest 12 | 13 | import latch # module under test 14 | 15 | 16 | class LatchTest(unittest.TestCase): 17 | def setUp(self): 18 | pass 19 | 20 | def tearDown(self): 21 | pass 22 | 23 | def testLatch(self): 24 | la = latch.Latches() 25 | print 'hi' 26 | #la.Wait('foo') 27 | success = la.Notify('foo') 28 | self.assertEqual(False, success) 29 | 30 | 31 | if __name__ == '__main__': 32 | unittest.main() 33 | -------------------------------------------------------------------------------- /latch/templates.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | """ 8 | templates.py 9 | """ 10 | 11 | import sys 12 | 13 | import jsontemplate 14 | 15 | 16 | -------------------------------------------------------------------------------- /live-plugins/demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | # 8 | # This is an idea to get live visualizations in the browser. 9 | # 10 | # You have a tiny shell/Python program which writes to stdout. It gets piped 11 | # to netcat, through ssh tunnel, and then to the webpipe server. 12 | # 13 | # Is netcat reliable? Yeah it seems to work better for this "stream" use case. 14 | # Still use it as a client only though. 15 | # 16 | # PLUGIN INTERFACE 17 | # 18 | # The plugins need to send the initial HTML and JS as they always do. 19 | # (NOTE: right now the server directly reads the JS; it isn't sent) 20 | # 21 | # NOTE: This isn't possible for static publishing... only live a live server. 22 | # (although a crazy idea is if you load a JS file that replays it somehow) 23 | # 24 | # webpipe server should listen on an auxiliary port perhaps. 25 | # And then every live plugin will make a new connection on that port, send its 26 | # ID? 27 | # 28 | # And then the server has to deliver those to different JS clients. 29 | # It probably needs some kind of ID. 30 | # 31 | # NOTE: This should possibly be a different server than webpipe. The model is 32 | # different; it's not compatible with static web hosting. 33 | # 34 | # And you probably don't want a "scroll". You probably want a page of live 35 | # 36 | # And you need a different protocol than just a hanging get. Although I guess 37 | # you can dispatch by URL. Every plugin can send the URL it wants? 38 | # 39 | # 8901 - jspipe? This name is too common. livepipe? 40 | # 41 | # Usage: 42 | # ./demo.sh 43 | 44 | set -o nounset 45 | set -o pipefail 46 | # set -o errexit # most scripts should set this 47 | 48 | cpu() { 49 | while true; do 50 | head -n1 /proc/stat 51 | sleep 1 52 | done 53 | } 54 | 55 | "$@" 56 | -------------------------------------------------------------------------------- /plugins/DEFAULT/render: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | 8 | readonly THIS_DIR=$(dirname $0) 9 | . $THIS_DIR/../webpipe-lib.sh 10 | 11 | checkDeps() { 12 | local msg="file not found. Run 'sudo apt-get install file'" 13 | which file >/dev/null || die "$msg" 14 | } 15 | 16 | showFile() { 17 | local eInput=$1 18 | file $eInput 19 | echo 20 | 21 | # Show file stat. TODO: show size pretty printed? 22 | # note: filename is escaped so could be in a different location 23 | stat $(readlink -f $eInput) 24 | } 25 | 26 | showHtml() { 27 | local eInput="$1" 28 | echo '(no render plugin, DEFAULT used)

' 29 | echo '

'
30 |   showFile $eInput | WP_HtmlEscape
31 |   echo '
' 32 | } 33 | 34 | main() { 35 | local eInput=$1 36 | local eOutput=$2 37 | 38 | checkDeps 39 | 40 | local html=$eOutput.html 41 | showHtml $eInput WP_HtmlEscape >$html 42 | echo $html 43 | } 44 | 45 | main "$@" 46 | 47 | -------------------------------------------------------------------------------- /plugins/R/render: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | # 8 | # Plugin that calls pygmentize to highlight R source files. 9 | # 10 | # TODO: how to choose a plugin that would provide a function outline with 11 | # hyperlinks? 12 | 13 | readonly THIS_DIR=$(dirname $0) 14 | . $THIS_DIR/../webpipe-lib.sh 15 | 16 | checkDeps() { 17 | local msg="pygmentize not found. Run 'sudo apt-get install python-pygments'" 18 | which pygmentize >/dev/null || die $msg 19 | } 20 | 21 | main() { 22 | local input=$1 23 | local output=$2 24 | 25 | checkDeps 26 | 27 | # fail if pygmentize fails, etc. 28 | set -o errexit 29 | 30 | mkdir -p $output 31 | 32 | local inputFilename=$(basename $input) 33 | local origOut=$output/$inputFilename 34 | 35 | cp $input $origOut 36 | 37 | echo $output # finished writing directory 38 | 39 | local html=${output}.html 40 | 41 | # Generate HTML. Note that without the 'full' options, there will be no color. 42 | # Unfortunately that produces 43 | pygmentize -f html -O full,style=emacs $input >>$html 44 | 45 | # This is bad 46 | cat >>$html < 48 | Download $inputFilename 49 |

50 | EOF 51 | 52 | echo $html 53 | } 54 | 55 | main "$@" 56 | -------------------------------------------------------------------------------- /plugins/R/testdata/tiny.R: -------------------------------------------------------------------------------- 1 | #!/usr/bin/Rscript 2 | 3 | print("hi") 4 | print(rnorm(10)) 5 | 6 | -------------------------------------------------------------------------------- /plugins/README: -------------------------------------------------------------------------------- 1 | The plugins/ directory is for webpipe plugins. 2 | 3 | Plugins are basically scripts (usually shell) that take a file as input and 4 | produce static HTML as output, and can have associated JS/CSS. 5 | 6 | Each subdirectory here is a plugin for a particular type of file. 7 | 8 | See doc/plugins.md for details. 9 | 10 | 11 | -------------------------------------------------------------------------------- /plugins/Rplot.png/render: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | # 8 | # Surround .png files with an tag. Right now this is used for R plots; 9 | # later we could have something more sophisticated (e.g. show underlying data). 10 | 11 | readonly THIS_DIR=$(dirname $0) 12 | . $THIS_DIR/../webpipe-lib.sh 13 | 14 | main() { 15 | local input=$1 16 | local output=$2 17 | 18 | # fail if cp fails, etc. 19 | set -o errexit 20 | 21 | mkdir -p $output 22 | cp $input $output 23 | 24 | echo $output # finished writing directory 25 | 26 | local basename=$(basename $input) 27 | 28 | cat >$output.html < 30 | 31 | 32 | EOF 33 | 34 | echo $output.html # finished writing 35 | } 36 | 37 | main "$@" 38 | -------------------------------------------------------------------------------- /plugins/_bin/render-tar.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | 8 | readonly THIS_DIR=$(dirname $0) 9 | . $THIS_DIR/../webpipe-lib.sh 10 | 11 | checkDeps() { 12 | local msg="tar not found. Run 'sudo apt-get install tar'" 13 | which tar >/dev/null || die "$msg" 14 | } 15 | 16 | showFile() { 17 | local eInput=$1 18 | local tarFlag=$2 19 | 20 | # TODO: 21 | # - factor into WP_GetFileSize? 22 | # - would be nice to sum the column, show compression ratio, etc. 23 | 24 | local size=$(stat --printf '%s' $eInput) 25 | echo "

$eInput, $size bytes

" 26 | 27 | echo '
'
28 | 
29 |   # notes:
30 |   # - assumes GNU tar
31 |   # - verbose shows file perms, etc.
32 |   tar --verbose --list $tarFlag < $eInput | WP_HtmlEscape 
33 | 
34 |   echo '
' 35 | } 36 | 37 | main() { 38 | local eInput=$1 39 | local eOutput=$2 40 | local tarFlag=$3 # -z, -j, etc. 41 | 42 | checkDeps 43 | 44 | set -o errexit 45 | 46 | local html=$eOutput.html 47 | 48 | showFile $eInput $tarFlag >$html 49 | 50 | echo $html 51 | } 52 | 53 | main "$@" 54 | 55 | -------------------------------------------------------------------------------- /plugins/ansi/README.md: -------------------------------------------------------------------------------- 1 | NOTE on generating testdata: 2 | 3 | script -c sh # use regular shell so prompt is more generic 4 | 5 | ls 6 | ls --color 7 | -------------------------------------------------------------------------------- /plugins/ansi/render: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | 8 | readonly THIS_DIR=$(dirname $0) 9 | . $THIS_DIR/../webpipe-lib.sh 10 | 11 | checkDeps() { 12 | local msg="aha not found. Run 'sudo apt-get install aha'" 13 | which aha >/dev/null || die "$msg" 14 | } 15 | 16 | main() { 17 | local input=$1 18 | local output=$2 19 | 20 | checkDeps 21 | 22 | aha -f $input > $output.html 23 | echo $output.html 24 | } 25 | 26 | main "$@" 27 | 28 | -------------------------------------------------------------------------------- /plugins/ansi/testdata/typescript: -------------------------------------------------------------------------------- 1 | Script started on Fri 04 Apr 2014 12:39:11 PM PDT 2 | $ ls 3 | Auto deps.sh doc.sh latch.sh NOTES README.md static _tmp webpipe webpipe-test.sh wp.sh wp-test.sh 4 | common doc latch Makefile plugins run.sh testdata typescript webpipe.R wp-dev.sh wp-stub.sh 5 | $ ls --color 6 | Auto deps.sh doc.sh latch.sh NOTES README.md static _tmp webpipe webpipe-test.sh wp.sh wp-test.sh 7 | common doc latch Makefile plugins run.sh testdata typescript webpipe.R wp-dev.sh wp-stub.sh 8 | $ exit 9 | 10 | Script done on Fri 04 Apr 2014 12:39:15 PM PDT 11 | -------------------------------------------------------------------------------- /plugins/csv/render: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | # 8 | # This is just so we can call keep the .py extension. 9 | 10 | exec $(dirname $0)/render.py "$@" 11 | -------------------------------------------------------------------------------- /plugins/csv/render-dev.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | # 8 | # Development stub. For now, when deployed, the plugin can import jsontemplate 9 | # because PYTHONPATH leaks from xrender.py to its child processes. May want to 10 | # change that at some point, i.e. create a $WEBPIPE_LIB_DIR variable or 11 | # something. 12 | # 13 | # Usage: 14 | # ./render-dev.sh 15 | 16 | export PYTHONPATH=~/hg/json-template/python 17 | 18 | exec $(dirname $0)/render.py "$@" 19 | -------------------------------------------------------------------------------- /plugins/csv/render.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | """ 8 | csv_plugin.py 9 | 10 | This is in process in xrender.py until we work out packaging issues. 11 | """ 12 | 13 | import csv 14 | import os 15 | import shutil 16 | import sys 17 | 18 | import jsontemplate 19 | 20 | class Error(Exception): 21 | pass 22 | 23 | # See http://datatables.net/usage/ 24 | # CDN: http://www.asp.net/ajaxlibrary/CDNjQueryDataTables194.ashx 25 | # 26 | # TODO: 27 | # - consider putting row numbers next to each row, in grey. It will make it 28 | # clearer. 29 | # - refactor template to remove duplication 30 | # - could you use R to show histograms of the rows? 31 | # - We can put this in the /static/ dir 32 | 33 | # TODO: 34 | # - generate a different table ID for each one, and then style only that? 35 | # - I think you can generate the ID in the web roll. That makes a lot more 36 | # sense, since it's dynamic. 37 | # -
38 | # - don't want every plugin to hard code jquery.getScript 39 | # - would be nicer to present 40 | 41 | def Commas(n): 42 | return '{:,}'.format(n) 43 | 44 | 45 | FORMATTERS = {'commas': Commas} 46 | 47 | # TODO: 48 | # - data table should just be normal script tag. 49 | # - all assets should be loaded from the server? Except maybe jQuery. 50 | 51 | TABLE_TEMPLATE = jsontemplate.Template("""\ 52 | 53 | 54 | 57 | 60 | 61 | 63 | 64 | 65 | 66 |

{basename} - {num_rows|commas} rows, {num_bytes|commas} 67 | bytes

68 | 69 | 70 | 71 | {.repeated section thead} {.end} 72 | 73 | 74 | {.repeated section rows} 75 | {.repeated section @} {.end} 76 | {.end} 77 | 78 |
{@}
{@}
79 | 80 | 81 | 84 | 85 | 86 | """, default_formatter='html', more_formatters=FORMATTERS) 87 | 88 | 89 | PREVIEW_TEMPLATE = jsontemplate.Template("""\ 90 |

{basename} - {num_rows|commas} rows, {num_bytes|commas} 91 | bytes

92 | 93 | 94 | 95 | {.repeated section thead} {.end} 96 | 97 | 98 | {# show either a preview, or full rows} 99 | 100 | {.if test head} 101 | {.repeated section head} 102 | {.repeated section @} {.end} 103 | {.end} 104 | 105 | 106 | 109 | 110 | 111 | {.repeated section tail} 112 | {.repeated section @} {.end} 113 | {.end} 114 | 115 | {.or} 116 | {.repeated section rows} 117 | {.repeated section @} {.end} 118 | {.end} 119 | 120 | {.end} 121 | 122 | 123 |
{@}
{@}
107 | ... {num_omitted|commas} rows omitted 108 |
{@}
{@}
124 | 125 |

Download Original CSV

126 | 127 | """, default_formatter='html', more_formatters=FORMATTERS) 128 | 129 | 130 | # User setting for how many lines of head/tail they want to see. 131 | wp_num_lines = os.getenv('WP_NUM_LINES', 5) 132 | 133 | # TODO: avoid loading the entire thing in memory? 134 | def CsvDataDict(f): 135 | """ 136 | Turn CSV into an HTML table. 137 | 138 | TODO: maximum number of rows. 139 | """ 140 | c = csv.reader(f) 141 | rows = [] 142 | d = {'rows': rows} 143 | 144 | num_rows = 0 145 | for i, row in enumerate(c): 146 | #print 'R', row 147 | if i == 0: 148 | d['thead'] = row 149 | else: 150 | rows.append(row) 151 | num_rows += 1 152 | 153 | # wp_num_lines is 5, then we should show in full anything less than 15 rows 154 | if len(rows) >= wp_num_lines * 3: 155 | d['num_omitted'] = num_rows - (wp_num_lines * 2) 156 | d['head'] = head = rows[ : wp_num_lines] 157 | d['tail'] = rows[-wp_num_lines : ] 158 | 159 | d['num_rows'] = num_rows - 1 # omit one that we counted as the header 160 | 161 | return d 162 | 163 | 164 | def main(argv): 165 | """Returns an exit code.""" 166 | 167 | # Assume we're in the output dir 168 | input_path, output = argv[1:] 169 | 170 | os.mkdir(output) 171 | basename = os.path.basename(input_path) 172 | orig = os.path.join(output, basename) 173 | 174 | num_bytes = os.path.getsize(input_path) 175 | 176 | # Copy the original 177 | shutil.copy(input_path, orig) 178 | 179 | full_html = os.path.join(output, 'full.html') 180 | with open(full_html, 'w') as outfile: 181 | with open(input_path) as infile: 182 | data_dict = CsvDataDict(infile) 183 | data_dict['num_bytes'] = num_bytes 184 | data_dict['output'] = output 185 | data_dict['basename'] = basename 186 | # So we can have a colspan 187 | data_dict['num_cols'] = len(data_dict['thead']) 188 | 189 | outfile.write(TABLE_TEMPLATE.expand(data_dict)) 190 | 191 | print output # finished the dir 192 | 193 | html = output + '.html' 194 | with open(html, 'w') as f: 195 | # TODO: check how many rows, and write head/tail, or full thing. 196 | f.write(PREVIEW_TEMPLATE.expand(data_dict)) 197 | 198 | print html # wrote html 199 | 200 | return 0 201 | 202 | 203 | if __name__ == '__main__': 204 | try: 205 | sys.exit(main(sys.argv)) 206 | except Error, e: 207 | print >> sys.stderr, e.args[0] 208 | sys.exit(1) 209 | -------------------------------------------------------------------------------- /plugins/csv/static/bar.txt: -------------------------------------------------------------------------------- 1 | bar 2 | -------------------------------------------------------------------------------- /plugins/csv/testdata/tiny.csv: -------------------------------------------------------------------------------- 1 | name,age 2 | ,10 3 | ,20 4 | 5 | -------------------------------------------------------------------------------- /plugins/dot/render: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | 8 | readonly THIS_DIR=$(dirname $0) 9 | . $THIS_DIR/../webpipe-lib.sh 10 | 11 | checkDeps() { 12 | local msg="dot not found. Run 'sudo apt-get install graphviz'" 13 | which dot >/dev/null || die $msg 14 | } 15 | 16 | main() { 17 | local input=$1 18 | local output=$2 19 | 20 | checkDeps 21 | 22 | # fail if dot fails, etc. 23 | set -o errexit 24 | 25 | mkdir -p $output 26 | 27 | local pngOut="$output/$(WP_BasenameNoExt $input).png" 28 | dot -T png -o $pngOut $input 29 | 30 | local inputFilename=$(basename $input) 31 | local origOut=$output/$inputFilename.txt 32 | 33 | cp $input $origOut 34 | 35 | echo $output # finished writing directory 36 | 37 | cat >$output.html < 39 | $inputFilename.txt 40 |
41 | Rendered dot image 42 |
43 |

44 | EOF 45 | 46 | echo $output.html # wrote it 47 | } 48 | 49 | main "$@" 50 | -------------------------------------------------------------------------------- /plugins/dot/testdata/cluster.dot: -------------------------------------------------------------------------------- 1 | # http://graphviz.org/content/cluster 2 | digraph G { 3 | 4 | subgraph cluster_0 { 5 | style=filled; 6 | color=lightgrey; 7 | node [style=filled,color=white]; 8 | a0 -> a1 -> a2 -> a3; 9 | label = "process #1"; 10 | } 11 | 12 | subgraph cluster_1 { 13 | node [style=filled]; 14 | b0 -> b1 -> b2 -> b3; 15 | label = "process #2"; 16 | color=blue 17 | } 18 | start -> a0; 19 | start -> b0; 20 | a1 -> b3; 21 | b2 -> a3; 22 | a3 -> a0; 23 | a3 -> end; 24 | b3 -> end; 25 | 26 | start [shape=Mdiamond]; 27 | end [shape=Msquare]; 28 | } 29 | -------------------------------------------------------------------------------- /plugins/html/render: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | 8 | readonly THIS_DIR=$(dirname $0) 9 | . $THIS_DIR/../webpipe-lib.sh 10 | 11 | main() { 12 | local input=$1 13 | local output=$2 14 | 15 | # TODO: 16 | # - It would be nice to check the doctype and display it. Maybe use W3C 17 | # tools or something? 18 | # 19 | # If the HTML is too big, show a preview? 20 | 21 | cp $input $output.html 22 | echo $output.html 23 | } 24 | 25 | main "$@" 26 | 27 | -------------------------------------------------------------------------------- /plugins/html/testdata/example.html: -------------------------------------------------------------------------------- 1 | hello.html 2 | -------------------------------------------------------------------------------- /plugins/html/testdata/tiny.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |

HTML

4 |
    5 |
  • Bold
    • 6 |
  • Italic
    • 7 |
  • Code
    • 8 |
9 | -------------------------------------------------------------------------------- /plugins/markdown/gallery.md: -------------------------------------------------------------------------------- 1 | [Markdown](https://daringfireball.net/projects/markdown/) files are rendered to HTML and displayed inline. 2 | -------------------------------------------------------------------------------- /plugins/markdown/render: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | 8 | readonly THIS_DIR=$(dirname $0) 9 | . $THIS_DIR/../webpipe-lib.sh 10 | 11 | checkDeps() { 12 | local msg="markdown not found. Run 'sudo apt-get install markdown'" 13 | which markdown >/dev/null || die "$msg" 14 | } 15 | 16 | main() { 17 | local input=$1 18 | local output=$2 19 | 20 | checkDeps 21 | 22 | # TODO: Test size? 23 | local html=$output.html 24 | markdown <$input >$html 25 | echo $html 26 | } 27 | 28 | main "$@" 29 | 30 | -------------------------------------------------------------------------------- /plugins/markdown/testdata/tiny.markdown: -------------------------------------------------------------------------------- 1 | title 2 | ===== 3 | 4 | This is *markdown* text. 5 | 6 | def foo(): 7 | for i in range(10): 8 | print i 9 | -------------------------------------------------------------------------------- /plugins/tar.bz2/render: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | 8 | thisDir=$(dirname $0) 9 | # pass input and output, append --gzip 10 | exec $thisDir/../_bin/render-tar.sh "$@" --bzip2 11 | -------------------------------------------------------------------------------- /plugins/tar.gz/render: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | 8 | thisDir=$(dirname $0) 9 | # pass input and output, append --gzip 10 | exec $thisDir/../_bin/render-tar.sh "$@" --gzip 11 | -------------------------------------------------------------------------------- /plugins/tar.xz/render: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | 8 | thisDir=$(dirname $0) 9 | # pass input and output, append --gzip 10 | exec $thisDir/../_bin/render-tar.sh "$@" --xz 11 | -------------------------------------------------------------------------------- /plugins/txt/render: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | 8 | readonly THIS_DIR=$(dirname $0) 9 | . $THIS_DIR/../webpipe-lib.sh 10 | 11 | set -o errexit 12 | 13 | main() { 14 | local input=$1 15 | local output=$2 16 | 17 | # If the text is too big, show a preview? 18 | # Show unicode properties, etc. 19 | 20 | local html=$output.html 21 | 22 | echo '
' >$html
23 | 
24 |   WP_HtmlEscape <$input >>$html
25 | 
26 |   echo '
' >>$html 27 | 28 | echo $html 29 | } 30 | 31 | main "$@" 32 | 33 | -------------------------------------------------------------------------------- /plugins/txt/testdata/example.txt: -------------------------------------------------------------------------------- 1 | Hello from text file. 2 | 3 | Rendered as HTML. 4 | 5 | -------------------------------------------------------------------------------- /plugins/txt/testdata/tiny.txt: -------------------------------------------------------------------------------- 1 | This a plain text file. & stuff aren't special. 2 | Line two. 3 | Line three. 4 | 5 | -------------------------------------------------------------------------------- /plugins/webpipe-lib-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | # Testing under /bin/sh. 8 | # 9 | # 10 | # Usage: 11 | # ./webpipe-lib-test.sh 12 | 13 | . $PWD/webpipe-lib.sh 14 | 15 | testBasenameExt() { 16 | WP_BasenameNoExt foo 17 | WP_BasenameNoExt foo.bar 18 | WP_BasenameNoExt /foo 19 | WP_BasenameNoExt /foo.bar 20 | WP_BasenameNoExt ../foo 21 | WP_BasenameNoExt ../foo.bar 22 | WP_BasenameNoExt spam/../foo 23 | WP_BasenameNoExt spam/../foo.bar 24 | } 25 | 26 | main() { 27 | testBasenameExt 28 | } 29 | 30 | main "$@" 31 | -------------------------------------------------------------------------------- /plugins/webpipe-lib.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Google Inc. All rights reserved. 2 | # Use of this source code is governed by a BSD-style license that can be found 3 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 4 | 5 | # webpipe-lib.sh 6 | # 7 | # Not executable, but should be sourceable by /bin/sh. 8 | 9 | # every script needs this 10 | set -o nounset 11 | 12 | # stdout is important, so provide something to log to stderr. 13 | log() { 14 | echo 1>&2 "$@" 15 | } 16 | 17 | # failure to create tools 18 | die() { 19 | log "$@" 20 | exit 1 21 | } 22 | 23 | # Extract base filename, without extension. Useful determining the output path 24 | # of converters. 25 | # 26 | # GNU basename doesn't seem to let you remove an arbitrary extension. 27 | WP_BasenameNoExt() { 28 | local path=$1 29 | python -S -c ' 30 | import os, sys 31 | filename = os.path.basename(sys.argv[1]) # spam/eggs.c -> eggs.c 32 | base, _ = os.path.splitext(filename) # eggs.c -> eggs 33 | print base' \ 34 | $path 35 | } 36 | 37 | # Takes raw text on stdin, and outputs HTML safe text on stdout. We always 38 | # escape & < > and ". (Don't use single quotes for attributes!) 39 | 40 | # NOTE: sed probably makes multiple passes to do this, but so does Python's 41 | # cgi.escape. 42 | WP_HtmlEscape() { 43 | sed 's|&|\&|g; s|<|\<|g; s|>|\>|g; s|"|\"|g' 44 | } 45 | 46 | -------------------------------------------------------------------------------- /plugins/zip/render: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | 8 | readonly THIS_DIR=$(dirname $0) 9 | . $THIS_DIR/../webpipe-lib.sh 10 | 11 | checkDeps() { 12 | local msg="zip not found. Run 'sudo apt-get install zip'" 13 | which zip >/dev/null || die "$msg" 14 | } 15 | 16 | main() { 17 | local input=$1 18 | local output=$2 19 | 20 | checkDeps 21 | 22 | local html=$output.html 23 | echo '
' >$html
24 |   unzip -l $input | WP_HtmlEscape >>$html
25 |   echo '
' >>$html 26 | 27 | echo $html 28 | } 29 | 30 | main "$@" 31 | 32 | -------------------------------------------------------------------------------- /plugins/zip/testdata/tiny.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andychu/webpipe/78fe629829af0aa3df1854501d29df68c8c010df/plugins/zip/testdata/tiny.zip -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | # 8 | # Misc scripts. 9 | # 10 | # Usage: 11 | # ./run.sh 12 | 13 | set -o nounset 14 | 15 | log() { 16 | echo 1>&2 "$@" 17 | } 18 | 19 | unit() { 20 | local this_dir=$(dirname $0) 21 | export PYTHONPATH=$this_dir:~/hg/tnet/python:~/hg/json-template/python 22 | "$@" 23 | } 24 | 25 | usage-sink() { 26 | # listen in UDP mode on port 8988 27 | log 'usage sink' 28 | nc -v -u -l localhost 8988 29 | } 30 | 31 | usage-send() { 32 | echo foo | nc -u localhost 8988 33 | } 34 | 35 | usage-config() { 36 | local addr='localhost:19999' 37 | echo $addr > webpipe/usage-address.txt 38 | echo $addr > latch/usage-address.txt 39 | } 40 | 41 | # 42 | # Gen testdata 43 | # 44 | 45 | make-Rplot-testdata() { 46 | local dir=plugins/Rplot.png/testdata 47 | mkdir -p $dir 48 | local path=$dir/tiny.Rplot.png 49 | 50 | R --vanilla --slave <$output/index.html < 21 | 22 | json $output 23 | 24 | 25 | 26 | 27 | 28 |
29 | 30 | 31 | 34 | 35 | 36 | EOF 37 | 38 | echo $output 39 | 40 | local html=$output.html 41 | 42 | # TODO: Preview in the snippet. Should give first 10 and 43 | # last 10 lines in here? Should we use Python, or just head/tail? 44 | # more stats 45 | # - depth of tree? 46 | # - histogram of node types? (object: 10, array: 20, etc.) 47 | # - histogram of object keys? 48 | 49 | local stats="$(wc $input)" 50 | # num lines, words, chars 51 | local numLines=$(echo $stats | awk '{print $1}') 52 | local numChars="$(echo $stats | awk '{print $3}')" 53 | 54 | cat >$html < $filename: $numChars chars, $numLines lines

56 | 57 | navigate json
58 | 59 | raw json
60 | EOF 61 | 62 | echo $html 63 | } 64 | 65 | main "$@" 66 | 67 | -------------------------------------------------------------------------------- /third_party/json/static/jsontree.css: -------------------------------------------------------------------------------- 1 | /* json string right " */ 2 | .json-object-expand:after { 3 | content: '+'; 4 | } 5 | .json-object-collapse:after { 6 | content: '-'; 7 | } 8 | 9 | /* properties for expand icon */ 10 | .json-object-expand { 11 | display: inline-block; 12 | font-weight: 500; 13 | color: #999; 14 | margin-left: 1px; 15 | margin-right: 1px; 16 | cursor: pointer; 17 | } 18 | /* properties for collapse icon */ 19 | .json-object-collapse { 20 | display: inline-block; 21 | font-weight: 500; 22 | color: #999; 23 | margin-left: 1px; 24 | margin-right: 1px; 25 | cursor: pointer; 26 | } 27 | /* global properties for json object */ 28 | .json-content { 29 | color: #666; 30 | font-family: "Lucida Console", Monaco, monospace; 31 | } 32 | /* json property name style */ 33 | .json-property { 34 | font-weight: 500; 35 | } 36 | /* json number style */ 37 | .json-number { 38 | color: #B627B6; 39 | } 40 | /* json string style */ 41 | .json-string { 42 | color: #2DB669; 43 | } 44 | /* json boolean style */ 45 | .json-boolean { 46 | color: #2525CC; 47 | } 48 | /* json null style */ 49 | .json-null { 50 | color: gray; 51 | } 52 | /* json object */ 53 | .json-object { 54 | padding-left: 2em; 55 | } 56 | /* visible object */ 57 | .json-visible { 58 | height: auto; 59 | } 60 | /* collapsed object */ 61 | .json-collapsed { 62 | display: none; 63 | } 64 | -------------------------------------------------------------------------------- /third_party/json/static/jsontree.js: -------------------------------------------------------------------------------- 1 | // JSONTree 2 | // 3 | // Heavily restructured version of: https://github.com/lmenezes/json-tree 4 | // That that code had some issues with "JSONTree.create". I changed it to just 5 | // use plain functions. 6 | 7 | var JSONTree = function() { 8 | 9 | function createNodes(data) { 10 | return divNode(jsValue(data), {'class': 'json-content'}); 11 | } 12 | 13 | var id = 0; 14 | 15 | var newId = function() { 16 | id += 1; 17 | return id; 18 | } 19 | 20 | var escapeMap = { 21 | '&': '&', 22 | '<': '<', 23 | '>': '>', 24 | '"': '"', 25 | "'": ''', 26 | '/': '/' 27 | }; 28 | 29 | var escape = function(text) { 30 | return text.replace(/[&<>'"]/g, function(t) { 31 | return escapeMap[t]; 32 | }); 33 | } 34 | 35 | var divNode = function(text, attrs) { 36 | return htmlNode('div', text, attrs); 37 | } 38 | 39 | var spanNode = function(text, attrs) { 40 | return htmlNode('span', text, attrs); 41 | } 42 | 43 | var htmlNode = function(type, text, attrs) { 44 | var html = '<' + type; 45 | if (attrs != null) { 46 | Object.keys(attrs).forEach(function(attr) { 47 | html += ' ' + attr + '=\"' + attrs[attr] + '\"'; 48 | }); 49 | } 50 | html += '>' + text + ''; 51 | return html; 52 | } 53 | 54 | /* icon for collapsing/expanding a json object/array */ 55 | var collapseIcon = function(id) { 56 | var attrs = {'onclick': "JSONTree.toggleVisible('collapse_json" + id + "')" }; 57 | return spanNode(collapse_icon, attrs); 58 | } 59 | 60 | /* a json value might be a string, number, boolean, object or an array of other values */ 61 | var jsValue = function(value) { 62 | if (value == null) { 63 | return jsText("null","null"); 64 | } 65 | var type = typeof value; 66 | if (type === 'boolean' || type === 'number') { 67 | return jsText(type,value); 68 | } else if (type === 'string') { 69 | return jsText(type,'"' + escape(value) + '"'); 70 | } else { 71 | var elementId = newId(); 72 | return value instanceof Array ? jsArray(elementId, value) : jsObject(elementId, value); 73 | } 74 | } 75 | 76 | /* json object is made of property names and jsonValues */ 77 | var jsObject = function(id, data) { 78 | var object_content = "{" + collapseIcon(id);; 79 | var object_properties = ''; 80 | Object.keys(data).forEach(function(name, position, names) { 81 | if (position == names.length - 1) { // dont add the comma 82 | object_properties += divNode(jsProperty(name, data[name])); 83 | } else { 84 | object_properties += divNode(jsProperty(name, data[name]) + ','); 85 | } 86 | }); 87 | object_content += divNode(object_properties, {'class': 'json-visible json-object', 'id': "collapse_json" + id}); 88 | return object_content += "}"; 89 | } 90 | 91 | /* a json property, name + value pair */ 92 | var jsProperty = function(name, value) { 93 | return spanNode('"' + escape(name) + '"', {'class': 'json-property'}) + " : " + jsValue(value); 94 | } 95 | 96 | /* array of jsonValues */ 97 | var jsArray = function(id, data) { 98 | var array_content = "[" + collapseIcon(id);; 99 | var values = ''; 100 | for (var i = 0; i < data.length; i++) { 101 | if (i == data.length - 1) { 102 | values += divNode(jsValue(data[i])); 103 | } else { 104 | values += divNode(jsValue(data[i]) + ','); 105 | } 106 | } 107 | array_content += divNode(values, {'class':'json-visible json-object', 'id': 'collapse_json' + id}); 108 | return array_content += "]"; 109 | } 110 | 111 | /* simple value(string, boolean, number...) */ 112 | var jsText = function(type, value) { 113 | return spanNode(value, {'class': "json-" + type}); 114 | } 115 | 116 | var toggleVisible = function(id) { 117 | var element = document.getElementById(id); 118 | var element_class = element.className; 119 | var classes = element_class.split(" "); 120 | var visible = false; 121 | for (var i = 0; i < classes.length; i++) { 122 | if (classes[i] === "json-visible") { 123 | visible = true; 124 | break; 125 | } 126 | } 127 | element.className = visible ? "json-collapsed json-object" : "json-object json-visible"; 128 | element.previousSibling.innerHTML = visible ? expand_icon : collapse_icon; 129 | } 130 | 131 | var configure = function(collapse_icon,expand_icon) { 132 | JSONTree.collapse_icon = collapse_icon; 133 | JSONTree.expand_icon = expand_icon; 134 | } 135 | 136 | var collapse_icon = spanNode('', {'class' : 'json-object-collapse'}); 137 | 138 | var expand_icon = spanNode('', {'class' : 'json-object-expand'}); 139 | 140 | // do an XHR for the JSON, and render it. 141 | var getAndRender = function(jsonUrl, elem) { 142 | var r = new XMLHttpRequest(); 143 | r.open("GET", jsonUrl, true); 144 | 145 | r.onreadystatechange = function() { 146 | if (r.readyState != 4 || r.status != 200) { 147 | return; 148 | } 149 | 150 | // TODO: catch SyntaxError. JSONP might not be valid -- we might want to 151 | // find the first { or [? 152 | var data = JSON.parse(r.responseText); 153 | 154 | // "content" matches the ID we genreated in the shell script 155 | elem.innerHTML = JSONTree.createNodes(data); 156 | }; 157 | 158 | // TODO: handle XHR error 159 | r.send(); 160 | } 161 | 162 | // Public functions 163 | 164 | return { 165 | createNodes: createNodes, 166 | toggleVisible: toggleVisible, 167 | getAndRender: getAndRender 168 | }; 169 | 170 | }(); 171 | 172 | -------------------------------------------------------------------------------- /third_party/json/testdata/tiny.json: -------------------------------------------------------------------------------- 1 | { "a": 1, "b": [1,2,3], 2 | "c": {"d": [4,5,6] } 3 | } 4 | 5 | -------------------------------------------------------------------------------- /third_party/treemap/COPYING: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2010 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /third_party/treemap/README: -------------------------------------------------------------------------------- 1 | Plugin to render a treemap in the browser. 2 | 3 | This code is from 4 | 5 | https://github.com/martine/webtreemap 6 | 7 | (slightly modified) 8 | -------------------------------------------------------------------------------- /third_party/treemap/render: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | readonly THIS_DIR=$(dirname $0) 4 | . $THIS_DIR/../webpipe-lib.sh 5 | 6 | # TODO: build this into the repo 7 | treesum() { 8 | ~/hg/treemap/treesum.py "$@" 9 | } 10 | 11 | old() { 12 | cat >$output/index.html < 14 | 15 | treemap $output 16 | 17 | 18 | 19 | 20 | 21 |
22 | 23 | 24 |

test

25 | 26 | 29 | 30 | 31 | EOF 32 | } 33 | 34 | main() { 35 | local input=$1 36 | local output=$2 37 | 38 | mkdir -p $output 39 | 40 | # Original treemap file, for reference. Not used in the visualization. 41 | cp $input $output/ 42 | 43 | local filename=$(basename $input) 44 | local json_name=${filename}.json 45 | local json_path=$output/$json_name 46 | 47 | # Transform 48 | 49 | echo 'var kTree = ' > $json_path 50 | treesum build <$input >>$json_path 51 | 52 | # NOTE: /plugins/ is 3 level up from a scroll entry, which is in 53 | # /s/2014-03-22/1/index.html 54 | 55 | # Use this html as a template. TODO: It is safe, but we should guarantee it. 56 | sed "s|__NAME__|$filename|g; s|__JSON_NAME__|$json_name|g" \ 57 | <$THIS_DIR/webtreemap.html >$output/index.html 58 | 59 | echo $output 60 | 61 | local html=$output.html 62 | 63 | # TODO: Preview in the snippet. Should give first 10 and 64 | # last 10 lines in here? Should we use Python, or just head/tail? 65 | # more stats 66 | # - depth of tree? 67 | # - histogram of node types? (object: 10, array: 20, etc.) 68 | # - histogram of object keys? 69 | 70 | local numLines="$(wc -l $input | awk '{print $1}')" 71 | 72 | cat >$html < $filename: $numLines lines

74 | 75 | navigate treemap
76 | 77 | original data
78 | 79 | raw JSON
80 | EOF 81 | 82 | echo $html 83 | } 84 | 85 | main "$@" 86 | 87 | -------------------------------------------------------------------------------- /third_party/treemap/static/webtreemap.css: -------------------------------------------------------------------------------- 1 | .webtreemap-node { 2 | /* Required attributes. */ 3 | position: absolute; 4 | overflow: hidden; /* To hide overlong captions. */ 5 | background: white; /* Nodes must be opaque for zIndex layering. */ 6 | border: solid 1px black; /* Calculations assume 1px border. */ 7 | 8 | /* Optional: CSS animation. */ 9 | -webkit-transition: top 0.3s, 10 | left 0.3s, 11 | width 0.3s, 12 | height 0.3s; 13 | 14 | /* PATCH: added -moz-transition. Disabled because Firefox is slow with a lot 15 | of DOM nodes. */ 16 | /* 17 | -moz-transition: top 0.3s, 18 | left 0.3s, 19 | width 0.3s, 20 | height 0.3s; 21 | */ 22 | } 23 | 24 | /* Optional: highlight nodes on mouseover. */ 25 | .webtreemap-node:hover { 26 | /* background: #eee; */ 27 | /* This doesn't work, because the children become gray too. Probably could 28 | * play with zIndex to get this working. 29 | * 30 | * The opposite, starting opacity at 0.2, and then raising to 0.8 here, works 31 | * to some degree. But then you see hidden DOM elements in the background? 32 | * Maybe you need a white block between visible and invisible? 33 | * 34 | * */ 35 | /* opacity: 0.5; */ 36 | } 37 | 38 | /* Optional: Different borders depending on level. */ 39 | .webtreemap-level0 { 40 | border: solid 1px #444; 41 | background-color: #F39DD4; 42 | } 43 | 44 | .webtreemap-level1 { 45 | border: solid 1px #666; 46 | background-color: #F8F087; 47 | } 48 | .webtreemap-level2 { 49 | border: solid 1px #888; 50 | background-color: #B7E3C0; 51 | } 52 | .webtreemap-level3 { 53 | border: solid 1px #aaa; 54 | background-color: #B8D0DD; 55 | } 56 | .webtreemap-level4 { 57 | border: solid 1px #ccc; 58 | background-color: #DBBAE5; 59 | } 60 | 61 | .webtreemap-level5 { 62 | border: solid 1px #ddd; 63 | background-color: lightgray; 64 | } 65 | 66 | .webtreemap-level6 { 67 | border: solid 1px #eee; 68 | background-color: olive; 69 | } 70 | 71 | /* highlight on hover */ 72 | 73 | .webtreemap-level0:hover { 74 | background-color: red; 75 | } 76 | 77 | .webtreemap-level1:hover { 78 | background-color: yellow; 79 | } 80 | 81 | .webtreemap-level2:hover { 82 | background-color: lightgreen; 83 | } 84 | 85 | .webtreemap-level3:hover { 86 | background-color: lightblue; 87 | } 88 | 89 | .webtreemap-level4:hover { 90 | background-color: #A666AC; 91 | } 92 | 93 | .webtreemap-level5:hover { 94 | background-color: lightslategray; 95 | } 96 | 97 | .webtreemap-level6:hover { 98 | background-color: olivedrab; 99 | } 100 | 101 | /* Optional: styling on node captions. */ 102 | .webtreemap-caption { 103 | font-family: sans-serif; 104 | font-size: 11px; 105 | padding: 2px; 106 | text-align: center; 107 | } 108 | 109 | /* Optional: styling on captions on mouse hover. */ 110 | /*.webtreemap-node:hover > .webtreemap-caption { 111 | text-decoration: underline; 112 | }*/ 113 | -------------------------------------------------------------------------------- /third_party/treemap/static/webtreemap.js: -------------------------------------------------------------------------------- 1 | // webtreemap 2 | // 3 | // NOTES: 4 | // - .css files uses -webkit-transition for animation. I added -moz-transition 5 | // too. Not as smooth, but it works OK. 6 | 7 | // Size of border around nodes. 8 | // We could support arbitrary borders using getComputedStyle(), but I am 9 | // skeptical the extra complexity (and performance hit) is worth it. 10 | var kBorderWidth = 1; 11 | 12 | // Padding around contents. 13 | // TODO: do this with a nested div to allow it to be CSS-styleable. 14 | var kPadding = 4; 15 | 16 | // Position a given DOM node with its left corner at (x, y), and resize it to 17 | // (width, height). 18 | function position(dom, x, y, width, height) { 19 | // CSS width/height does not include border. 20 | width -= kBorderWidth*2; 21 | height -= kBorderWidth*2; 22 | 23 | dom.style.left = x + 'px'; 24 | dom.style.top = y + 'px'; 25 | dom.style.width = Math.max(width, 0) + 'px'; 26 | dom.style.height = Math.max(height, 0) + 'px'; 27 | } 28 | 29 | // Given a list of rectangles |nodes|, the 1-d space available 30 | // |space|, and a starting rectangle index |start|, compute an span of 31 | // rectangles that optimizes a pleasant aspect ratio. 32 | // 33 | // Returns [end, sum], where end is one past the last rectangle and sum is the 34 | // 2-d sum of the rectangles' areas. 35 | function selectSpan(nodes, space, start) { 36 | // Add rectangle one by one, stopping when aspect ratios begin to go 37 | // bad. Result is [start,end) covering the best run for this span. 38 | // http://scholar.google.com/scholar?cluster=5972512107845615474 39 | var node = nodes[start]; 40 | var rmin = node.data['$area']; // Smallest seen child so far. 41 | var rmax = rmin; // Largest child. 42 | var rsum = 0; // Sum of children in this span. 43 | var last_score = 0; // Best score yet found. 44 | for (var end = start; node = nodes[end]; ++end) { 45 | var size = node.data['$area']; 46 | if (size < rmin) 47 | rmin = size; 48 | if (size > rmax) 49 | rmax = size; 50 | rsum += size; 51 | 52 | // This formula is from the paper, but you can easily prove to 53 | // yourself it's taking the larger of the x/y aspect ratio or the 54 | // y/x aspect ratio. The additional magic fudge constant of 5 55 | // makes us prefer wider rectangles to taller ones. 56 | var score = Math.max(5*space*space*rmax / (rsum*rsum), 57 | 1*rsum*rsum / (space*space*rmin)); 58 | if (last_score && score > last_score) { 59 | rsum -= size; // Undo size addition from just above. 60 | break; 61 | } 62 | last_score = score; 63 | } 64 | return [end, rsum]; 65 | } 66 | 67 | // 68 | // TreeMap 69 | // 70 | 71 | function TreeMap(dom, data, statusNode, options) { 72 | this.dom = dom; 73 | this.data = data; 74 | this.focused = null; 75 | this.statusNode = statusNode; 76 | 77 | options = options || {}; 78 | // not implemented yet: this is supposed to limit the number of boxes drawn? 79 | // Not sure how hard that is. Need to look at zIndex and so forth. 80 | this.maxLevel = options.maxLevel || 0; // 0 is no max 81 | 82 | // for getting rid of small boxes 83 | this.minWidth = options.minWidth || 60; 84 | this.minHeight = options.minHeight || 40; 85 | 86 | // TODO: this could just be inline. Or should the caller do it? 87 | this.appendTreemap(dom, data); 88 | } 89 | 90 | // Append a treemap to a DOM node, using data from a JSON tree. 91 | TreeMap.prototype.appendTreemap = function(dom, data) { 92 | var style = getComputedStyle(dom, null); 93 | var width = parseInt(style.width); 94 | var height = parseInt(style.height); 95 | // Make the root node and add it do the document. 96 | if (!data.dom) 97 | this.makeDom(data, 0); 98 | dom.appendChild(data.dom); 99 | 100 | // Position the outermost div. 101 | position(data.dom, 0, 0, width, height); 102 | 103 | // Recursively layout the tree. 104 | this.layout(data, 0, width, height); 105 | // Show status initially. TODO: refactor with focus()? 106 | this.showStatus(data); 107 | } 108 | 109 | TreeMap.prototype.showStatus = function(tree) { 110 | var parts = []; 111 | var t = tree; 112 | while (t) { 113 | parts.push(t.name); 114 | t = t.parent; 115 | } 116 | parts.reverse(); 117 | this.statusNode.innerHTML = parts.join(' / '); 118 | } 119 | 120 | // Focus on a given subtree. 121 | // tree: JSON subtree 122 | TreeMap.prototype.focus = function(tree) { 123 | this.focused = tree; 124 | this.showStatus(tree); 125 | 126 | // Hide all visible siblings of all our ancestors by lowering them. 127 | var level = 0; 128 | var root = tree; 129 | while (root.parent) { 130 | root = root.parent; 131 | level += 1; 132 | for (var i = 0, sibling; sibling = root.children[i]; ++i) { 133 | if (sibling.dom) 134 | sibling.dom.style.zIndex = 0; 135 | } 136 | } 137 | var width = root.dom.offsetWidth; 138 | var height = root.dom.offsetHeight; 139 | // Unhide (raise) and maximize us and our ancestors. 140 | for (var t = tree; t.parent; t = t.parent) { 141 | // Shift off by border so we don't get nested borders. 142 | // TODO: actually make nested borders work (need to adjust width/height). 143 | 144 | position(t.dom, -kBorderWidth, -kBorderWidth, width, height); 145 | t.dom.style.zIndex = 1; 146 | } 147 | // And layout into the topmost box. 148 | this.layout(tree, level, width, height); 149 | } 150 | 151 | // Make a div for a box, and a div for its label. Attaches it the box to the 152 | // 'dom' property of the subtree. Each node in the JSON tree is lazily created. 153 | // 154 | // Called from appendTreemap() and layout(). 155 | // 156 | // Args: 157 | // tree: JSON subtree 158 | // level: integer depth, 0.. 159 | // 160 | // Returns: 161 | // DOM node representing the box div. 162 | 163 | TreeMap.prototype.makeDom = function(tree, level) { 164 | var dom = document.createElement('div'); 165 | dom.style.zIndex = 1; 166 | dom.className = 'webtreemap-node webtreemap-level' + Math.min(level, 6); 167 | 168 | // inside the event handler, 'this' is the element, not TreeMap() instance. 169 | var that = this; 170 | 171 | // Register the click. 172 | dom.onmousedown = function(e) { 173 | if (e.button == 0) { // What does this do? 174 | if (that.focused && tree == that.focused && that.focused.parent) { 175 | that.focus(that.focused.parent); // zoom out 176 | } else { 177 | that.focus(tree); // zoom in 178 | } 179 | } 180 | e.stopPropagation(); 181 | return true; 182 | }; 183 | 184 | var caption = document.createElement('div'); 185 | caption.className = 'webtreemap-caption'; 186 | caption.innerHTML = tree.name; 187 | dom.appendChild(caption); 188 | 189 | tree.dom = dom; 190 | return dom; 191 | } 192 | 193 | // Layout a JSON subtree. 194 | // 195 | // It should have 'name', '$area', and then a 'children' array of further 196 | // subtrees. 197 | TreeMap.prototype.layout = function(tree, level, width, height) { 198 | if (!('children' in tree)) 199 | return; 200 | 201 | var total = tree.data['$area']; 202 | 203 | // XXX why do I need an extra -1/-2 here for width/height to look right? 204 | var x1 = 0, y1 = 0, x2 = width - 1, y2 = height - 2; 205 | x1 += kPadding; y1 += kPadding; 206 | x2 -= kPadding; y2 -= kPadding; 207 | y1 += 14; // XXX get first child height for caption spacing 208 | 209 | var pixels_to_units = Math.sqrt(total / ((x2 - x1) * (y2 - y1))); 210 | 211 | for (var start = 0, child; child = tree.children[start]; ++start) { 212 | // AC: I think this is getting rid of small nodes, which speeds things up. 213 | if (x2 - x1 < this.minWidth || y2 - y1 < this.minHeight) { 214 | // AC: push it to the back? We may or may not have a dom node. 215 | if (child.dom) { 216 | child.dom.style.zIndex = 0; 217 | position(child.dom, -2, -2, 0, 0); 218 | } 219 | //console.log('Stopping after small dimensions: ' + (x2-x1) + ' ' + (y2-y1)); 220 | continue; 221 | } 222 | 223 | // In theory we can dynamically decide whether to split in x or y based 224 | // on aspect ratio. In practice, changing split direction with this 225 | // layout doesn't look very good. 226 | // var ysplit = (y2 - y1) > (x2 - x1); 227 | var ysplit = true; 228 | 229 | var space; // Space available along layout axis. 230 | if (ysplit) 231 | space = (y2 - y1) * pixels_to_units; 232 | else 233 | space = (x2 - x1) * pixels_to_units; 234 | 235 | var span = selectSpan(tree.children, space, start); 236 | var end = span[0], rsum = span[1]; 237 | 238 | // Now that we've selected a span, lay out rectangles [start,end) in our 239 | // available space. 240 | var x = x1, y = y1; 241 | for (var i = start; i < end; ++i) { 242 | child = tree.children[i]; 243 | if (!child.dom) { 244 | child.parent = tree; 245 | child.dom = this.makeDom(child, level + 1); 246 | tree.dom.appendChild(child.dom); 247 | } else { 248 | child.dom.style.zIndex = 1; 249 | } 250 | var size = child.data['$area']; 251 | var frac = size / rsum; 252 | if (ysplit) { 253 | width = rsum / space; 254 | height = size / width; 255 | } else { 256 | height = rsum / space; 257 | width = size / height; 258 | } 259 | width /= pixels_to_units; 260 | height /= pixels_to_units; 261 | width = Math.round(width); 262 | height = Math.round(height); 263 | position(child.dom, x, y, width, height); 264 | if ('children' in child) { 265 | this.layout(child, level + 1, width, height); 266 | } 267 | if (ysplit) 268 | y += height; 269 | else 270 | x += width; 271 | } 272 | 273 | // Shrink our available space based on the amount we used. 274 | if (ysplit) 275 | x1 += Math.round((rsum / space) / pixels_to_units); 276 | else 277 | y1 += Math.round((rsum / space) / pixels_to_units); 278 | 279 | // end points one past where we ended, which is where we want to 280 | // begin the next iteration, but subtract one to balance the ++ in 281 | // the loop. 282 | start = end - 1; 283 | } 284 | } 285 | 286 | // zoom to root 287 | TreeMap.prototype.focusRoot = function() { 288 | this.focus(this.data); 289 | } 290 | 291 | // zoom out 292 | TreeMap.prototype.focusUp = function() { 293 | var p = this.focused.parent; 294 | if (p) { 295 | this.focus(p); 296 | } 297 | } 298 | 299 | -------------------------------------------------------------------------------- /third_party/treemap/testdata/tiny.treemap: -------------------------------------------------------------------------------- 1 | 12288 ./.webpipe-test.sh.swp 2 | 8 ./testdata/file.unknown 3 | 861851 ./testdata/CrimeStatebyState.csv 4 | 45914 ./testdata/census_marriage.csv 5 | 66 ./testdata/file with spaces.txt 6 | 560 ./deps.sh 7 | 3038 ./latch.sh 8 | 808 ./run.sh 9 | 6629 ./webpipe/serve.py 10 | 6269 ./webpipe/xrender.pyc 11 | 0 ./webpipe/__init__.py 12 | 3548 ./webpipe/publish.py 13 | 16 ./webpipe/usage-address.txt 14 | 469 ./webpipe/handlers_test.py 15 | 129 ./webpipe/__init__.pyc 16 | 7723 ./webpipe/handlers.py 17 | 8087 ./webpipe/handlers.pyc 18 | 4217 ./webpipe/csv_plugin.pyc 19 | 7879 ./webpipe/serve.pyc 20 | 8324 ./webpipe/xrender.py 21 | 2660 ./webpipe/index.html 22 | 904 ./webpipe/xrender_test.py 23 | 2988 ./webpipe/recv.py 24 | 569 ./webpipe/serve_test.py 25 | 8363 ./latch/latch.pyc 26 | 412 ./latch/latch_test.py 27 | 7477 ./latch/latch.py 28 | 16 ./latch/usage-address.txt 29 | 74 ./latch/templates.py 30 | 1200 ./latch/NOTES 31 | 855 ./latch/latch.js 32 | 223 ./latch/templates.pyc 33 | 3919 ./common/spy.py 34 | 0 ./common/__init__.py 35 | 608 ./common/spy_test.py 36 | 4562 ./common/httpd.py 37 | 128 ./common/__init__.pyc 38 | 3923 ./common/spy.pyc 39 | 4472 ./common/httpd.pyc 40 | 526 ./common/httpd_test.py 41 | 701 ./common/util.py 42 | 1527 ./common/util.pyc 43 | 503 ./doc/wp-help-advanced.txt 44 | 1163 ./doc/html.jsont 45 | 4341 ./doc/webpipe.md 46 | 582 ./doc/wp-help.txt 47 | 3205 ./webpipe-test.sh 48 | 359 ./wp-test.sh 49 | 87 ./.RData 50 | 870 ./README.md 51 | 17 ./.gitignore 52 | 63 ./static/webpipe.css 53 | 331 ./wp-dev.sh 54 | 1608 ./webpipe.R 55 | 2840 ./_tmp/webpipe.tar.gz 56 | 982 ./_tmp/README.md-body.html 57 | 4699 ./_tmp/webpipe.md-body.html 58 | 5191 ./_tmp/webpipe.html 59 | 967 ./_tmp/_gen.sh 60 | 1448 ./_tmp/README.html 61 | 454 ./_tmp/json-tree/json-tree-master/example.html 62 | 2514 ./_tmp/json-tree/json-tree-master/jsontree.min.js 63 | 13035 ./_tmp/json-tree/json-tree-master/imgs/example_1.png 64 | 9663 ./_tmp/json-tree/json-tree-master/imgs/example_2.png 65 | 658 ./_tmp/json-tree/json-tree-master/README.md 66 | 4083 ./_tmp/json-tree/json-tree-master/jsontree.js 67 | 1088 ./_tmp/json-tree/json-tree-master/css/jsontree.css 68 | 27211 ./_tmp/json-tree/master.zip 69 | 98 ./_tmp/Package.stamp 70 | 4 ./_tmp/foo.txt 71 | 1688 ./Auto 72 | 624 ./doc.sh 73 | 984 ./NOTES 74 | 6286 ./wp.sh 75 | 71 ./Makefile 76 | 2983 ./wp-stub.sh 77 | 16 ./.Rhistory 78 | 18 ./plugins/html/testdata/example.html 79 | 155 ./plugins/html/testdata/tiny.html 80 | 336 ./plugins/html/render 81 | 969 ./plugins/webpipe-lib.sh 82 | 50 ./plugins/json/testdata/tiny.json 83 | 1424 ./plugins/json/render 84 | 1052 ./plugins/json/static/jsontree.css 85 | 4540 ./plugins/json/static/jsontree.js 86 | 0 ./plugins/treemap/testdata/tiny.treemap 87 | 1854 ./plugins/treemap/render 88 | 12288 ./plugins/treemap/.render.swp 89 | 2553 ./plugins/treemap/static/webtreemap.css 90 | 9180 ./plugins/treemap/static/webtreemap.js 91 | 1550 ./plugins/treemap/webtreemap.html 92 | 582 ./plugins/png/render 93 | 814 ./plugins/ansi/testdata/typescript 94 | 312 ./plugins/ansi/render 95 | 105 ./plugins/ansi/README.md 96 | 96 ./plugins/markdown/testdata/tiny.markdown 97 | 362 ./plugins/markdown/render 98 | 31 ./plugins/csv/testdata/tiny.csv 99 | 102 ./plugins/csv/render 100 | 5107 ./plugins/csv/render.py 101 | 4 ./plugins/csv/static/bar.txt 102 | 386 ./plugins/csv/render-dev.sh 103 | 760 ./plugins/dot/render 104 | 470 ./plugins/dot/examples/cluster.dot 105 | 79 ./plugins/txt/testdata/tiny.txt 106 | 42 ./plugins/txt/testdata/example.txt 107 | 333 ./plugins/txt/render 108 | 401 ./plugins/webpipe-lib-test.sh 109 | 970 ./plugins/R/render 110 | 178 ./plugins/zip/testdata/tiny.zip 111 | 388 ./plugins/zip/render 112 | -------------------------------------------------------------------------------- /third_party/treemap/webtreemap.html: -------------------------------------------------------------------------------- 1 | 2 | Treemap: __NAME__ 3 | 4 | 5 | 36 | 37 | 38 |

39 | Treemaps 40 |

41 | 42 |

Treemap: __NAME__

43 | 44 |

The size of a box is proportional to the size of the node it represents. 45 | Click on a box to zoom in.

46 | 47 |

48 | 49 | 50 |

51 | 52 | 53 |

54 | 55 |
56 | 57 | 58 | 59 | 60 | 81 | -------------------------------------------------------------------------------- /webpipe-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | # 8 | # Usage: 9 | # ./webpipe-test.sh 10 | 11 | set -o nounset 12 | 13 | # TODO: make this a proper dep 14 | . ~/hg/taste/taste.sh 15 | 16 | # 17 | # Components 18 | # 19 | 20 | # client 21 | wp() { 22 | ./wp-dev.sh "$@" 23 | } 24 | 25 | xrender() { 26 | ./wp-dev.sh xrender "$@" 27 | } 28 | 29 | serve() { 30 | ./wp-dev.sh serve "$@" 31 | } 32 | 33 | # 34 | # Demo files 35 | # 36 | 37 | # http://hci.stanford.edu/jheer/workshop/data/ 38 | big-csv() { 39 | ln --verbose --force -s $PWD/testdata/census_marriage.csv ~/webpipe/input 40 | } 41 | 42 | # TODO: need a client to generate this data file. 43 | # 44 | # fs-treemap 45 | treemap-testdata() { 46 | ~/hg/treemap/run.sh find-with-format-string '%s %p\n' . \ 47 | | tee plugins/treemap/testdata/tiny.treemap 48 | } 49 | 50 | zip-testdata() { 51 | echo foo > _tmp/foo.txt 52 | zip plugins/zip/testdata/tiny.zip _tmp/foo.txt 53 | } 54 | 55 | treemap-plugin() { 56 | local dest=${1:-~/webpipe/input} 57 | plugins/treemap/render $dest/demo.treemap 3 58 | } 59 | 60 | write-tiny() { 61 | for p in html txt markdown csv json treemap zip; do 62 | echo $p 63 | wp show plugins/$p/testdata/tiny.$p 64 | sleep 0.5 65 | done 66 | } 67 | 68 | write-demo() { 69 | local dest=~/webpipe/input 70 | set -x 71 | 72 | write-tiny 73 | 74 | # TODO: generate png testdata 75 | touch $dest/Rplot001.png 76 | sleep 1 77 | 78 | # TODO: typescript should be its own file type. People might want to use a 79 | # different plugin. Need aliases. 80 | wp show plugins/typescript/testdata/typescript 81 | sleep 1 82 | 83 | wp show plugins/dot/examples/cluster.dot 84 | sleep 1 85 | 86 | wp show testdata/file.unknown 87 | sleep 1 88 | } 89 | 90 | # 91 | # Tests 92 | # 93 | 94 | test-xrender-badport() { 95 | xrender -p invalid 96 | check $? -eq 1 97 | } 98 | 99 | test-xrender() { 100 | xrender ~/webpipe/input ~/webpipe/s/webpipe-test <one' > $session/1.html 120 | { echo '2:{}'; echo 1.html; } | $dev serve serve $session 121 | } 122 | 123 | # not fatal 124 | test-recv-bad-fields() { 125 | echo -n '0:}8:1:a,1:b,}' | ./wp-dev.sh recv ~/webpipe/input 126 | echo $? 127 | } 128 | 129 | # fatal, because the stream could be messed up 130 | test-recv-bad-message() { 131 | echo -n 'abc' | ./wp-dev.sh recv ~/webpipe/input 132 | echo $? 133 | } 134 | 135 | test-recv-empty() { 136 | echo -n '' | ./wp-dev.sh recv ~/webpipe/input 137 | echo $? 138 | } 139 | 140 | # TODO: 'wp-stub send' tests are broken. Semantics are unclear. 141 | 142 | test-send() { 143 | ( echo wp-stub.sh; 144 | echo nonexistent ) \ 145 | | ./wp-stub.sh send 146 | } 147 | 148 | # Since the stub can be copied to many machines, test that it an run with a 149 | # non-bash shell. 150 | test-stub-with-busybox() { 151 | ( echo wp-stub.sh; 152 | echo nonexistent ) \ 153 | | busybox sh wp-stub.sh send 154 | } 155 | 156 | test-send-recv() { 157 | local out=~/webpipe/input/wp-stub.sh 158 | rm $out 159 | echo wp-stub.sh \ 160 | | ./wp-stub.sh send \ 161 | | ./wp-dev.sh recv ~/webpipe/input 162 | ls -al $out 163 | diff wp-stub.sh $out 164 | echo $? 165 | } 166 | 167 | # 168 | # Publishing 169 | # 170 | 171 | publish-demo() { 172 | ./wp-dev.sh publish ~/webpipe/s/2014-03-23/5 dreamhost 173 | } 174 | 175 | "$@" 176 | -------------------------------------------------------------------------------- /webpipe.R: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Google Inc. All rights reserved. 2 | # Use of this source code is governed by a BSD-style license that can be found 3 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 4 | 5 | # Simple R functions to use web pipe. 6 | # 7 | # Examples: 8 | # 9 | # > web.plot(1:10) # alternative to plot(1:10) 10 | # > web.hist(rnorm(10)) # hist(1:10) 11 | 12 | # Underlying function is web.png. 13 | # 14 | # > web.png(hist, rnorm(10)) # alternative to hist(rnorm(10)) 15 | # > web.plot(hist(rnorm(10), plot=F)) # another alternative 16 | # 17 | # The webpipe.png.args option is used to determine additional arguments to the 18 | # png device. 19 | # 20 | # Example: 21 | # options(webpipe.png.args=list(width=800, height=600)) 22 | 23 | # TODO: 24 | # - Can the user configure the input dir? Environment var? 25 | 26 | # NOTE: png() accepts ~/webpipe, but CairoPNG doesn't. (It doesn't expand ~). 27 | web.plot.dest = path.expand('~/webpipe/input') # expand ~ 28 | 29 | web.plot.num = 0 30 | 31 | # Wrap a function that writes to a graphic device so that it writes a .png to 32 | # the webpipe input directory. 33 | 34 | web.png = function(func, ...) { 35 | # NOTE: giving it the "dual extension" .Rplot.png, so we can possibly do 36 | # different things with it, vs. a regular png. 37 | plot.path = file.path(web.plot.dest, sprintf('%03d.Rplot.png', web.plot.num)) 38 | 39 | # Let the user pass their own dimensions, etc. filename can't be overridden. 40 | png.args = getOption("webpipe.png.args", list()) 41 | png.args$file = plot.path 42 | 43 | # NOTE: in an interactive session, it's possible to do png <- CairoPNG to 44 | # avoid "x11 is not available" errors. 45 | do.call(png, png.args) 46 | func(...) 47 | dev.off() 48 | 49 | # Debug: assert that the file was created. 50 | #if (!file.exists(plot.path)) { 51 | # print(paste0('ERROR ', plot.path)) 52 | #} 53 | 54 | cat(sprintf('%s\n', plot.path)) 55 | 56 | web.plot.num <<- web.plot.num + 1 # increment global 57 | 58 | # Notify webpipe that there is a plot. 59 | 60 | # NOTE: no shQuote necessary because we construct the filename. 61 | cmd = sprintf('echo %s | nc localhost 8988', plot.path) 62 | exit.code = system(cmd) 63 | if (exit.code != 0) { 64 | cat(sprintf("Command '%s' failed.\n", cmd)) 65 | # Possibly give the hint to install nc 66 | code2 = system('nc -h') 67 | if (code2 != 0) { 68 | cat("Couldn't execute 'nc' (netcat). Is it installed?\n") 69 | } 70 | } 71 | 72 | invisible() # no return value printed in REPL 73 | } 74 | 75 | # Like R plot(), but writes to the webpipe input directory. 76 | 77 | web.plot = function(...) { 78 | web.png(plot, ...) 79 | } 80 | 81 | # Like R hist(). 82 | 83 | web.hist = function(...) { 84 | web.plot(hist(..., plot=F)) 85 | } 86 | -------------------------------------------------------------------------------- /webpipe/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/andychu/webpipe/78fe629829af0aa3df1854501d29df68c8c010df/webpipe/__init__.py -------------------------------------------------------------------------------- /webpipe/handlers.py: -------------------------------------------------------------------------------- 1 | """ 2 | wait_server.py 3 | 4 | Server that does a hanging get. Doesn't need to be a WSGI app I think. 5 | 6 | Override SimpelHttpServer. 7 | 8 | Only things like / everywhere -- reuse templates 38 | # - doctype, charset, etc. 39 | 40 | HOME_PAGE = jsontemplate.Template("""\ 41 | 42 | 43 | webpipe home 44 | 45 | 46 | 47 |

48 | plugins 49 | - home 50 |

51 | 52 |

webpipe

53 | 54 |
55 | 56 |

57 | Active Scroll: {active_scroll} 58 |

59 | 60 |

Old Scrolls

61 | 62 | {.repeated section scrolls} 63 | {@}
64 | {.end} 65 | 66 |

67 | (browse) 68 |

69 |
70 | 71 | 72 | 73 | """, default_formatter='html') 74 | 75 | 76 | # TODO: put file system paths here? So people can easily find their plugins. 77 | PLUGINS_PAGE = jsontemplate.Template("""\ 78 |

79 | home 80 |

81 | 82 |

webpipe Plugins

83 | 84 |

User plugins installed in ~/webpipe

85 | 86 |

User plugins override packaged plugins.

87 | 88 |
    89 | {.repeated section user} 90 |
  • {name} {.if test static} static {.end}
    • 91 | {.end} 92 |
93 | 94 |

Packaged Plugins

95 | 96 |
    97 | {.repeated section package} 98 |
  • {name} {.if test static} static {.end}
    • 99 | {.end} 100 |
101 | """, default_formatter='html') 102 | 103 | 104 | # /s///.html 105 | PATH_RE = re.compile(r'/s/(\S+)/(\d+).html$') 106 | 107 | 108 | def _ListPlugins(root_dir): 109 | """ 110 | Returns a template data dictionary. Plugins are directories. Plugins with 111 | 'static' dirs are marked. 112 | """ 113 | plugin_root = os.path.join(root_dir, 'plugins') 114 | data = [] 115 | for name in os.listdir(plugin_root): 116 | path = os.path.join(plugin_root, name) 117 | if not os.path.isdir(path): 118 | continue 119 | p = os.path.join(path, 'static') 120 | s = os.path.isdir(p) 121 | # e.g. {name: csv, static: True} 122 | data.append({'name': name, 'static': s}) 123 | data.sort(key=lambda d: d['name']) 124 | return data 125 | 126 | 127 | class WaitingRequestHandler(httpd.BaseRequestHandler): 128 | """ 129 | differences: 130 | - block on certain paths 131 | - don't always serve in the current directory, let the user do it. 132 | - daemon threads so we don't block the process 133 | - what about cache headers? I think I saw a bug where the browser would 134 | cache instead of waiting. 135 | """ 136 | server_version = "webpipe" 137 | root_dir = None # from BaseRequestHandler, not used 138 | user_dir = None # initialize to ~/webpipe 139 | package_dir = None # initialize to //webpipe 140 | waiters = None 141 | active_scroll = None 142 | 143 | def send_webpipe_index(self): 144 | self.send_response(200) 145 | self.send_header('Content-Type', 'text/html') 146 | self.end_headers() 147 | 148 | s_root = os.path.join(self.user_dir, 's') 149 | scrolls = os.listdir(s_root) 150 | scrolls.sort(reverse=True) 151 | 152 | # Show the active one separately 153 | try: 154 | scrolls.remove(self.active_scroll) 155 | except ValueError: 156 | pass 157 | 158 | h = HOME_PAGE.expand({'scrolls': scrolls, 'active_scroll': self.active_scroll}) 159 | self.wfile.write(h) 160 | 161 | def send_plugins_index(self): 162 | self.send_response(200) 163 | self.send_header('Content-Type', 'text/html') 164 | self.end_headers() 165 | 166 | # Session are saved on disk; allow the user to choose one. 167 | 168 | u = _ListPlugins(self.user_dir) 169 | p = _ListPlugins(self.package_dir) 170 | 171 | html = PLUGINS_PAGE.expand({'user': u, 'package': p}) 172 | self.wfile.write(html) 173 | 174 | def url_to_fs_path(self, url): 175 | """Translate a URL to a local file system path. 176 | 177 | By default, we just treat URLs as paths relative to self.user_dir. 178 | 179 | If it returns None, then a 404 is generated, without looking at disk. 180 | 181 | Called from send_head() (see SimpleHTTPServer). 182 | 183 | NOTE: This is adapted from Python stdlib SimpleHTTPServer.py. 184 | """ 185 | # Disallow path traversal with '..' 186 | parts = [p for p in url.split('/') if p and p not in ('.', '..')] 187 | if not parts: # corresponds to /, which is already handled by send_webpipe_index 188 | return None 189 | first_part = parts[0] 190 | rest = parts[1:] 191 | 192 | if first_part == 'static': 193 | return os.path.join(self.package_dir, *parts) 194 | 195 | if first_part == 's': 196 | return os.path.join(self.user_dir, *parts) 197 | 198 | if first_part == 'plugins': 199 | # looking for ['plugins', , 'static']. 200 | # Note these can be files OR directories. Directories will be listed. 201 | if len(parts) >= 3 and parts[2] == 'static': 202 | packaged_res = os.path.join(self.package_dir, *parts) 203 | user_res = os.path.join(self.user_dir, *parts) 204 | 205 | # Return the one that exists, starting with the user dir. 206 | if os.path.exists(user_res): 207 | return user_res 208 | if os.path.exists(packaged_res): 209 | return packaged_res 210 | 211 | return None 212 | 213 | def do_GET(self): 214 | """Serve a GET request.""" 215 | 216 | if self.path == '/': 217 | self.send_webpipe_index() 218 | return 219 | 220 | if self.path == '/plugins': 221 | # As is done in send_head 222 | self.send_response(301) 223 | self.send_header("Location", self.path + "/") 224 | self.end_headers() 225 | return 226 | 227 | if self.path == '/plugins/': 228 | self.send_plugins_index() 229 | return 230 | 231 | m = PATH_RE.match(self.path) 232 | if m: 233 | session, num = m.groups() 234 | num = int(num) 235 | 236 | waiter = self.waiters.get(session) 237 | if waiter is not None: 238 | log('PATH: %s', self.path) 239 | 240 | log('MaybeWait session %r, part %d', session, num) 241 | result = waiter.MaybeWait(num) 242 | log('Done %d', num) 243 | # result could be: 244 | # 404: too big 245 | # 503: 503 246 | 247 | # Serve static file. 248 | 249 | # NOTE: url_to_fs_path is called in send_head. 250 | f = self.send_head() 251 | 252 | # f is None if the file doesn't exist, and send_error(404) was called. 253 | if f: 254 | self.copyfile(f, self.wfile) 255 | f.close() 256 | 257 | 258 | 259 | WAIT_OK, WAIT_TOO_BIG, WAIT_TOO_BUSY = range(3) 260 | 261 | class SequenceWaiter(object): 262 | """ 263 | Call Notify() for every item. Then you can call MaybeWait(n) for. 264 | """ 265 | def __init__(self, max_waiters=None): 266 | # If this limit it 267 | self.max_waiters = max_waiters 268 | 269 | # even, odd scheme. When one event is notified, the other is reset. 270 | self.events = [threading.Event(), threading.Event()] 271 | self.lock = threading.Lock() # protects self.events 272 | self.counter = 1 273 | 274 | def SetCounter(self, n): 275 | # TODO: Make this a constructor param? 276 | assert self.counter == 1, "Only call before using" 277 | self.counter = n 278 | 279 | def MaybeWait(self, n): 280 | """ 281 | Returns: 282 | success. 283 | 200 it's OK to proceed (we may have waited) 284 | 404: index is too big? 285 | 503: maximum waiters exceeded. 286 | """ 287 | i = self.counter 288 | 289 | # Block for the next item 290 | if i > n: 291 | #print '%d / %d' % (i, n) 292 | #print self.items 293 | return WAIT_OK 294 | elif i == n: 295 | log('Waiting for event %d (%d)', i, i % 2) 296 | self.events[i % 2].wait() # wait for it to be added 297 | return WAIT_OK 298 | else: 299 | return WAIT_TOO_BIG 300 | 301 | def Notify(self): 302 | # *Atomically* increment counter and add event event N+1. 303 | with self.lock: 304 | n = self.counter 305 | self.counter += 1 306 | 307 | # instantiate a new event in the other space 308 | self.events[self.counter % 2] = threading.Event() 309 | 310 | # unblock all MaybeWait() calls 311 | self.events[n % 2].set() 312 | 313 | def Length(self): 314 | return self.counter 315 | 316 | -------------------------------------------------------------------------------- /webpipe/handlers_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python -S 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | """ 8 | handlers_test.py: Tests for handlers.py 9 | """ 10 | 11 | import unittest 12 | 13 | import handlers # module under test 14 | 15 | 16 | class WaitTest(unittest.TestCase): 17 | 18 | def testSequenceWaiter(self): 19 | s = handlers.SequenceWaiter() 20 | result = s.MaybeWait(2) 21 | self.assertEqual(handlers.WAIT_TOO_BIG, result) 22 | 23 | def testListPlugins(self): 24 | # This takes the place of the package dir. 25 | print handlers._ListPlugins('.') 26 | 27 | 28 | if __name__ == '__main__': 29 | unittest.main() 30 | -------------------------------------------------------------------------------- /webpipe/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | webpipe session 5 | 6 | 9 | 10 | 68 | 69 | 74 | 75 | 76 | 77 |

78 | home 79 |

80 | 81 |

webpipe session

82 | 83 |
84 |
85 | 86 | 87 |

Waiting for part...

88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /webpipe/publish.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # 3 | # Copyright 2014 Google Inc. All rights reserved. 4 | # Use of this source code is governed by a BSD-style license that can be found 5 | # in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd 6 | 7 | """ 8 | publish.py 9 | 10 | Implement publishing plugins. 11 | """ 12 | 13 | import os 14 | import re 15 | import subprocess 16 | import sys 17 | 18 | from common import util 19 | 20 | class Error(Exception): 21 | pass 22 | 23 | 24 | log = util.Logger(util.ANSI_BLUE) 25 | 26 | 27 | # ../../../plugins/ because a scroll is 3 levels up from the root. 28 | DEP_RE = re.compile(r'\.\./\.\./\.\./(plugins/\S+/static)/') 29 | 30 | def ScanForStaticDeps(root_dir, rel_path): 31 | # TODO: Also scan the 1.html entry. 32 | 33 | d = os.path.join(root_dir, rel_path) 34 | 35 | # Some plugins don't even create this dir. 36 | if not os.path.exists(d): 37 | return [] 38 | 39 | all_deps = [] 40 | 41 | for name in os.listdir(d): 42 | path = os.path.join(d, name) 43 | if not path.endswith('.html'): 44 | continue 45 | log('Scanning %s for dependencies', path) 46 | 47 | # Search in the first 10 KB, since