├── .gitignore ├── LICENSE ├── README.md ├── data_analysis ├── Gruntfile.js ├── __init__.py ├── images │ ├── bg.gif │ └── style.css ├── models.py ├── package.json ├── run.py ├── static │ ├── images │ │ └── AdMaster_Logo.png │ ├── javascript │ │ ├── CodeFlower.js │ │ ├── amcharts.js │ │ ├── d3.geom.js │ │ ├── d3.js │ │ ├── d3.layout.js │ │ ├── data.json │ │ ├── dataConverter.js │ │ ├── detector.js │ │ ├── glimpse.js │ │ ├── glimpse.toastr.js │ │ ├── jquery-1.10.2.min.js │ │ ├── jquery-2.0.3.min.js │ │ ├── main.js │ │ ├── pie.js │ │ ├── rainbow-min.js │ │ ├── serial.js │ │ └── toastr.js │ └── stylesheet │ │ ├── blue.css │ │ ├── pure-min.css │ │ └── toastr.css ├── templates │ └── posts │ │ ├── d3.html │ │ ├── index.html │ │ └── trouble.html └── views.py ├── dump └── fetch_data │ ├── celery.bson │ ├── celery.metadata.json │ ├── mongo.bson │ ├── mongo.metadata.json │ └── system.indexes.bson └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib 19 | lib64 20 | __pycache__ 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage 27 | .tox 28 | nosetests.xml 29 | 30 | # Translations 31 | *.mo 32 | 33 | # Mr Developer 34 | .mr.developer.cfg 35 | .project 36 | .pydevproject 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | data-analysis 2 | ============= 3 | 4 | A data analysis site by flask and mongoengine 5 | 6 | ![6](https://user-images.githubusercontent.com/841395/31398950-4516db04-adb0-11e7-8735-81e313992fab.png) 7 | ![7](https://user-images.githubusercontent.com/841395/31398952-457c3ee0-adb0-11e7-9557-ea1f2eddcec0.png) 8 | ![8](https://user-images.githubusercontent.com/841395/31398954-45b66660-adb0-11e7-8feb-3322d701c390.png) 9 | ![9](https://user-images.githubusercontent.com/841395/31398955-45f7b2c8-adb0-11e7-9e00-165de67a6a1f.png) 10 | ![10](https://user-images.githubusercontent.com/841395/31398957-463a0506-adb0-11e7-82ee-60227efea4e1.png) 11 | ![11](https://user-images.githubusercontent.com/841395/31398959-46700c78-adb0-11e7-9424-4c229809b5b1.png) 12 | ![12](https://user-images.githubusercontent.com/841395/31398961-46ada7fe-adb0-11e7-8c8c-210edbed8c2a.png) 13 | 14 | 15 | #### USEAGE 16 | 17 | ``` 18 | $pip install -r requirements.txt 19 | $cd /path/to/data-analysis 20 | $npm install 21 | $mongorestore -d fetch_data --directoryperdb dump/fetch_data 22 | $python run.py 23 | ``` 24 | -------------------------------------------------------------------------------- /data_analysis/Gruntfile.js: -------------------------------------------------------------------------------- 1 | module.exports = function (grunt) { 2 | 3 | grunt.initConfig({ 4 | pkg: grunt.file.readJSON('package.json'), 5 | watch: { 6 | scripts: { 7 | //files: ['static/javascript/*.js'], 8 | files: ['templates/posts/index.html', 9 | 'static/javascript/main.js'], 10 | options: { 11 | livereload: true, 12 | }, 13 | } 14 | } 15 | }); 16 | 17 | grunt.loadNpmTasks('grunt-contrib-watch'); 18 | 19 | grunt.registerTask('default', ['watch']); 20 | 21 | }; 22 | -------------------------------------------------------------------------------- /data_analysis/__init__.py: -------------------------------------------------------------------------------- 1 | from flask import Flask 2 | from flask.ext.mongoengine import MongoEngine 3 | 4 | app = Flask(__name__) 5 | app.config["MONGODB_SETTINGS"] = {'DB': "fetch_data"} 6 | app.config["SECRET_KEY"] = "Keepssdf12q" 7 | 8 | db = MongoEngine(app) 9 | -------------------------------------------------------------------------------- /data_analysis/images/bg.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dongweiming/data-analysis/49005c06590c8488f0dc7ba0992acadfb6407aec/data_analysis/images/bg.gif -------------------------------------------------------------------------------- /data_analysis/images/style.css: -------------------------------------------------------------------------------- 1 | body { 2 | background-color: #D9D9D9; 3 | margin-left: 20px; 4 | margin-top: 20px; 5 | margin-right: 20px; 6 | margin-bottom: 20px; 7 | background-image: url(../images/bg.gif); 8 | font-family: Arial; 9 | font-size: 12px; 10 | line-height:16px; 11 | } 12 | 13 | hr{ 14 | border: 1px solid #FFFFFF; 15 | } 16 | 17 | a:link{ 18 | color:#0055CC; 19 | } 20 | a:visited{ 21 | color:#990099; 22 | } 23 | a:hover{ 24 | color:#CC0000; 25 | } 26 | 27 | th{ 28 | background-color:#FFFFFF; 29 | font-weight:bold; 30 | } 31 | 32 | td{ 33 | background-color:#ecf7fe; 34 | text-align:center; 35 | } -------------------------------------------------------------------------------- /data_analysis/models.py: -------------------------------------------------------------------------------- 1 | from data_analysis import db 2 | 3 | class Apidist(db.Document): 4 | name = db.StringField(max_length=255, required=True) 5 | call = db.IntField(required=True) 6 | include = db.StringField(max_length=255, required=True) 7 | 8 | class Celery(db.Document): 9 | cost = db.FloatField(required=True) 10 | time = db.DateTimeField(required=True) 11 | file = db.StringField(max_length=25, required=True) 12 | task = db.StringField(max_length=255, required=True) 13 | 14 | class Mongo(db.Document): 15 | total = db.IntField(required=True) 16 | database = db.StringField(max_length=255, required=True) 17 | hour = db.DictField(required=True) 18 | -------------------------------------------------------------------------------- /data_analysis/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tencent_data", 3 | "version": "0.1.0", 4 | "devDependencies": { 5 | "grunt": "*", 6 | "grunt-contrib-watch": "*" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /data_analysis/run.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | sys.path.append(os.path.abspath( 5 | os.path.join(os.path.dirname(__file__), '..'))) 6 | 7 | from data_analysis.views import posts 8 | from data_analysis import app 9 | app.register_blueprint(posts) 10 | 11 | if __name__ == "__main__": 12 | app.run(host='0.0.0.0', debug=True) 13 | -------------------------------------------------------------------------------- /data_analysis/static/images/AdMaster_Logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dongweiming/data-analysis/49005c06590c8488f0dc7ba0992acadfb6407aec/data_analysis/static/images/AdMaster_Logo.png -------------------------------------------------------------------------------- /data_analysis/static/javascript/CodeFlower.js: -------------------------------------------------------------------------------- 1 | var CodeFlower = function(selector, w, h) { 2 | this.w = w; 3 | this.h = h; 4 | 5 | d3.select(selector).selectAll("svg").remove(); 6 | 7 | this.svg = d3.select(selector).append("svg:svg") 8 | .attr('width', w) 9 | .attr('height', h); 10 | 11 | this.svg.append("svg:rect") 12 | .style("stroke", "#999") 13 | .style("fill", "#fff") 14 | .attr('width', w) 15 | .attr('height', h); 16 | 17 | this.force = d3.layout.force() 18 | .on("tick", this.tick.bind(this)) 19 | .charge(function(d) { return d._children ? -d.size / 100 : -40; }) 20 | .linkDistance(function(d) { return d.target._children ? 80 : 25; }) 21 | .size([h, w]); 22 | }; 23 | 24 | CodeFlower.prototype.update = function(json) { 25 | if (json) this.json = json; 26 | 27 | this.json.fixed = true; 28 | this.json.x = this.w / 2; 29 | this.json.y = this.h / 2; 30 | 31 | var nodes = this.flatten(this.json); 32 | var links = d3.layout.tree().links(nodes); 33 | var total = nodes.length || 1; 34 | 35 | // remove existing text (will readd it afterwards to be sure it's on top) 36 | this.svg.selectAll("text").remove(); 37 | 38 | // Restart the force layout 39 | this.force 40 | .gravity(Math.atan(total / 50) / Math.PI * 0.4) 41 | .nodes(nodes) 42 | .links(links) 43 | .start(); 44 | 45 | // Update the links 46 | this.link = this.svg.selectAll("line.link") 47 | .data(links, function(d) { return d.target.name; }); 48 | 49 | // Enter any new links 50 | this.link.enter().insert("svg:line", ".node") 51 | .attr("class", "link") 52 | .attr("x1", function(d) { return d.source.x; }) 53 | .attr("y1", function(d) { return d.source.y; }) 54 | .attr("x2", function(d) { return d.target.x; }) 55 | .attr("y2", function(d) { return d.target.y; }); 56 | 57 | // Exit any old links. 58 | this.link.exit().remove(); 59 | 60 | // Update the nodes 61 | this.node = this.svg.selectAll("circle.node") 62 | .data(nodes, function(d) { return d.name; }) 63 | .classed("collapsed", function(d) { return d._children ? 1 : 0; }); 64 | 65 | this.node.transition() 66 | .attr("r", function(d) { return d.children ? 3.5 : Math.pow(d.size, 2/5) || 1; }); 67 | 68 | // Enter any new nodes 69 | this.node.enter().append('svg:circle') 70 | .attr("class", "node") 71 | .classed('directory', function(d) { return (d._children || d.children) ? 1 : 0; }) 72 | .attr("r", function(d) { return d.children ? 3.5 : Math.pow(d.size, 2/5) || 1; }) 73 | .style("fill", function color(d) { 74 | return "hsl(" + parseInt(360 / total * d.id, 10) + ",90%,70%)"; 75 | }) 76 | .call(this.force.drag) 77 | .on("click", this.click.bind(this)) 78 | .on("mouseover", this.mouseover.bind(this)) 79 | .on("mouseout", this.mouseout.bind(this)); 80 | 81 | // Exit any old nodes 82 | this.node.exit().remove(); 83 | 84 | this.text = this.svg.append('svg:text') 85 | .attr('class', 'nodetext') 86 | .attr('dy', 0) 87 | .attr('dx', 0) 88 | .attr('text-anchor', 'middle'); 89 | 90 | return this; 91 | }; 92 | 93 | CodeFlower.prototype.flatten = function(root) { 94 | var nodes = [], i = 0; 95 | 96 | function recurse(node) { 97 | if (node.children) { 98 | node.size = node.children.reduce(function(p, v) { 99 | return p + recurse(v); 100 | }, 0); 101 | } 102 | if (!node.id) node.id = ++i; 103 | nodes.push(node); 104 | return node.size; 105 | } 106 | 107 | root.size = recurse(root); 108 | return nodes; 109 | }; 110 | 111 | CodeFlower.prototype.click = function(d) { 112 | // Toggle children on click. 113 | if (d.children) { 114 | d._children = d.children; 115 | d.children = null; 116 | } else { 117 | d.children = d._children; 118 | d._children = null; 119 | } 120 | this.update(); 121 | }; 122 | 123 | CodeFlower.prototype.mouseover = function(d) { 124 | this.text.attr('transform', 'translate(' + d.x + ',' + (d.y - 5 - (d.children ? 3.5 : Math.sqrt(d.size) / 2)) + ')') 125 | .text(d.name + ": " + d.size + " loc") 126 | .style('display', null); 127 | }; 128 | 129 | CodeFlower.prototype.mouseout = function(d) { 130 | this.text.style('display', 'none'); 131 | }; 132 | 133 | CodeFlower.prototype.tick = function() { 134 | var h = this.h; 135 | var w = this.w; 136 | this.link.attr("x1", function(d) { return d.source.x; }) 137 | .attr("y1", function(d) { return d.source.y; }) 138 | .attr("x2", function(d) { return d.target.x; }) 139 | .attr("y2", function(d) { return d.target.y; }); 140 | 141 | this.node.attr("transform", function(d) { 142 | return "translate(" + Math.max(5, Math.min(w - 5, d.x)) + "," + Math.max(5, Math.min(h - 5, d.y)) + ")"; 143 | }); 144 | }; 145 | 146 | CodeFlower.prototype.cleanup = function() { 147 | this.update([]); 148 | this.force.stop(); 149 | }; -------------------------------------------------------------------------------- /data_analysis/static/javascript/d3.geom.js: -------------------------------------------------------------------------------- 1 | (function(){d3.geom = {}; 2 | /** 3 | * Computes a contour for a given input grid function using the marching 5 | * squares algorithm. Returns the contour polygon as an array of points. 6 | * 7 | * @param grid a two-input function(x, y) that returns true for values 8 | * inside the contour and false for values outside the contour. 9 | * @param start an optional starting point [x, y] on the grid. 10 | * @returns polygon [[x1, y1], [x2, y2], …] 11 | */ 12 | d3.geom.contour = function(grid, start) { 13 | var s = start || d3_geom_contourStart(grid), // starting point 14 | c = [], // contour polygon 15 | x = s[0], // current x position 16 | y = s[1], // current y position 17 | dx = 0, // next x direction 18 | dy = 0, // next y direction 19 | pdx = NaN, // previous x direction 20 | pdy = NaN, // previous y direction 21 | i = 0; 22 | 23 | do { 24 | // determine marching squares index 25 | i = 0; 26 | if (grid(x-1, y-1)) i += 1; 27 | if (grid(x, y-1)) i += 2; 28 | if (grid(x-1, y )) i += 4; 29 | if (grid(x, y )) i += 8; 30 | 31 | // determine next direction 32 | if (i === 6) { 33 | dx = pdy === -1 ? -1 : 1; 34 | dy = 0; 35 | } else if (i === 9) { 36 | dx = 0; 37 | dy = pdx === 1 ? -1 : 1; 38 | } else { 39 | dx = d3_geom_contourDx[i]; 40 | dy = d3_geom_contourDy[i]; 41 | } 42 | 43 | // update contour polygon 44 | if (dx != pdx && dy != pdy) { 45 | c.push([x, y]); 46 | pdx = dx; 47 | pdy = dy; 48 | } 49 | 50 | x += dx; 51 | y += dy; 52 | } while (s[0] != x || s[1] != y); 53 | 54 | return c; 55 | }; 56 | 57 | // lookup tables for marching directions 58 | var d3_geom_contourDx = [1, 0, 1, 1,-1, 0,-1, 1,0, 0,0,0,-1, 0,-1,NaN], 59 | d3_geom_contourDy = [0,-1, 0, 0, 0,-1, 0, 0,1,-1,1,1, 0,-1, 0,NaN]; 60 | 61 | function d3_geom_contourStart(grid) { 62 | var x = 0, 63 | y = 0; 64 | 65 | // search for a starting point; begin at origin 66 | // and proceed along outward-expanding diagonals 67 | while (true) { 68 | if (grid(x,y)) { 69 | return [x,y]; 70 | } 71 | if (x === 0) { 72 | x = y + 1; 73 | y = 0; 74 | } else { 75 | x = x - 1; 76 | y = y + 1; 77 | } 78 | } 79 | } 80 | /** 81 | * Computes the 2D convex hull of a set of points using Graham's scanning 82 | * algorithm. The algorithm has been implemented as described in Cormen, 83 | * Leiserson, and Rivest's Introduction to Algorithms. The running time of 84 | * this algorithm is O(n log n), where n is the number of input points. 85 | * 86 | * @param vertices [[x1, y1], [x2, y2], …] 87 | * @returns polygon [[x1, y1], [x2, y2], …] 88 | */ 89 | d3.geom.hull = function(vertices) { 90 | if (vertices.length < 3) return []; 91 | 92 | var len = vertices.length, 93 | plen = len - 1, 94 | points = [], 95 | stack = [], 96 | i, j, h = 0, x1, y1, x2, y2, u, v, a, sp; 97 | 98 | // find the starting ref point: leftmost point with the minimum y coord 99 | for (i=1; i= (x2*x2 + y2*y2)) { 129 | points[i].index = -1; 130 | } else { 131 | points[u].index = -1; 132 | a = points[i].angle; 133 | u = i; 134 | v = j; 135 | } 136 | } else { 137 | a = points[i].angle; 138 | u = i; 139 | v = j; 140 | } 141 | } 142 | 143 | // initialize the stack 144 | stack.push(h); 145 | for (i=0, j=0; i<2; ++j) { 146 | if (points[j].index !== -1) { 147 | stack.push(points[j].index); 148 | i++; 149 | } 150 | } 151 | sp = stack.length; 152 | 153 | // do graham's scan 154 | for (; j 0; 177 | } 178 | // Note: requires coordinates to be counterclockwise and convex! 179 | d3.geom.polygon = function(coordinates) { 180 | 181 | coordinates.area = function() { 182 | var i = 0, 183 | n = coordinates.length, 184 | a = coordinates[n - 1][0] * coordinates[0][1], 185 | b = coordinates[n - 1][1] * coordinates[0][0]; 186 | while (++i < n) { 187 | a += coordinates[i - 1][0] * coordinates[i][1]; 188 | b += coordinates[i - 1][1] * coordinates[i][0]; 189 | } 190 | return (b - a) * .5; 191 | }; 192 | 193 | coordinates.centroid = function(k) { 194 | var i = -1, 195 | n = coordinates.length - 1, 196 | x = 0, 197 | y = 0, 198 | a, 199 | b, 200 | c; 201 | if (!arguments.length) k = -1 / (6 * coordinates.area()); 202 | while (++i < n) { 203 | a = coordinates[i]; 204 | b = coordinates[i + 1]; 205 | c = a[0] * b[1] - b[0] * a[1]; 206 | x += (a[0] + b[0]) * c; 207 | y += (a[1] + b[1]) * c; 208 | } 209 | return [x * k, y * k]; 210 | }; 211 | 212 | // The Sutherland-Hodgman clipping algorithm. 213 | coordinates.clip = function(subject) { 214 | var input, 215 | i = -1, 216 | n = coordinates.length, 217 | j, 218 | m, 219 | a = coordinates[n - 1], 220 | b, 221 | c, 222 | d; 223 | while (++i < n) { 224 | input = subject.slice(); 225 | subject.length = 0; 226 | b = coordinates[i]; 227 | c = input[(m = input.length) - 1]; 228 | j = -1; 229 | while (++j < m) { 230 | d = input[j]; 231 | if (d3_geom_polygonInside(d, a, b)) { 232 | if (!d3_geom_polygonInside(c, a, b)) { 233 | subject.push(d3_geom_polygonIntersect(c, d, a, b)); 234 | } 235 | subject.push(d); 236 | } else if (d3_geom_polygonInside(c, a, b)) { 237 | subject.push(d3_geom_polygonIntersect(c, d, a, b)); 238 | } 239 | c = d; 240 | } 241 | a = b; 242 | } 243 | return subject; 244 | }; 245 | 246 | return coordinates; 247 | }; 248 | 249 | function d3_geom_polygonInside(p, a, b) { 250 | return (b[0] - a[0]) * (p[1] - a[1]) < (b[1] - a[1]) * (p[0] - a[0]); 251 | } 252 | 253 | // Intersect two infinite lines cd and ab. 254 | function d3_geom_polygonIntersect(c, d, a, b) { 255 | var x1 = c[0], x2 = d[0], x3 = a[0], x4 = b[0], 256 | y1 = c[1], y2 = d[1], y3 = a[1], y4 = b[1], 257 | x13 = x1 - x3, 258 | x21 = x2 - x1, 259 | x43 = x4 - x3, 260 | y13 = y1 - y3, 261 | y21 = y2 - y1, 262 | y43 = y4 - y3, 263 | ua = (x43 * y13 - y43 * x13) / (y43 * x21 - x43 * y21); 264 | return [x1 + ua * x21, y1 + ua * y21]; 265 | } 266 | // Adapted from Nicolas Garcia Belmonte's JIT implementation: 267 | // http://blog.thejit.org/2010/02/12/voronoi-tessellation/ 268 | // http://blog.thejit.org/assets/voronoijs/voronoi.js 269 | // See lib/jit/LICENSE for details. 270 | 271 | // Notes: 272 | // 273 | // This implementation does not clip the returned polygons, so if you want to 274 | // clip them to a particular shape you will need to do that either in SVG or by 275 | // post-processing with d3.geom.polygon's clip method. 276 | // 277 | // If any vertices are coincident or have NaN positions, the behavior of this 278 | // method is undefined. Most likely invalid polygons will be returned. You 279 | // should filter invalid points, and consolidate coincident points, before 280 | // computing the tessellation. 281 | 282 | /** 283 | * @param vertices [[x1, y1], [x2, y2], …] 284 | * @returns polygons [[[x1, y1], [x2, y2], …], …] 285 | */ 286 | d3.geom.voronoi = function(vertices) { 287 | var polygons = vertices.map(function() { return []; }); 288 | 289 | d3_voronoi_tessellate(vertices, function(e) { 290 | var s1, 291 | s2, 292 | x1, 293 | x2, 294 | y1, 295 | y2; 296 | if (e.a === 1 && e.b >= 0) { 297 | s1 = e.ep.r; 298 | s2 = e.ep.l; 299 | } else { 300 | s1 = e.ep.l; 301 | s2 = e.ep.r; 302 | } 303 | if (e.a === 1) { 304 | y1 = s1 ? s1.y : -1e6; 305 | x1 = e.c - e.b * y1; 306 | y2 = s2 ? s2.y : 1e6; 307 | x2 = e.c - e.b * y2; 308 | } else { 309 | x1 = s1 ? s1.x : -1e6; 310 | y1 = e.c - e.a * x1; 311 | x2 = s2 ? s2.x : 1e6; 312 | y2 = e.c - e.a * x2; 313 | } 314 | var v1 = [x1, y1], 315 | v2 = [x2, y2]; 316 | polygons[e.region.l.index].push(v1, v2); 317 | polygons[e.region.r.index].push(v1, v2); 318 | }); 319 | 320 | // Reconnect the polygon segments into counterclockwise loops. 321 | return polygons.map(function(polygon, i) { 322 | var cx = vertices[i][0], 323 | cy = vertices[i][1]; 324 | polygon.forEach(function(v) { 325 | v.angle = Math.atan2(v[0] - cx, v[1] - cy); 326 | }); 327 | return polygon.sort(function(a, b) { 328 | return a.angle - b.angle; 329 | }).filter(function(d, i) { 330 | return !i || (d.angle - polygon[i - 1].angle > 1e-10); 331 | }); 332 | }); 333 | }; 334 | 335 | var d3_voronoi_opposite = {"l": "r", "r": "l"}; 336 | 337 | function d3_voronoi_tessellate(vertices, callback) { 338 | 339 | var Sites = { 340 | list: vertices 341 | .map(function(v, i) { 342 | return { 343 | index: i, 344 | x: v[0], 345 | y: v[1] 346 | }; 347 | }) 348 | .sort(function(a, b) { 349 | return a.y < b.y ? -1 350 | : a.y > b.y ? 1 351 | : a.x < b.x ? -1 352 | : a.x > b.x ? 1 353 | : 0; 354 | }), 355 | bottomSite: null 356 | }; 357 | 358 | var EdgeList = { 359 | list: [], 360 | leftEnd: null, 361 | rightEnd: null, 362 | 363 | init: function() { 364 | EdgeList.leftEnd = EdgeList.createHalfEdge(null, "l"); 365 | EdgeList.rightEnd = EdgeList.createHalfEdge(null, "l"); 366 | EdgeList.leftEnd.r = EdgeList.rightEnd; 367 | EdgeList.rightEnd.l = EdgeList.leftEnd; 368 | EdgeList.list.unshift(EdgeList.leftEnd, EdgeList.rightEnd); 369 | }, 370 | 371 | createHalfEdge: function(edge, side) { 372 | return { 373 | edge: edge, 374 | side: side, 375 | vertex: null, 376 | "l": null, 377 | "r": null 378 | }; 379 | }, 380 | 381 | insert: function(lb, he) { 382 | he.l = lb; 383 | he.r = lb.r; 384 | lb.r.l = he; 385 | lb.r = he; 386 | }, 387 | 388 | leftBound: function(p) { 389 | var he = EdgeList.leftEnd; 390 | do { 391 | he = he.r; 392 | } while (he != EdgeList.rightEnd && Geom.rightOf(he, p)); 393 | he = he.l; 394 | return he; 395 | }, 396 | 397 | del: function(he) { 398 | he.l.r = he.r; 399 | he.r.l = he.l; 400 | he.edge = null; 401 | }, 402 | 403 | right: function(he) { 404 | return he.r; 405 | }, 406 | 407 | left: function(he) { 408 | return he.l; 409 | }, 410 | 411 | leftRegion: function(he) { 412 | return he.edge == null 413 | ? Sites.bottomSite 414 | : he.edge.region[he.side]; 415 | }, 416 | 417 | rightRegion: function(he) { 418 | return he.edge == null 419 | ? Sites.bottomSite 420 | : he.edge.region[d3_voronoi_opposite[he.side]]; 421 | } 422 | }; 423 | 424 | var Geom = { 425 | 426 | bisect: function(s1, s2) { 427 | var newEdge = { 428 | region: {"l": s1, "r": s2}, 429 | ep: {"l": null, "r": null} 430 | }; 431 | 432 | var dx = s2.x - s1.x, 433 | dy = s2.y - s1.y, 434 | adx = dx > 0 ? dx : -dx, 435 | ady = dy > 0 ? dy : -dy; 436 | 437 | newEdge.c = s1.x * dx + s1.y * dy 438 | + (dx * dx + dy * dy) * .5; 439 | 440 | if (adx > ady) { 441 | newEdge.a = 1; 442 | newEdge.b = dy / dx; 443 | newEdge.c /= dx; 444 | } else { 445 | newEdge.b = 1; 446 | newEdge.a = dx / dy; 447 | newEdge.c /= dy; 448 | } 449 | 450 | return newEdge; 451 | }, 452 | 453 | intersect: function(el1, el2) { 454 | var e1 = el1.edge, 455 | e2 = el2.edge; 456 | if (!e1 || !e2 || (e1.region.r == e2.region.r)) { 457 | return null; 458 | } 459 | var d = (e1.a * e2.b) - (e1.b * e2.a); 460 | if (Math.abs(d) < 1e-10) { 461 | return null; 462 | } 463 | var xint = (e1.c * e2.b - e2.c * e1.b) / d, 464 | yint = (e2.c * e1.a - e1.c * e2.a) / d, 465 | e1r = e1.region.r, 466 | e2r = e2.region.r, 467 | el, 468 | e; 469 | if ((e1r.y < e2r.y) || 470 | (e1r.y == e2r.y && e1r.x < e2r.x)) { 471 | el = el1; 472 | e = e1; 473 | } else { 474 | el = el2; 475 | e = e2; 476 | } 477 | var rightOfSite = (xint >= e.region.r.x); 478 | if ((rightOfSite && (el.side === "l")) || 479 | (!rightOfSite && (el.side === "r"))) { 480 | return null; 481 | } 482 | return { 483 | x: xint, 484 | y: yint 485 | }; 486 | }, 487 | 488 | rightOf: function(he, p) { 489 | var e = he.edge, 490 | topsite = e.region.r, 491 | rightOfSite = (p.x > topsite.x); 492 | 493 | if (rightOfSite && (he.side === "l")) { 494 | return 1; 495 | } 496 | if (!rightOfSite && (he.side === "r")) { 497 | return 0; 498 | } 499 | if (e.a === 1) { 500 | var dyp = p.y - topsite.y, 501 | dxp = p.x - topsite.x, 502 | fast = 0, 503 | above = 0; 504 | 505 | if ((!rightOfSite && (e.b < 0)) || 506 | (rightOfSite && (e.b >= 0))) { 507 | above = fast = (dyp >= e.b * dxp); 508 | } else { 509 | above = ((p.x + p.y * e.b) > e.c); 510 | if (e.b < 0) { 511 | above = !above; 512 | } 513 | if (!above) { 514 | fast = 1; 515 | } 516 | } 517 | if (!fast) { 518 | var dxs = topsite.x - e.region.l.x; 519 | above = (e.b * (dxp * dxp - dyp * dyp)) < 520 | (dxs * dyp * (1 + 2 * dxp / dxs + e.b * e.b)); 521 | 522 | if (e.b < 0) { 523 | above = !above; 524 | } 525 | } 526 | } else /* e.b == 1 */ { 527 | var yl = e.c - e.a * p.x, 528 | t1 = p.y - yl, 529 | t2 = p.x - topsite.x, 530 | t3 = yl - topsite.y; 531 | 532 | above = (t1 * t1) > (t2 * t2 + t3 * t3); 533 | } 534 | return he.side === "l" ? above : !above; 535 | }, 536 | 537 | endPoint: function(edge, side, site) { 538 | edge.ep[side] = site; 539 | if (!edge.ep[d3_voronoi_opposite[side]]) return; 540 | callback(edge); 541 | }, 542 | 543 | distance: function(s, t) { 544 | var dx = s.x - t.x, 545 | dy = s.y - t.y; 546 | return Math.sqrt(dx * dx + dy * dy); 547 | } 548 | }; 549 | 550 | var EventQueue = { 551 | list: [], 552 | 553 | insert: function(he, site, offset) { 554 | he.vertex = site; 555 | he.ystar = site.y + offset; 556 | for (var i=0, list=EventQueue.list, l=list.length; i next.ystar || 559 | (he.ystar == next.ystar && 560 | site.x > next.vertex.x)) { 561 | continue; 562 | } else { 563 | break; 564 | } 565 | } 566 | list.splice(i, 0, he); 567 | }, 568 | 569 | del: function(he) { 570 | for (var i=0, ls=EventQueue.list, l=ls.length; i top.y) { 646 | temp = bot; 647 | bot = top; 648 | top = temp; 649 | pm = "r"; 650 | } 651 | e = Geom.bisect(bot, top); 652 | bisector = EdgeList.createHalfEdge(e, pm); 653 | EdgeList.insert(llbnd, bisector); 654 | Geom.endPoint(e, d3_voronoi_opposite[pm], v); 655 | p = Geom.intersect(llbnd, bisector); 656 | if (p) { 657 | EventQueue.del(llbnd); 658 | EventQueue.insert(llbnd, p, Geom.distance(p, bot)); 659 | } 660 | p = Geom.intersect(bisector, rrbnd); 661 | if (p) { 662 | EventQueue.insert(bisector, p, Geom.distance(p, bot)); 663 | } 664 | } else { 665 | break; 666 | } 667 | }//end while 668 | 669 | for (lbnd = EdgeList.right(EdgeList.leftEnd); 670 | lbnd != EdgeList.rightEnd; 671 | lbnd = EdgeList.right(lbnd)) { 672 | callback(lbnd.edge); 673 | } 674 | } 675 | /** 676 | * @param vertices [[x1, y1], [x2, y2], …] 677 | * @returns triangles [[[x1, y1], [x2, y2], [x3, y3]], …] 678 | */ 679 | d3.geom.delaunay = function(vertices) { 680 | var edges = vertices.map(function() { return []; }), 681 | triangles = []; 682 | 683 | // Use the Voronoi tessellation to determine Delaunay edges. 684 | d3_voronoi_tessellate(vertices, function(e) { 685 | edges[e.region.l.index].push(vertices[e.region.r.index]); 686 | }); 687 | 688 | // Reconnect the edges into counterclockwise triangles. 689 | edges.forEach(function(edge, i) { 690 | var v = vertices[i], 691 | cx = v[0], 692 | cy = v[1]; 693 | edge.forEach(function(v) { 694 | v.angle = Math.atan2(v[0] - cx, v[1] - cy); 695 | }); 696 | edge.sort(function(a, b) { 697 | return a.angle - b.angle; 698 | }); 699 | for (var j = 0, m = edge.length - 1; j < m; j++) { 700 | triangles.push([v, edge[j], edge[j + 1]]); 701 | } 702 | }); 703 | 704 | return triangles; 705 | }; 706 | // Constructs a new quadtree for the specified array of points. A quadtree is a 707 | // two-dimensional recursive spatial subdivision. This implementation uses 708 | // square partitions, dividing each square into four equally-sized squares. Each 709 | // point exists in a unique node; if multiple points are in the same position, 710 | // some points may be stored on internal nodes rather than leaf nodes. Quadtrees 711 | // can be used to accelerate various spatial operations, such as the Barnes-Hut 712 | // approximation for computing n-body forces, or collision detection. 713 | d3.geom.quadtree = function(points, x1, y1, x2, y2) { 714 | var p, 715 | i = -1, 716 | n = points.length; 717 | 718 | // Type conversion for deprecated API. 719 | if (n && isNaN(points[0].x)) points = points.map(d3_geom_quadtreePoint); 720 | 721 | // Allow bounds to be specified explicitly. 722 | if (arguments.length < 5) { 723 | if (arguments.length === 3) { 724 | y2 = x2 = y1; 725 | y1 = x1; 726 | } else { 727 | x1 = y1 = Infinity; 728 | x2 = y2 = -Infinity; 729 | 730 | // Compute bounds. 731 | while (++i < n) { 732 | p = points[i]; 733 | if (p.x < x1) x1 = p.x; 734 | if (p.y < y1) y1 = p.y; 735 | if (p.x > x2) x2 = p.x; 736 | if (p.y > y2) y2 = p.y; 737 | } 738 | 739 | // Squarify the bounds. 740 | var dx = x2 - x1, 741 | dy = y2 - y1; 742 | if (dx > dy) y2 = y1 + dx; 743 | else x2 = x1 + dy; 744 | } 745 | } 746 | 747 | // Recursively inserts the specified point p at the node n or one of its 748 | // descendants. The bounds are defined by [x1, x2] and [y1, y2]. 749 | function insert(n, p, x1, y1, x2, y2) { 750 | if (isNaN(p.x) || isNaN(p.y)) return; // ignore invalid points 751 | if (n.leaf) { 752 | var v = n.point; 753 | if (v) { 754 | // If the point at this leaf node is at the same position as the new 755 | // point we are adding, we leave the point associated with the 756 | // internal node while adding the new point to a child node. This 757 | // avoids infinite recursion. 758 | if ((Math.abs(v.x - p.x) + Math.abs(v.y - p.y)) < .01) { 759 | insertChild(n, p, x1, y1, x2, y2); 760 | } else { 761 | n.point = null; 762 | insertChild(n, v, x1, y1, x2, y2); 763 | insertChild(n, p, x1, y1, x2, y2); 764 | } 765 | } else { 766 | n.point = p; 767 | } 768 | } else { 769 | insertChild(n, p, x1, y1, x2, y2); 770 | } 771 | } 772 | 773 | // Recursively inserts the specified point p into a descendant of node n. The 774 | // bounds are defined by [x1, x2] and [y1, y2]. 775 | function insertChild(n, p, x1, y1, x2, y2) { 776 | // Compute the split point, and the quadrant in which to insert p. 777 | var sx = (x1 + x2) * .5, 778 | sy = (y1 + y2) * .5, 779 | right = p.x >= sx, 780 | bottom = p.y >= sy, 781 | i = (bottom << 1) + right; 782 | 783 | // Recursively insert into the child node. 784 | n.leaf = false; 785 | n = n.nodes[i] || (n.nodes[i] = d3_geom_quadtreeNode()); 786 | 787 | // Update the bounds as we recurse. 788 | if (right) x1 = sx; else x2 = sx; 789 | if (bottom) y1 = sy; else y2 = sy; 790 | insert(n, p, x1, y1, x2, y2); 791 | } 792 | 793 | // Create the root node. 794 | var root = d3_geom_quadtreeNode(); 795 | 796 | root.add = function(p) { 797 | insert(root, p, x1, y1, x2, y2); 798 | }; 799 | 800 | root.visit = function(f) { 801 | d3_geom_quadtreeVisit(f, root, x1, y1, x2, y2); 802 | }; 803 | 804 | // Insert all points. 805 | points.forEach(root.add); 806 | return root; 807 | }; 808 | 809 | function d3_geom_quadtreeNode() { 810 | return { 811 | leaf: true, 812 | nodes: [], 813 | point: null 814 | }; 815 | } 816 | 817 | function d3_geom_quadtreeVisit(f, node, x1, y1, x2, y2) { 818 | if (!f(node, x1, y1, x2, y2)) { 819 | var sx = (x1 + x2) * .5, 820 | sy = (y1 + y2) * .5, 821 | children = node.nodes; 822 | if (children[0]) d3_geom_quadtreeVisit(f, children[0], x1, y1, sx, sy); 823 | if (children[1]) d3_geom_quadtreeVisit(f, children[1], sx, y1, x2, sy); 824 | if (children[2]) d3_geom_quadtreeVisit(f, children[2], x1, sy, sx, y2); 825 | if (children[3]) d3_geom_quadtreeVisit(f, children[3], sx, sy, x2, y2); 826 | } 827 | } 828 | 829 | function d3_geom_quadtreePoint(p) { 830 | return { 831 | x: p[0], 832 | y: p[1] 833 | }; 834 | } 835 | })(); 836 | -------------------------------------------------------------------------------- /data_analysis/static/javascript/data.json: -------------------------------------------------------------------------------- 1 | {"name":"root","children":[{"name":"speed","children":[{"name":"jquery-basis.js","size":4304,"language":"Javascript"},{"name":"filter.html","size":181,"language":"HTML"},{"name":"find.html","size":177,"language":"HTML"},{"name":"benchmarker.js","size":139,"language":"Javascript"},{"name":"css.html","size":68,"language":"HTML"},{"name":"index.html","size":62,"language":"HTML"},{"name":"benchmarker.css","size":52,"language":"CSS"},{"name":"event.html","size":46,"language":"HTML"},{"name":"slice.vs.concat.html","size":37,"language":"HTML"},{"name":"closest.html","size":36,"language":"HTML"},{"name":"benchmark.js","size":12,"language":"Javascript"}],"size":5114},{"name":"test","children":[{"name":"unit","children":[{"name":"event.js","size":1926,"language":"Javascript"},{"name":"ajax.js","size":1745,"language":"Javascript"},{"name":"effects.js","size":1617,"language":"Javascript"},{"name":"manipulation.js","size":1566,"language":"Javascript"},{"name":"core.js","size":1006,"language":"Javascript"},{"name":"attributes.js","size":971,"language":"Javascript"},{"name":"css.js","size":738,"language":"Javascript"},{"name":"traversing.js","size":545,"language":"Javascript"},{"name":"data.js","size":490,"language":"Javascript"},{"name":"offset.js","size":418,"language":"Javascript"},{"name":"deferred.js","size":346,"language":"Javascript"},{"name":"dimensions.js","size":325,"language":"Javascript"},{"name":"callbacks.js","size":271,"language":"Javascript"},{"name":"queue.js","size":247,"language":"Javascript"},{"name":"selector.js","size":127,"language":"Javascript"},{"name":"serialize.js","size":96,"language":"Javascript"},{"name":"support.js","size":31,"language":"Javascript"},{"name":"exports.js","size":5,"language":"Javascript"},{"name":"deprecated.js","size":1,"language":"Javascript"}],"size":12471},{"name":"index.html","size":288,"language":"HTML"},{"name":"data","children":[{"name":"testrunner.js","size":245,"language":"Javascript"},{"name":"testinit.js","size":202,"language":"Javascript"},{"name":"testsuite.css","size":112,"language":"CSS"},{"name":"selector","children":[{"name":"html5_selector.html","size":94,"language":"HTML"},{"name":"sizzle_cache.html","size":19,"language":"HTML"}],"size":113},{"name":"offset","children":[{"name":"table.html","size":43,"language":"HTML"},{"name":"absolute.html","size":41,"language":"HTML"},{"name":"scroll.html","size":39,"language":"HTML"},{"name":"fixed.html","size":34,"language":"HTML"},{"name":"relative.html","size":31,"language":"HTML"},{"name":"static.html","size":31,"language":"HTML"},{"name":"body.html","size":26,"language":"HTML"}],"size":245},{"name":"manipulation","children":[{"name":"iframe-denied.html","size":33,"language":"HTML"}],"size":33},{"name":"support","children":[{"name":"bodyBackground.html","size":27,"language":"HTML"},{"name":"shrinkWrapBlocks.html","size":23,"language":"HTML"},{"name":"csp.php","size":17,"language":"PHP"},{"name":"testElementCrash.html","size":17,"language":"HTML"},{"name":"csp.js","size":3,"language":"Javascript"}],"size":87},{"name":"with_fries.xml","size":25,"language":"XML"},{"name":"ajax","children":[{"name":"unreleasedXHR.html","size":24,"language":"HTML"}],"size":24},{"name":"name.php","size":23,"language":"PHP"},{"name":"core","children":[{"name":"cc_on.html","size":22,"language":"HTML"}],"size":22},{"name":"dimensions","children":[{"name":"documentSmall.html","size":21,"language":"HTML"},{"name":"documentLarge.html","size":17,"language":"HTML"}],"size":38},{"name":"readywaitloader.js","size":19,"language":"Javascript"},{"name":"event","children":[{"name":"onbeforeunload.html","size":19,"language":"HTML"},{"name":"syncReady.html","size":17,"language":"HTML"},{"name":"promiseReady.html","size":16,"language":"HTML"},{"name":"longLoadScript.php","size":4,"language":"PHP"}],"size":56},{"name":"etag.php","size":16,"language":"PHP"},{"name":"if_modified_since.php","size":15,"language":"PHP"},{"name":"jsonp.php","size":14,"language":"PHP"},{"name":"params_html.php","size":12,"language":"PHP"},{"name":"json.php","size":12,"language":"PHP"},{"name":"text.php","size":12,"language":"PHP"},{"name":"headers.php","size":12,"language":"PHP"},{"name":"script.php","size":11,"language":"PHP"},{"name":"dashboard.xml","size":11,"language":"XML"},{"name":"iframe.html","size":8,"language":"HTML"},{"name":"test.php","size":7,"language":"PHP"},{"name":"with_fries_over_jsonp.php","size":7,"language":"PHP"},{"name":"test.html","size":7,"language":"HTML"},{"name":"cleanScript.html","size":7,"language":"HTML"},{"name":"test2.html","size":5,"language":"HTML"},{"name":"errorWithJSON.php","size":4,"language":"PHP"},{"name":"atom+xml.php","size":4,"language":"PHP"},{"name":"statusText.php","size":3,"language":"PHP"},{"name":"errorWithText.php","size":3,"language":"PHP"},{"name":"test.js","size":3,"language":"Javascript"},{"name":"test3.html","size":3,"language":"HTML"},{"name":"nocontent.php","size":3,"language":"PHP"},{"name":"badjson.js","size":1,"language":"Javascript"},{"name":"badcall.js","size":1,"language":"Javascript"},{"name":"readywaitasset.js","size":1,"language":"Javascript"},{"name":"evalScript.php","size":1,"language":"PHP"},{"name":"echoQuery.php","size":1,"language":"PHP"},{"name":"json_obj.js","size":1,"language":"Javascript"},{"name":"jquery-1.8.2.ajax_xhr.min.js","size":1,"language":"Javascript"},{"name":"echoData.php","size":1,"language":"PHP"},{"name":"name.html","size":1,"language":"HTML"}],"size":1432},{"name":"delegatetest.html","size":214,"language":"HTML"},{"name":"hovertest.html","size":148,"language":"HTML"},{"name":"networkerror.html","size":76,"language":"HTML"},{"name":"localfile.html","size":74,"language":"HTML"},{"name":"readywait.html","size":61,"language":"HTML"},{"name":"xhtml.php","size":4,"language":"PHP"},{"name":"jquery.js","size":4,"language":"Javascript"}],"size":14772},{"name":"src","children":[{"name":"event.js","size":588,"language":"Javascript"},{"name":"effects.js","size":560,"language":"Javascript"},{"name":"ajax.js","size":525,"language":"Javascript"},{"name":"core.js","size":518,"language":"Javascript"},{"name":"manipulation.js","size":454,"language":"Javascript"},{"name":"css.js","size":389,"language":"Javascript"},{"name":"attributes.js","size":374,"language":"Javascript"},{"name":"traversing.js","size":226,"language":"Javascript"},{"name":"data.js","size":202,"language":"Javascript"},{"name":"callbacks.js","size":138,"language":"Javascript"},{"name":"offset.js","size":121,"language":"Javascript"},{"name":"queue.js","size":117,"language":"Javascript"},{"name":"deferred.js","size":103,"language":"Javascript"},{"name":"selector-native.js","size":100,"language":"Javascript"},{"name":"ajax","children":[{"name":"xhr.js","size":82,"language":"Javascript"},{"name":"script.js","size":51,"language":"Javascript"},{"name":"jsonp.js","size":50,"language":"Javascript"}],"size":183},{"name":"serialize.js","size":71,"language":"Javascript"},{"name":"support.js","size":60,"language":"Javascript"},{"name":"dimensions.js","size":25,"language":"Javascript"},{"name":"event-alias.js","size":12,"language":"Javascript"},{"name":"sizzle-jquery.js","size":8,"language":"Javascript"},{"name":"exports.js","size":8,"language":"Javascript"},{"name":"outro.js","size":1,"language":"Javascript"},{"name":"intro.js","size":1,"language":"Javascript"},{"name":"deprecated.js","size":1,"language":"Javascript"}],"size":4785},{"name":"Gruntfile.js","size":335,"language":"Javascript"},{"name":"build","children":[{"name":"release.js","size":193,"language":"Javascript"},{"name":"release-notes.js","size":46,"language":"Javascript"}],"size":239}],"size":25245} -------------------------------------------------------------------------------- /data_analysis/static/javascript/dataConverter.js: -------------------------------------------------------------------------------- 1 | var convertToJSON = function(data, origin) { 2 | return (origin == 'cloc') ? convertFromClocToJSON(data) : convertFromWcToJSON(data); 3 | }; 4 | 5 | /** 6 | * Convert the output of cloc in csv to JSON format 7 | * 8 | * > cloc . --csv --exclude-dir=vendor,tmp --by-file --report-file=data.cloc 9 | */ 10 | var convertFromClocToJSON = function(data) { 11 | var lines = data.split("\n"); 12 | lines.shift(); // drop the header line 13 | 14 | var json = {}; 15 | lines.forEach(function(line) { 16 | var cols = line.split(','); 17 | var filename = cols[1]; 18 | if (!filename) return; 19 | var elements = filename.split(/[\/\\]/); 20 | var current = json; 21 | elements.forEach(function(element) { 22 | if (!current[element]) { 23 | current[element] = {}; 24 | } 25 | current = current[element]; 26 | }); 27 | current.api = cols[0]; 28 | current.size = parseInt(cols[4], 10); 29 | }); 30 | 31 | json = getChildren(json)[0]; 32 | json.name = 'root'; 33 | 34 | return json; 35 | }; 36 | 37 | /** 38 | * Convert the output of wc to JSON format 39 | * 40 | * > git ls-files | xargs wc -l 41 | */ 42 | var convertFromWcToJSON = function(data) { 43 | var lines = data.split("\n"); 44 | 45 | var json = {}; 46 | var filename, size, cols, elements, current; 47 | lines.forEach(function(line) { 48 | cols = line.trim().split(' '); 49 | size = parseInt(cols[0], 10); 50 | if (!size) return; 51 | filename = cols[1]; 52 | if (filename === "total") return; 53 | if (!filename) return; 54 | elements = filename.split(/[\/\\]/); 55 | current = json; 56 | elements.forEach(function(element) { 57 | if (!current[element]) { 58 | current[element] = {}; 59 | } 60 | current = current[element]; 61 | }); 62 | current.size = size; 63 | }); 64 | 65 | json.children = getChildren(json); 66 | json.name = 'root'; 67 | 68 | return json; 69 | }; 70 | 71 | /** 72 | * Convert a simple json object into another specifying children as an array 73 | * Works recursively 74 | * 75 | * example input: 76 | * { a: { b: { c: { size: 12 }, d: { size: 34 } }, e: { size: 56 } } } 77 | * example output 78 | * { name: a, children: [ 79 | * { name: b, children: [ 80 | * { name: c, size: 12 }, 81 | * { name: d, size: 34 } 82 | * ] }, 83 | * { name: e, size: 56 } 84 | * ] } } 85 | */ 86 | var getChildren = function(json) { 87 | var children = []; 88 | if (json.api) return children; 89 | for (var key in json) { 90 | var child = { name: key }; 91 | if (json[key].size) { 92 | // value node 93 | child.size = json[key].size; 94 | child.api = json[key].api; 95 | } else { 96 | // children node 97 | var childChildren = getChildren(json[key]); 98 | if (childChildren) child.children = childChildren; 99 | } 100 | children.push(child); 101 | delete json[key]; 102 | } 103 | return children; 104 | }; 105 | 106 | // Recursively count all elements in a tree 107 | var countElements = function(node) { 108 | var nbElements = 1; 109 | if (node.children) { 110 | nbElements += node.children.reduce(function(p, v) { return p + countElements(v); }, 0); 111 | } 112 | return nbElements; 113 | }; 114 | -------------------------------------------------------------------------------- /data_analysis/static/javascript/detector.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Chrome AppSniffer 3 | * 4 | * Detect apps run on current page and send back to background page. 5 | * Some part of this script was refered from Wappalyzer Firefox Addon. 6 | * 7 | * @author Bao Nguyen 8 | * @license GPLv3 9 | **/ 10 | 11 | (function () { 12 | var _apps = {}; 13 | var doc = document.documentElement; 14 | var a 15 | 16 | // 1: detect by meta tags, the first matching group will be version 17 | var metas = doc.getElementsByTagName("meta"); 18 | var meta_tests = { 19 | 'generator': { 20 | 'Joomla': /joomla!?\s*([\d\.]+)?/i, 21 | 'vBulletin': /vBulletin\s*(.*)/i, 22 | 'WordPress': /WordPress\s*(.*)/i, 23 | 'XOOPS': /xoops/i, 24 | 'Plone': /plone/i, 25 | 'MediaWiki': /MediaWiki/i, 26 | 'CMSMadeSimple': /CMS Made Simple/i, 27 | 'SilverStripe': /SilverStripe/i, 28 | 'Movable Type': /Movable Type/i, 29 | 'Amiro.CMS': /Amiro/i, 30 | 'Koobi': /koobi/i, 31 | 'bbPress': /bbPress/i, 32 | 'DokuWiki': /dokuWiki/i, 33 | 'TYPO3': /TYPO3/i, 34 | 'PHP-Nuke': /PHP-Nuke/i, 35 | 'DotNetNuke': /DotNetNuke/i, 36 | 'Sitefinity': /Sitefinity\s+(.*)/i, 37 | 'WebGUI': /WebGUI/i, 38 | 'ez Publish': /eZ\s*Publish/i, 39 | 'BIGACE': /BIGACE/i, 40 | 'TypePad': /typepad\.com/i, 41 | 'Blogger': /blogger/i, 42 | 'PrestaShop': /PrestaShop/i, 43 | 'SharePoint': /SharePoint/, 44 | 'JaliosJCMS': /Jalios JCMS/i, 45 | 'ZenCart': /zen-cart/i, 46 | 'WPML': /WPML/i, 47 | 'PivotX': /PivotX/i, 48 | 'OpenACS': /OpenACS/i, 49 | 'AlphaCMS': /alphacms\s+(.*)/i, 50 | 'concrete5': /concrete5 -\s*(.*)$/, 51 | 'Webnode': /Webnode/, 52 | 'GetSimple': /GetSimple/, 53 | 'DataLifeEngine': /DataLife Engine/, 54 | 'ClanSphere': /ClanSphere/, 55 | }, 56 | 'copyright': { 57 | 'phpBB': /phpBB/i 58 | }, 59 | 'elggrelease': { 60 | 'Elgg': /.+/ 61 | }, 62 | 'powered-by': { 63 | 'Serendipity': /Serendipity/i, 64 | }, 65 | 'author': { 66 | 'Avactis': /Avactis Team/i 67 | } 68 | }; 69 | 70 | for (var idx in metas) 71 | { 72 | var m = metas[idx]; 73 | var name = m.name ? m.name.toLowerCase() : ""; 74 | 75 | if (!meta_tests[name]) continue; 76 | 77 | for (var t in meta_tests[name]) 78 | { 79 | if (t in _apps) continue; 80 | 81 | var r = meta_tests[name][t].exec(m.content); 82 | if (r) 83 | { 84 | _apps[t] = r[1] ? r[1] : -1; 85 | } 86 | } 87 | } 88 | 89 | // 2: detect by script tags 90 | var scripts = doc.getElementsByTagName("script"); 91 | 92 | var script_tests = { 93 | 'Google Analytics': /google-analytics.com\/(ga|urchin).js/i, 94 | 'Quantcast': /quantserve\.com\/quant\.js/i, 95 | 'Prototype': /prototype\.js/i, 96 | 'Joomla': /\/components\/com_/, 97 | 'Ubercart': /uc_cart/i, 98 | 'Closure': /\/goog\/base\.js/i, 99 | 'MODx': /\/min\/b=.*f=.*/, 100 | 'MooTools': /mootools/i, 101 | 'Dojo': /dojo(\.xd)?\.js/i, 102 | 'script.aculo.us': /scriptaculous\.js/i, 103 | 'Disqus': /disqus.com\/forums/i, 104 | 'GetSatisfaction': /getsatisfaction\.com\/feedback/i, 105 | 'Wibiya': /wibiya\.com\/Loaders\//i, 106 | 'reCaptcha': /(google\.com\/recaptcha|api\.recaptcha\.net\/)/i, 107 | 'Mollom': /mollom\/mollom\.js/i, // only work on Drupal now 108 | 'ZenPhoto': /zp-core\/js/i, 109 | 'Gallery2': /main\.php\?.*g2_.*/i, 110 | 'AdSense': /pagead\/show_ads\.js/, 111 | 'XenForo': /js\/xenforo\//i, 112 | 'Cappuccino': /Frameworks\/Objective-J\/Objective-J\.js/, 113 | 'Avactis': /\/avactis-themes\//i, 114 | 'Volusion': /a\/j\/javascripts\.js/, 115 | 'AddThis': /addthis\.com\/js/, 116 | 'BuySellAds': /buysellads.com\/.*bsa\.js/, 117 | 'Weebly': /weebly\.com\/weebly\//, 118 | 'Bootstrap': /bootstrap-.*\.js/, 119 | 'Jigsy': /javascripts\/asterion\.js/, // may change later 120 | 'Yola': /analytics\.yola\.net/, // may change later 121 | 'Alfresco': /(alfresco)+(-min)?(\/scripts\/menu)?\.js/ // both Alfresco Share and Explorer apps 122 | }; 123 | 124 | for (var idx in scripts) 125 | { 126 | var s = scripts[idx]; 127 | if (!s.src) continue; 128 | s = s.src; 129 | 130 | for (var t in script_tests) 131 | { 132 | if (t in _apps) continue; 133 | if (script_tests[t].test(s)) 134 | { 135 | _apps[t] = -1; 136 | } 137 | } 138 | } 139 | 140 | // 3: detect by domains 141 | 142 | // 4: detect by regexp 143 | var text = document.documentElement.outerHTML; 144 | var text_tests = { 145 | 'SMF': / 4 | 5 | 6 | 7 | 8 | 36 | 48 | {% endblock %} 49 | {% block select %} 50 | {% endblock %} -------------------------------------------------------------------------------- /data_analysis/templates/posts/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Mongodb数据分析 4 | 5 | 6 | 7 | 8 | 9 | {% block media %} 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | {% endblock %} 18 | 19 | 20 | 21 |
22 | 23 | 24 | 25 | 59 |
60 |
61 |

{{ title }}

62 | 63 | 64 |

{{ des }}

65 | 66 |
67 | {% block body %} 68 | {% block select %} 69 |
70 |
71 |
72 | 73 | 78 |
79 |
80 | 81 | 83 |
84 |
85 | 86 | 88 |
89 | 90 |
91 |
92 | {% endblock %} 93 |
94 |
95 |
96 |
97 | {% endblock %} 98 | 129 |
130 | 170 | 171 | 172 | -------------------------------------------------------------------------------- /data_analysis/templates/posts/trouble.html: -------------------------------------------------------------------------------- 1 | {% extends "posts/index.html" %} 2 | {% block media %} 3 | {% endblock %} 4 | {% block body %} 5 |
6 |

我也推荐神奇的dex: https://github.com/mongolab/dex

7 | 10 |

索引字段不够(需要复合索引)

11 |
xx/models.py 143-146行
 12 | statuses = list(coll.find(                                                 
 13 |         {"is_update": False},                                                 
 14 |         {"uid": 1}                                                             
 15 |     ).sort('date', -1).limit(200))
16 |

 17 | > db.status_5008fe57b3158aa4709ecd8f_201311.getIndexes()
 18 | [
 19 | 	{
 20 | 		"v" : 1,
 21 | 		"key" : {
 22 | 			"_id" : 1
 23 | 		},
 24 | 		"ns" : "dongwm.status_5008fe57b3158aa4709ecd8f_201311",
 25 | 		"name" : "_id_"
 26 | 	},
 27 | 	{
 28 | 		"v" : 1,
 29 | 		"key" : {
 30 | 			"date" : -1
 31 | 		},
 32 | 		"ns" : "dongwm.status_5008fe57b3158aa4709ecd8f_201311",
 33 | 		"name" : "cdate_-1"
 34 | 	},
 35 | 	{
 36 | 		"v" : 1,
 37 | 		"key" : {
 38 | 			"idate" : -1
 39 | 		},
 40 | 		"ns" : "dongwm.status_5008fe57b3158aa4709ecd8f_201311",
 41 | 		"name" : "idate_-1"
 42 | 	}
 43 | ]
 44 | 		
45 |

 46 | db["status_5008fe57b3158aa4709ecd8f_201311"].ensureIndex({"is_update": 1, "date": 1}, {"background": true})
 47 | 		
48 |

没有索引

49 |
xx/tasks.py 551-564行
 50 | pre_date = update_date - timedelta(hours=1)                            
 51 | pre_infcoll = s_db["inf_%s_%s" % (                       
 52 |     key["_id"], pre_date.strftime("%Y%m")                              
 53 | )]                                                                     
 54 | pre_ho = {}                                                      
 55 | pre_inf = (                                                    
 56 |     pre_infcoll.find_one({                                      
 57 |         "day": update_date.replace(                                    
 58 |             hour=0,                                                  
 59 |             minute=0,                                                 
 60 |             second=0,                                                 
 61 |             microsecond=0                                              
 62 |         )                                                              
 63 |     }) or {}).get(str(pre_date.hour), [])
 64 | 			
65 |

 66 | > db.dongwm.influence_51ee2552fb0dd8d290a7749a_201311.getIndexes()
 67 | [ ]
 68 | 			
69 |

 70 | db["inf_51ee2552fb0dd8d290a7749a_201311"].ensureIndex({"day": 1}, {"background": true})
 71 | 			
72 |

索引顺序问题

74 |
xx/models.py 1110-1119行
 75 | def get_o_s(uid, date):                                                                 
 76 |     return list(db.st.find(                                                                                         
 77 |         {                                                                      
 78 |             'user_id': uid,                                                    
 79 |             'created_at': {'$gte': date},                                      
 80 |             'rs': 0                                           
 81 |         },                                                                     
 82 |         {'rsince_timestamp': 1, 'csince_timestamp': 1, 'last_cid': 1}          
 83 |     ))   
 84 | 		
85 |

 86 | > db.status.getIndexes()
 87 | [
 88 | 	{
 89 | 		"v" : 1,
 90 | 		"key" : {
 91 | 			"_id" : 1
 92 | 		},
 93 | 		"ns" : "dongwm.status",
 94 | 		"name" : "_id_"
 95 | 	}
 96 | ]
 97 | 		
98 |

 99 | db["status"].ensureIndex({"user_id": 1, "rs": 1, "created_at": 1}, {"background": true})
100 | 		
101 | 102 |
103 | 104 | {% endblock %} -------------------------------------------------------------------------------- /data_analysis/views.py: -------------------------------------------------------------------------------- 1 | #coding=utf-8 2 | import random 3 | import datetime 4 | from flask import (Blueprint, request, redirect, render_template, url_for, 5 | jsonify) 6 | from flask.views import MethodView 7 | from data_analysis.models import Apidist, Celery, Mongo 8 | 9 | posts = Blueprint('posts', __name__, template_folder='templates') 10 | 11 | color = ["#FF0F00", "#FF6600", "#FF9E01", "#FCD202", "#F8FF01", "#B0DE09", 12 | "#04D215", "#0D8ECF", "#0D52D1", "#2A0CD0", "#8A0CCF", "#CD0D74", 13 | "#754DEB", "#2c3e50", "#2c3e50", "#e67e22", "#e74c3c", "#ecf0f1", 14 | "#95a5a6", "#d35400", "#8e44ad", "#8e44ad", "#bdc3c7", "#d35400", 15 | "#1abc9c", "#2ecc71", "#3498db", "#9b59b6", "#34495e"] 16 | 17 | celery = dict(task=[('count', ["simple_column", "pie"]), 18 | ('cost', ["multi_column"])], 19 | time=[('count', ["simple_column", "pie"])]) 20 | 21 | mongo = dict(all=[('getmore', ["multi_column"]), ('update', ["multi_column"]), 22 | ('insert', ["multi_column"]), ('command', ["multi_column"]), 23 | ('query', ["multi_column"])], 24 | sentiment=[('getmore', ["multi_column"]), ('update', ["multi_column"]), 25 | ('insert', ["multi_column"]), ('command', ["multi_column"]), 26 | ('query', ["multi_column"])], 27 | qq_online=[('getmore', ["multi_column"]), ('update', ["multi_column"]), 28 | ('insert', ["multi_column"]), ('command', ["multi_column"]), 29 | ('query', ["multi_column"])], 30 | total=[('getmore', ["simple_column", "pie"]), 31 | ('update', ["simple_column", "pie"]), 32 | ('insert', ["simple_column", "pie"]), 33 | ('command', ["simple_column", "pie"]), 34 | ('query', ["simple_column", "pie"]), 35 | ('all_op', ["simple_column", "pie"]), 36 | ('all_slow', ["simple_column", "pie"])]) 37 | d3 = dict() 38 | 39 | 40 | def make_json(res, title, category, value, chart, key, des, total, titles=None, 41 | values=None): 42 | color1, color2, color3 = None, None, None 43 | if titles is not None: 44 | func= lambda x:x[values[key]] 45 | else: 46 | func= lambda x:x[value] 47 | dict_list = sorted(res, key=func, reverse=True) 48 | if titles is None: 49 | l = len(dict_list) 50 | color_choice = random.sample(color, l) 51 | for num, d in enumerate(dict_list): 52 | d['color'] = color_choice[num] 53 | else: 54 | color_choice = random.sample(color, 3) 55 | for d in dict_list: 56 | d['color1'] = color_choice[0] 57 | d['color2'] = color_choice[1] 58 | d['color3'] = color_choice[2] 59 | return jsonify(result=dict_list, title=title, category=category, des=des, 60 | value=value, titles=titles, values=values, chart=chart, 61 | total=total) 62 | 63 | def make_factory(change_dict): 64 | dict = {'title': 'name', 'value': 'call'} 65 | dict.update(change_dict) 66 | return dict 67 | 68 | class JsonView(MethodView): 69 | 70 | def get(self, url): 71 | if url == 'celery': 72 | return jsonify(celery) 73 | elif url == 'mongo': 74 | return jsonify(mongo) 75 | 76 | class IndexView(MethodView): 77 | 78 | def get(self): 79 | return redirect(url_for('.mongo')) 80 | 81 | class D3View(MethodView): 82 | 83 | def get(self): 84 | return render_template('posts/d3.html', des=u'celery任务调用次数略览', 85 | title=u'Celery Task调用分布') 86 | 87 | class TroubleView(MethodView): 88 | 89 | def get(self): 90 | return render_template('posts/trouble.html', title=u'Mongodb存在问题分析', 91 | des=u'总结错误索引原因和解决方案') 92 | 93 | class MongoView(MethodView): 94 | 95 | def get(self): 96 | l = Mongo.objects.distinct(field="database") 97 | #l.insert(0, 'all') 98 | l.insert(0, 'total') 99 | return render_template('posts/index.html', first_type=u'Mongodb分析', 100 | title=u'Mongodb使用数据分析', l=l, 101 | des=u'数据库操作数据分析') 102 | 103 | def all(self, type): 104 | return self.time('all', type) 105 | 106 | def sentiment(self, type): 107 | return self.time('sentiment', type) 108 | 109 | def qq_online(self, type): 110 | return self.time('qq_online', type) 111 | 112 | def time(self, database, type): 113 | if database == 'all': 114 | obj = Mongo.objects()['hour'] 115 | else: 116 | obj = Mongo.objects(database=database)[0]['hour'] 117 | o = [] 118 | all_total = 0 119 | slow_total = 0 120 | for i in range(24): 121 | i = str(i) 122 | d = {} 123 | d['name'] = u'{0}时'.format(str(i)) 124 | d['op'] = obj[i]['op'][type] 125 | d['slow'] = obj[i]['slow'][type] 126 | all_total += d['op'] 127 | slow_total += d['slow'] 128 | o.append(d) 129 | titles = (u'慢查询', u'全部查询') 130 | values = ('slow', 'op') 131 | return make_factory({'titles': titles, type: o, 132 | 'total': '慢查询:{0}, 全部查询:{1}'.format( 133 | slow_total, all_total), 134 | 'des': '[数据库{0} 类型{1}] 总量'.format(database, 135 | type), 136 | 'category': u'mongodb分析', 'values': values}) 137 | 138 | def total_data(self, type): 139 | getmore, insert, update, command, query = 0, 0, 0, 0, 0 140 | for i in range(24): 141 | for obj in Mongo.objects(): 142 | insert += obj['hour'][str(i)][type]['insert'] 143 | update += obj['hour'][str(i)][type]['update'] 144 | command += obj['hour'][str(i)][type]['command'] 145 | query += obj['hour'][str(i)][type]['query'] 146 | getmore += obj['hour'][str(i)][type]['getmore'] 147 | count = [] 148 | for name, call in ((u'查询', query), (u'插入', insert), 149 | (u'命令', command), (u'更新', update), 150 | ('getomre', getmore)): 151 | d = {} 152 | d['name'] = name 153 | d['call'] = call 154 | count.append(d) 155 | total = getmore + insert + update + command + query 156 | return count, total 157 | 158 | def all_op(self): 159 | data = self.total_data('op') 160 | return make_factory({'all_op': data[0], 'des': u'数据库操作总量', 161 | 'category': u'mongodb操作分布', 162 | 'total': data[1]}) 163 | 164 | def all_slow(self): 165 | data = self.total_data('slow') 166 | return make_factory({'all_slow': data[0], 'des': u'数据库慢查询总量', 167 | 'category': u'mongodb慢操作分布', 168 | 'total': data[1]}) 169 | 170 | def total(self, type): 171 | if type == 'all_op': 172 | return self.all_op() 173 | elif type == 'all_slow': 174 | return self.all_slow() 175 | call = 0 176 | call_total = 0 177 | count = [] 178 | for i in range(24): 179 | d = {} 180 | for obj in Mongo.objects(): 181 | call += obj['hour'][str(i)]['total'] 182 | d['name'] = u'{0}时'.format(str(i)) 183 | d['call'] = call 184 | call_total += call 185 | call = 0 186 | count.append(d) 187 | return make_factory({type: count, 'category':u'mongodb分布', 188 | 'des': u'数据库查询总量', 189 | 'total': call_total}) 190 | 191 | def post(self): 192 | return get_form(request, self, mongo, key=1) 193 | 194 | 195 | def get_form(request, self, type, key=2): 196 | vt = request.form.getlist('vt')[0] 197 | data_type = request.form.getlist('type')[0] 198 | data_type = data_type if data_type else type[vt][0][0] 199 | chart = request.form.getlist('chart')[0] 200 | if not chart: 201 | for t in type[vt]: 202 | if data_type == t[0]: 203 | chart = t[1][0] 204 | data = getattr(self, vt)(data_type) 205 | if chart in ["simple_column", "pie"]: 206 | return make_json(data[data_type], data['title'], data['category'], 207 | data['value'], chart, key, data['des'], data['total']) 208 | elif chart == 'multi_column': 209 | return make_json(data[data_type], data['title'], data['category'], 210 | data['value'], chart, key, data['des'], data['total'], 211 | titles=data['titles'], values=data['values']) 212 | 213 | 214 | class CeleryView(MethodView): 215 | def time(self, type): 216 | count = [] 217 | total = 0 218 | for i in range(23): 219 | d = {} 220 | d['name'] = u'{0}时'.format(str(i)) 221 | d['call'] = Celery.objects( 222 | time__gte=datetime.datetime(2013, 10, 27, i), 223 | time__lt=datetime.datetime(2013, 10, 27, i+1)).count() 224 | total += d['call'] 225 | count.append(d) 226 | return make_factory(dict(count=count, category=u'celery分布', 227 | des=u'celery总量', total=total)) 228 | 229 | def post(self): 230 | return get_form(request, self, celery) 231 | 232 | def task(self, type): 233 | count = [] 234 | cost = [] 235 | total = 0 236 | for i in Celery.objects.distinct(field='task'): 237 | d = {} 238 | c = {} 239 | data = Celery.objects(task=i) 240 | d['call'] = data.count() 241 | d['name'], c['name'] = i, i 242 | c['avg'] = data.order_by('cost')[0].cost, # min 243 | c['min'] = data.average('cost'), 244 | c['max'] = data.order_by('-cost')[0].cost # max 245 | total += d['call'] 246 | cost.append(c) 247 | count.append(d) 248 | titles = (u'最小值', u'平均值', u'最大值') 249 | values = ('avg', 'min', 'max') 250 | return make_factory(dict(titles=titles, count=count, cost=cost, 251 | category=u'celery分析', values=values, 252 | des=u'celery总量', total=total)) 253 | 254 | def get(self): 255 | l = ['task', 'time'] 256 | return render_template('posts/index.html', first_type=u'监控类型', 257 | title=u'mongo日志分析', 258 | des=u'分析mongodb全天日志整理的分布数据', l=l) 259 | 260 | posts.add_url_rule('/mongo/', view_func=MongoView.as_view('mongo')) 261 | posts.add_url_rule('/', view_func=IndexView.as_view('index')) 262 | posts.add_url_rule('/celery/', view_func=CeleryView.as_view('celery')) 263 | posts.add_url_rule('/json//', view_func=JsonView.as_view('json')) 264 | posts.add_url_rule('/d3/', view_func=D3View.as_view('d3')) 265 | posts.add_url_rule('/trouble/', view_func=TroubleView.as_view('trouble')) 266 | -------------------------------------------------------------------------------- /dump/fetch_data/celery.bson: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dongweiming/data-analysis/49005c06590c8488f0dc7ba0992acadfb6407aec/dump/fetch_data/celery.bson -------------------------------------------------------------------------------- /dump/fetch_data/celery.metadata.json: -------------------------------------------------------------------------------- 1 | { "indexes" : [ { "v" : 1, "key" : { "_id" : 1 }, "ns" : "fetch_data.celery", "name" : "_id_" }, { "v" : 1, "key" : { "time" : -1 }, "ns" : "fetch_data.celery", "name" : "time_-1" }, { "v" : 1, "key" : { "cost" : -1 }, "ns" : "fetch_data.celery", "name" : "cost_-1" }, { "v" : 1, "key" : { "task" : 1 }, "ns" : "fetch_data.celery", "name" : "task_1" } ] } -------------------------------------------------------------------------------- /dump/fetch_data/mongo.bson: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dongweiming/data-analysis/49005c06590c8488f0dc7ba0992acadfb6407aec/dump/fetch_data/mongo.bson -------------------------------------------------------------------------------- /dump/fetch_data/mongo.metadata.json: -------------------------------------------------------------------------------- 1 | { "indexes" : [ { "v" : 1, "key" : { "_id" : 1 }, "ns" : "fetch_data.mongo", "name" : "_id_" } ] } -------------------------------------------------------------------------------- /dump/fetch_data/system.indexes.bson: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dongweiming/data-analysis/49005c06590c8488f0dc7ba0992acadfb6407aec/dump/fetch_data/system.indexes.bson -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | flask 2 | flask-mongoengine 3 | --------------------------------------------------------------------------------