47 |
48 |
49 |
50 |
51 |
24 |
25 |
32 |
38 |
39 |
40 |
41 | §[messages.file-overview]§ | §[messages.glam-tool]§ (Cassandra)
5 |
6 |
7 |
8 |
12 |
16 |
17 |
18 |
19 |
20 | §[messages.file-overview]§ | §[messages.glam-tool]§ (Cassandra)
5 |
6 |
7 |
8 |
12 |
16 |
17 |
18 |
19 |
20 |
21 | ![§[messages.glam-tool]§](/assets/img/glam-logo-horizontal.svg)
6 |
7 |
--------------------------------------------------------------------------------
/app/pages/views/templates/mobile-sidebar.html:
--------------------------------------------------------------------------------
1 |
28 |
--------------------------------------------------------------------------------
/app/pages/views/templates/secondary-sidebar.html:
--------------------------------------------------------------------------------
1 |
4 |
--------------------------------------------------------------------------------
/app/pages/views/templates/sidebar.html:
--------------------------------------------------------------------------------
1 | §[messages.file-overview]§ | §[messages.glam-tool]§ (Cassandra)
5 |
6 |
7 |
8 |
12 |
16 |
20 |
21 |
22 |
23 |
24 | (https://synapta.it/)",
5 | "license": "MIT",
6 | "private": true,
7 | "dependencies": {
8 | "@sentry/node": "^5.2.0",
9 | "apicache": "~0.1.5",
10 | "cookie-parser": "^1.4.5",
11 | "csv": "^5.1.1",
12 | "express": "^4.16.4",
13 | "express-recaptcha": "^5.0.2",
14 | "http-auth": "^3.2.4",
15 | "http-proxy-middleware": "^2.0.0",
16 | "ini": "^2.0.0",
17 | "iso-639-1": "^2.1.9",
18 | "jsonwebtoken": "^8.5.1",
19 | "mariadb": "^2.1.1",
20 | "mime-types": "^2.1.31",
21 | "mongodb": "~3.1.13",
22 | "morgan": "^1.9.1",
23 | "mustache": "^4.2.0",
24 | "nodemailer": "^6.7.0",
25 | "nodemailer-mailgun-transport": "^2.1.3",
26 | "pg": "~7.4.0",
27 | "puppeteer": "^5.5.0",
28 | "raml2html": "latest",
29 | "request": "^2.88.0",
30 | "utf8": "~2.1.2"
31 | },
32 | "scripts": {
33 | "postinstall": "raml2html ./docs/api.raml > ./app/pages/docs.html",
34 | "start": "cd app && node server.js"
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/recommender/run.py:
--------------------------------------------------------------------------------
1 | import fcntl
2 | import json
3 | import logging
4 | import os
5 | import subprocess
6 | import sys
7 | from subprocess import SubprocessError
8 |
9 | import pymongo
10 | import sentry_sdk
11 | from sentry_sdk.integrations.logging import LoggingIntegration
12 |
13 | config_file = '../config/config.json'
14 |
15 | logging.basicConfig(level=logging.INFO,
16 | format='%(asctime)s %(levelname)s %(message)s')
17 |
18 |
19 | def run_recommender(glam):
20 | logging.info('Running recommender for %s', glam['name'])
21 |
22 | try:
23 | subprocess.run(['python3', 'similarity.py', glam['database']], check=True)
24 | except SubprocessError:
25 | logging.error('Subprocess similarity.py failed')
26 |
27 |
28 | def main():
29 | config = json.load(open(config_file))
30 |
31 | try:
32 | logging.info('External error reporting enabled')
33 |
34 | sentry_logging = LoggingIntegration(
35 | level=logging.INFO,
36 | event_level=logging.ERROR
37 | )
38 | sentry_sdk.init(
39 | dsn=config['raven']['glamtoolsetl']['DSN'],
40 | integrations=[sentry_logging]
41 | )
42 | except KeyError:
43 | logging.info('External error reporting DISABLED')
44 | pass
45 |
46 | client = pymongo.MongoClient(config['mongodb']['url'])
47 | db = client[config['mongodb']['database']]
48 | collection = db[config['mongodb']['collection']]
49 |
50 | for glam in collection.find():
51 | if 'status' in glam:
52 | if glam['status'] == 'paused':
53 | logging.info('Glam %s is paused', glam['name'])
54 | continue
55 |
56 | if glam['status'] == 'failed':
57 | logging.info('Glam %s is failed', glam['name'])
58 | continue
59 |
60 | run_recommender(glam)
61 |
62 |
63 | if __name__ == '__main__':
64 | # change the working directory to the script's own directory
65 | script_dir = os.path.dirname(sys.argv[0])
66 | if script_dir != '':
67 | os.chdir(script_dir)
68 |
69 | try:
70 | lockfile = open('/tmp/cassandra_recommender.lock', 'w')
71 | fcntl.flock(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
72 | main()
73 | fcntl.flock(lockfile, fcntl.LOCK_UN)
74 | except IOError:
75 | raise SystemExit('Recommender is already running')
76 |
--------------------------------------------------------------------------------
/recommender/similarity.py:
--------------------------------------------------------------------------------
1 | import json
2 | import logging
3 | import sys
4 | from datetime import date, timedelta
5 |
6 | import mwclient
7 | import psycopg2
8 | import requests
9 | from psycopg2.errors import UniqueViolation
10 |
11 | if len(sys.argv) == 1:
12 | print('Missing database')
13 | sys.exit(1)
14 |
15 | database = sys.argv[1]
16 |
17 | logging.basicConfig(level=logging.INFO,
18 | format='%(asctime)s %(levelname)s %(message)s')
19 |
20 | with open('../config/config.json', 'r') as fp:
21 | config = json.load(fp)
22 |
23 | conn = psycopg2.connect(host=config['postgres']['host'],
24 | port=config['postgres']['port'],
25 | dbname=database,
26 | user=config['postgres']['user'],
27 | password=config['postgres']['password'])
28 |
29 | conn.set_session(autocommit=True)
30 |
31 | cur = conn.cursor()
32 |
33 | last_update = date.today() - timedelta(days=7)
34 |
35 | cur.execute("""SELECT i.img_name
36 | FROM images i
37 | LEFT JOIN usages u ON i.img_name = u.gil_to
38 | LEFT JOIN recommendations r ON i.img_name = r.img_name
39 | WHERE (i.is_alive = TRUE OR i.is_alive IS NULL)
40 | AND u.is_alive IS NULL
41 | AND (r.last_update < %s OR r.last_update IS NULL)
42 | GROUP BY i.img_name""", (last_update,))
43 |
44 | images = cur.fetchall()
45 | image_counter = 1
46 |
47 | site = mwclient.Site('commons.wikimedia.org')
48 |
49 |
50 | def compute_category(image):
51 | entities = []
52 |
53 | page = site.images[image]
54 |
55 | for cat in page.categories(show='!hidden'):
56 | iwlinks = cat.iwlinks()
57 | for iw in iwlinks:
58 | if iw[0] == 'd':
59 | entities.append(iw[1])
60 |
61 | return entities
62 |
63 |
64 | def process_entities(image, entities, scores):
65 | for e in entities:
66 | try:
67 | r = requests.get(
68 | 'http://www.wikidata.org/wiki/Special:EntityData/' + e + '.json')
69 | entity = r.json()['entities'][e]
70 |
71 | if 'sitelinks' not in entity:
72 | continue
73 |
74 | if 'claims' not in entity:
75 | continue
76 |
77 | if len(entity['sitelinks']) == 0:
78 | continue
79 |
80 | if not ('enwiki' in entity['sitelinks'] or 'dewiki' in entity['sitelinks'] or 'frwiki' in entity['sitelinks'] or 'itwiki' in entity['sitelinks']):
81 | continue
82 |
83 | # instance of
84 | if 'P31' not in entity['claims']:
85 | continue
86 |
87 | instance_of = entity['claims']['P31'][0]['mainsnak']['datavalue']['value']['id']
88 |
89 | # disambiguation, category, events, template
90 | if instance_of in ['Q4167410', 'Q4167836', 'Q18340514', 'Q11119738']:
91 | continue
92 |
93 | except (ValueError, KeyError):
94 | continue
95 |
96 | try:
97 | if scores is not None:
98 | cur.execute("""INSERT INTO recommendations
99 | (img_name, site, title, url, score, last_update)
100 | VALUES(%s, %s, %s, %s, %s, %s)""", (image, 'wikidata', e, 'https://www.wikidata.org/wiki/' + e, float(scores[e]), date.today()))
101 | else:
102 | cur.execute("""INSERT INTO recommendations
103 | (img_name, site, title, url, last_update)
104 | VALUES(%s, %s, %s, %s, %s)""", (image, 'wikidata', e, 'https://www.wikidata.org/wiki/' + e, date.today()))
105 | except UniqueViolation:
106 | continue
107 |
108 |
109 | for image in images:
110 | logging.info('Processing image %s of %s: %s', image_counter, len(images), image[0])
111 | image_counter += 1
112 |
113 | try:
114 | cur.execute("""DELETE FROM recommendations
115 | WHERE img_name = %s""", (image[0],))
116 |
117 | entities = compute_category(image[0])
118 | process_entities(image[0], entities, None)
119 |
120 | except ValueError:
121 | continue
122 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | psycopg2-binary
2 | pymongo
3 | sentry-sdk
4 | mwclient
5 | requests
--------------------------------------------------------------------------------
33 |
37 |
34 |
36 | §[messages.introduction]§
35 |
42 |
65 |
69 |
73 |
77 |
81 |
85 |
86 |
87 |
88 |
89 |
--------------------------------------------------------------------------------
/app/pages/views/category-network/tpl/category-network.tpl:
--------------------------------------------------------------------------------
1 | {{#each nodes}}
2 |
43 |
61 |
62 |
63 |
64 |
44 |
60 |
45 |
48 |
59 | Welcome to Cassandra
46 |A GLAM statistical application for the impact analysis of Wikimedia projects
47 |48 |
49 | The GLAM statistical tool named Cassandra is a project launched by Wikimedia CH in 2015, developed in collaboration with Swiss cultural institutions and GLAM partners of Wikimedia CH. The aim of the project is to support GLAM institutions (Galleries, Libraries, Archives and Museums) in gaining statistical insights regarding their shared collections across the Wikimedia projects in the world. 50 |
51 |52 | Cassandra visualizes statistical data of media files uploaded under a free license in Wikimedia Commons. The accumulated data is presented in various diagrams and graphs, reflecting views and usage of freely accessible and shared collections, based on several selected indices and various time frames. Moreover, a sophisticated algorithm suggests unused media files to be illustrated in the respective Wikipedia articles, offering the possibility to engage the Wikimedia community and improve the impact as well as the quality of joint projects. 53 |
54 |55 | For the benefit of other GLAM institutions in the world, Wikimedia CH has decided in 2020 to stepwise share Cassandra within the Wikimedia movement and with other Wikimedia chapters in the world, by launching a global project (https://stats.wikimedia.global). 56 |
57 | 58 |
3 |
78 | {{/each}}
--------------------------------------------------------------------------------
/app/pages/views/category-network/tpl/unused-file-list.tpl:
--------------------------------------------------------------------------------
1 |
4 |
75 |
76 |
77 |
5 |
6 | {{name}}
7 |
8 |
50 |
9 |
15 | §[messages.view-on-commons]§
16 | ![§[messages.go]§](/assets/img/link-out.svg)
17 |
18 | {{#unless hideDetails}}
19 |
25 | §[messages.drilldown]§
26 |
27 | {{/unless}}
28 |
35 | §[messages.view-used-files]§
36 |
37 | {{!-- --}}
48 |
49 |
17 |
18 | {{#unless hideDetails}}
19 |
25 | §[messages.drilldown]§
26 |
27 | {{/unless}}
28 |
35 | §[messages.view-used-files]§
36 |
37 | {{!-- --}}
48 |
51 |
74 |
52 |
62 |
53 | §[messages.level]§
54 |
55 |
59 | {{group}}
60 |
61 |
63 |
73 |
64 | §[messages.files]§
65 |
66 |
70 | {{files}}
71 |
72 |
2 |
--------------------------------------------------------------------------------
/app/pages/views/dashboard-metabase/functions.js:
--------------------------------------------------------------------------------
1 | const glam = window.location.href.toString().split("/")[3];
2 | let SUBCATEGORY;
3 |
4 | function setCategoryCb(category) {
5 | SUBCATEGORY = category;
6 | }
7 |
8 | $(document).ready(function () {
9 | const baseurl = document.location.href;
10 | let h = baseurl.split("/");
11 | let cat = h[5];
12 | if (cat) {
13 | const newUrl = h.slice(0, 5).join("/");
14 | if (newUrl) {
15 | window.location = newUrl;
16 | }
17 | }
18 | setCategory(setCategoryCb);
19 | $("#download_dashboard_link").attr("href", "/api/" + glam + "/dashboard/download");
20 | $.getJSON("/api/" + glam + "/dashboard", function (res) {
21 | const iframe = document.getElementById("dashboard-metabase");
22 | iframe.src = res.iframeUrl;
23 | });
24 | });
25 |
--------------------------------------------------------------------------------
/app/pages/views/file-page/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
3 |
4 |
5 |
23 |
24 |
26 |
29 |
30 | 
31 |
32 |
38 |
39 |
40 |
41 |
64 |
65 |
86 |
87 |
88 |
92 |
96 |
100 |
104 |
108 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
--------------------------------------------------------------------------------
/app/pages/views/file-page/tpl/file-template.tpl:
--------------------------------------------------------------------------------
1 |
31 |
32 |
38 |
42 |
43 |
63 |
44 |
61 |
62 |
45 |
46 |
60 |
47 |
52 |
53 | §[messages.usage]§
48 |
49 |
50 |
51 |
54 |
59 | §[messages.views]§
55 |
56 |
57 |
58 |
2 |
--------------------------------------------------------------------------------
/app/pages/views/page-views/tpl/views.tpl:
--------------------------------------------------------------------------------
1 | {{#each files}}
2 |
3 | {{image}}
4 |
5 |
6 | {{#if thumbnail_url}}
7 |
21 |
8 | 
9 |
10 | {{/if}}
11 | 12 | 17 | {{image_name}} 18 | 19 |
20 |
22 |
30 |
31 | {{#each cats}}
32 |
40 |
41 | {{#if usage}}
42 | 23 | §[messages.part-of]§ 24 | 25 | {{cat_number}} 26 | 27 | {{cat_title}} 28 |
29 || 34 | {{cat_name}} 35 | | 36 |
43 |
55 |
56 | {{#each wikis}}
57 |
74 |
75 | {{else if recommender.length}}
76 | 44 | §[messages.used-in]§ 45 | 46 | {{usage}} 47 | 48 | §[messages.pages-of]§ 49 | 50 | {{projects}} 51 | 52 | §[messages.projects]§ 53 |
54 || 59 | 60 | {{wiki_name}} 61 | 62 | | 63 |64 | {{#each wiki_links}} 65 | 66 | {{wiki_page}} 67 | 68 | {{/each}} 69 | | 70 |
77 |
81 |
82 | {{#each recommender}}
83 |
100 |
101 | {{/if}}
102 | 78 | §[messages.related-wikidata-entities]§ 79 |
80 || 85 | 86 | {{label}} 87 | 88 | | 89 | {{#each wikis}} 90 |91 | 92 | {{site}} 93 | 94 | | 95 | {{/each}} 96 |
103 |
107 |
108 |
140 |
141 | 104 | §[messages.views-stats]§ 105 |
106 || 110 | §[messages.views-total]§ 111 | | 112 |113 | 114 | {{tot}} 115 | 116 | | 117 |
| 120 | §[messages.daily-average]§ 121 | | 122 |123 | 124 | {{av}} 125 | 126 | | 127 |
| 130 | §[messages.daily-median]§ 131 | | 132 |133 | 134 | {{median}} 135 | 136 | | 137 |
3 |
76 | {{/each}}
--------------------------------------------------------------------------------
/app/pages/views/recommender-page/recommender.tpl:
--------------------------------------------------------------------------------
1 |
4 |
74 |
75 |
5 |
6 | {{img_name_text}}
7 |
8 |
31 |
32 |
33 |
73 |
34 |
46 |
35 |
36 | §[messages.total]§
37 |
38 |
39 |
43 | {{tot}}
44 |
45 |
47 |
59 |
48 |
49 | §[messages.avg-day]§
50 |
51 |
52 |
56 | {{av}}
57 |
58 |
60 |
72 |
61 |
62 | §[messages.median]§
63 |
64 |
65 |
69 | {{median}}
70 |
71 |
2 |
--------------------------------------------------------------------------------
/app/pages/views/search-page/functions.js:
--------------------------------------------------------------------------------
1 | let page = 0;
2 | let limit = false;
3 | function search(append) {
4 | const baseUrl = window.location.href.toString();
5 | const urlSplit = baseUrl.split("/");
6 | const db = urlSplit[3];
7 | const query = urlSplit[5] && !urlSplit[5].includes("?lang") ? urlSplit[5] : "";
8 |
9 | const params = "?page=" + page + "&limit=100";
10 | const url = "/api/" + db + "/search/%25" + query + "%25" + params;
11 | if (page === 0) {
12 | $("#resultsSearch")
13 | .off("scroll")
14 | .scroll(loadMoreOnScroll.bind($("#resultsSearch")));
15 | }
16 | $("#searchFilesBar").val(decodeURI(query).replace("_", " "));
17 | // let template_source = "/views/category-network/tpl/unused-file-list-dropdown.tpl";
18 | let target = "#resultsSearch";
19 | let tpl =
20 | " {{#each files}}\n" +
21 | "
3 |
4 | §[messages.file]§:
5 |
6 |
11 |
20 | {{image_name}}
12 | 13 | 19 |
21 |
58 |
22 |
57 |
23 |
28 |
29 |
30 |
31 |
56 | \n" +
22 | ' {{file}}\n' +
23 | "
\n" +
24 | " {{/each}}";
25 |
26 | $.getJSON(url, d => {
27 | let template = Handlebars.compile(tpl);
28 | let temp = [];
29 | if (d.length === 0) {
30 | // show "no more elements"
31 | $("#resultsSearch").append('§[messages.no-more]§
'); 32 | limit = true; 33 | // remove handler 34 | $("#resultsSearch").off("scroll", loadMoreOnScroll); 35 | } else { 36 | d.forEach(file => { 37 | let url = "/" + db + "/file/" + file; 38 | temp.push({ 39 | url: url, 40 | file: cleanImageName(file.replace(/_/g, " ")) 41 | }); 42 | }); 43 | if (append) { 44 | $(target).append(template({ files: temp })); 45 | } else { 46 | $(target).html(template({ files: temp })); 47 | } 48 | } 49 | }); 50 | } 51 | 52 | function searchFiles(force) { 53 | let search = $("#searchFilesBar").val(); 54 | if (event && event.keyCode === 13) { 55 | force = true; 56 | } 57 | if (force) { 58 | if (search.length >= 3) { 59 | let db = window.location.href.split("/")[3]; 60 | search = search.replace(/\s/g, "_"); 61 | window.location.href = "/" + db + "/search/" + search; 62 | } else { 63 | $("#resultsSearchBar").popover("show"); 64 | } 65 | } 66 | } 67 | 68 | function loadMoreOnScroll() { 69 | if ( 70 | $("#resultsSearch").scrollTop() + $("#resultsSearch").innerHeight() >= 71 | $("#resultsSearch")[0].scrollHeight && 72 | !limit 73 | ) { 74 | // if reached end of div and there are more elements to load 75 | // calc new page number 76 | page++; 77 | search(true); 78 | } 79 | } 80 | 81 | $(function () { 82 | search(); 83 | setCategory(); 84 | let db = window.location.href.toString().split("/")[3]; 85 | $("#institutionId").attr("href", "/" + db); 86 | }); 87 | -------------------------------------------------------------------------------- /app/pages/views/search-page/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |
27 |
30 |
31 |
32 |
33 |
39 |
60 |
61 |
62 |
69 |
70 |
84 |
85 |
86 |
90 |
94 |
98 |
102 |
106 |
107 |
108 |
109 |
110 |
111 |
--------------------------------------------------------------------------------
/app/pages/views/templates/footer.html:
--------------------------------------------------------------------------------
1 |
10 |
--------------------------------------------------------------------------------
/app/pages/views/templates/glam-homepage.tpl:
--------------------------------------------------------------------------------
1 |
2 |
32 |
33 |
39 |
47 |
55 |
58 |
59 |
63 |
65 | §[messages.files]§
64 |
66 |
67 |
68 |
3 | 
4 |
10 |
--------------------------------------------------------------------------------
/app/pages/views/templates/glam-preview.tpl:
--------------------------------------------------------------------------------
1 |
5 |
9 | 6 | {{title}} 7 |
8 |
2 |
--------------------------------------------------------------------------------
/app/pages/views/templates/mobile-header.html:
--------------------------------------------------------------------------------
1 |
3 |
18 |
41 | 19 | {{glamFullName}} 20 | ( 21 | {{glamID}} 22 | ) 23 |
24 |25 | 26 | §[admin.cat]§: 27 | 28 | 29 | {{glamCategory}} 30 | 31 |
32 | {{#if lastrun}} 33 |34 | 35 | §[admin.lastrun]§: 36 | 37 | {{lastrun}} 38 |
39 | {{/if}} 40 |
42 |
43 |
44 |
45 |
2 | 
3 |
4 |
5 |
3 |
6 |
7 |
--------------------------------------------------------------------------------
/app/pages/views/templates/mobile-sidebar.html:
--------------------------------------------------------------------------------
1 |
28 |
--------------------------------------------------------------------------------
/app/pages/views/templates/secondary-sidebar.html:
--------------------------------------------------------------------------------
1 |
4 |
--------------------------------------------------------------------------------
/app/pages/views/templates/sidebar.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | ![§[messages.glam-tool]§](/assets/img/glam-logo.svg)
4 |
5 |
6 |
9 | §[messages.about-project]§
16 |
23 |
35 |
4 |
5 |
36 |
37 |
52 |
57 |
--------------------------------------------------------------------------------
/app/pages/views/unused-files-page/categories.tpl:
--------------------------------------------------------------------------------
1 | {{#each nodes}}
2 |
38 |
39 | 
40 |
41 |
3 |
46 | {{/each}}
--------------------------------------------------------------------------------
/app/pages/views/unused-files-page/functions.js:
--------------------------------------------------------------------------------
1 | const glam = window.location.href.toString().split("/")[3];
2 | const db = window.location.href.toString().split("/")[3];
3 | const query = window.location.href.toString().split("/")[5];
4 | const unused = window.location.href.toString().split("/")[6] === 'unused' ? 'true' : 'false';
5 | let page = 0;
6 | let ACTIVE_ITEM_ID;
7 | let SORT_BY = "by_name";
8 | let limit = true;
9 |
10 | if (unused === 'false') {
11 | $('#page-title').text('§[messages.used-files]§');
12 | }
13 |
14 | function getUnusedUrl() {
15 | let queryS = `?unused=${unused}`;
16 | if (query) {
17 | queryS = `?unused=${unused}&cat=${query}`;
18 | }
19 | return "/api/" + db + "/category" + queryS;
20 | }
21 |
22 | function highlight() {
23 | if (ACTIVE_ITEM_ID !== undefined) {
24 | $("#" + ACTIVE_ITEM_ID)
25 | .closest(".list_item")
26 | .addClass("list_item_active");
27 | showUnusedFilesItem();
28 | }
29 | $(".list_item").on("click", function () {
30 | let element = $(this).find(".id").attr("id");
31 | if ($(this).hasClass("list_item_active")) {
32 | // reset
33 | resetHighlighted();
34 | ACTIVE_ITEM_ID = undefined;
35 | } else {
36 | // reset
37 | resetHighlighted();
38 | // highlight item
39 | $(this).addClass("list_item_active");
40 | ACTIVE_ITEM_ID = element;
41 | showUnusedFilesItem();
42 | }
43 | });
44 | }
45 |
46 | function resetHighlighted() {
47 | // reset item highlight
48 | $(".list_item").removeClass("list_item_active");
49 | $(".viewFiles").addClass("hiddenBtn");
50 | hideUnusedFilesItem();
51 | }
52 |
53 | function sorting_table() {
54 | $("#by_total").on("click", function () {
55 | if ($("#by_total").hasClass("active_order")) {
56 | //console.log("già selezionato");
57 | } else {
58 | $("#by_level").toggleClass("active_order");
59 | $("#by_total").toggleClass("active_order");
60 | getCategories("desc_order");
61 | $("#by_total").css("cursor", "default");
62 | $("#by_level").css("cursor", "pointer");
63 | }
64 | });
65 |
66 | $("#by_level").on("click", function () {
67 | if ($("#by_level").hasClass("active_order")) {
68 | //console.log("già selezionato")
69 | } else {
70 | $("#by_level").toggleClass("active_order");
71 | $("#by_total").toggleClass("active_order");
72 | $("#by_level").css("cursor", "default");
73 | $("#by_total").css("cursor", "pointer");
74 | getCategories("by_name");
75 | }
76 | });
77 | }
78 |
79 | function unusedFilesLink(id, size, pageSel) {
80 | const queryS = `?unused=${unused}&limit=${size}&page=${pageSel}`;
81 | return "/api/" + db + "/category/" + encodeURIComponent(id) + "/" + queryS;
82 | }
83 | function getFiles(id, target, templateSource, total) {
84 | $.get(templateSource, tpl => {
85 | $.getJSON(unusedFilesLink(id, total < 1500 ? total : 1500, 0), d => {
86 | let template = Handlebars.compile(tpl);
87 | let temp = [];
88 | d.forEach(file => {
89 | let url = "/" + glam + "/file/" + file;
90 | temp.push({
91 | url: url,
92 | file: cleanImageName(file.replace(/[-_]/g, " "))
93 | });
94 | });
95 |
96 | $(target).html(template({ files: temp }));
97 | });
98 | });
99 | }
100 |
101 | function showUnusedFilesItem() {
102 | let id = $("#" + ACTIVE_ITEM_ID).data("category");
103 | let tot = $("#" + ACTIVE_ITEM_ID).data("total");
104 | let target = "#category" + ACTIVE_ITEM_ID;
105 | let template_source = "/views/unused-files-page/unused-file-list-dropdown.tpl";
106 | getFiles(id, target, template_source, tot);
107 | $("#files" + ACTIVE_ITEM_ID).show();
108 | }
109 |
110 | function hideUnusedFilesItem() {
111 | let target = "#category" + ACTIVE_ITEM_ID;
112 | $(target).html("");
113 | $("#files" + ACTIVE_ITEM_ID).hide();
114 | }
115 |
116 | function getCategories(order) {
117 | if (order === undefined) {
118 | order = SORT_BY;
119 | } else {
120 | SORT_BY = order;
121 | }
122 | let data_source = getUnusedUrl();
123 | let target = "#resultsSearch";
124 | $.getJSON(data_source, function (d) {
125 | sortNodes(d, order);
126 | if (d.nodes.length > 0) {
127 | let template_source = "/views/unused-files-page/categories.tpl";
128 | $.get(template_source, function (tpl) {
129 | let template = Handlebars.compile(tpl);
130 | $(target).html(template(d));
131 | sorting_table();
132 | highlight();
133 | });
134 | } else {
135 | $(target).html("
4 |
43 |
44 |
45 |
5 |
11 | {{name}}
12 |
13 |
14 |
15 |
42 |
16 |
28 |
17 |
18 | §[messages.level]§
19 |
20 |
21 |
25 | {{group}}
26 |
27 |
29 |
41 |
30 |
31 | §[messages.files]§
32 |
33 |
34 |
38 | {{files}}
39 |
40 | §[messages.no-files]§
"); 136 | } 137 | }); 138 | } 139 | 140 | $(document).ready(function () { 141 | setCategory(); 142 | $("#institutionId").attr("href", "/" + db); 143 | getCategories(); 144 | }); 145 | -------------------------------------------------------------------------------- /app/pages/views/unused-files-page/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |
30 |
33 |
34 |
35 |
36 |
42 |
43 |
44 |
45 |
57 |
58 |
72 |
73 |
77 |
78 |
79 |
83 |
87 |
91 |
95 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
--------------------------------------------------------------------------------
/app/pages/views/unused-files-page/unused-file-list-dropdown.tpl:
--------------------------------------------------------------------------------
1 |
35 |
36 |
42 |
46 |
48 | §[messages.unused-files]§
47 |
49 |
56 |
50 | §[messages.categories]§
51 | §[messages.by-level]§
52 | §[messages.by-total]§
53 |
54 |
55 |
2 | {{#each files}}
3 |
--------------------------------------------------------------------------------
/app/pages/views/usage/tpl/usage.tpl:
--------------------------------------------------------------------------------
1 | {{#each files}}
2 |
4 |
5 | {{file}}
6 |
7 |
8 | {{/each}}
9 |
3 |
93 | {{/each}}
--------------------------------------------------------------------------------
/app/pages/views/user-contributions/functions.js:
--------------------------------------------------------------------------------
1 | let ACTIVE_ITEM_ID;
2 | let SUBCATEGORY;
3 |
4 | function getUrl() {
5 | const url = window.location.href.toString();
6 | const urlSplit = url.split("/");
7 | const db = urlSplit[3];
8 | const subcat = urlSplit[5] && !urlSplit[5].includes("?lang") ? urlSplit[5] : "";
9 | let groupby = $("#groupby-select").val();
10 | let query = subcat ? "?groupby=" + groupby + "&cat=" + subcat : "?groupby=" + groupby;
11 | return "/api/" + db + "/file/upload-date" + query;
12 | }
13 |
14 | function getUrlDataset() {
15 | const url = window.location.href.toString();
16 | const urlSplit = url.split("/");
17 | const db = urlSplit[3];
18 | const subcat = urlSplit[5] && !urlSplit[5].includes("?lang") ? urlSplit[5] : "";
19 | let groupby = $("#groupby-select").val();
20 | let query = subcat ? "?groupby=" + groupby + "&cat=" + subcat : "?groupby=" + groupby;
21 | return "/api/" + db + "/file/upload-date/dataset" + query;
22 | }
23 |
24 | function getFileName(subcat) {
25 | let groupby = $("#groupby-select").val();
26 | return subcat + " - user_contributions - " + groupby + "ly.csv";
27 | }
28 |
29 | function getUrlAll() {
30 | const url = window.location.href.toString();
31 | const urlSplit = url.split("/");
32 | const db = urlSplit[3];
33 | const subcat = urlSplit[5] && !urlSplit[5].includes("?lang") ? urlSplit[5] : "";
34 |
35 | let groupby = $("#groupby-select").val();
36 | let query = subcat ? "?groupby=" + groupby + "&cat=" + subcat : "?groupby=" + groupby;
37 | return "/api/" + db + "/file/upload-date-all" + query;
38 | }
39 |
40 | function pad(str, max) {
41 | str = str.toString();
42 | return str.length < max ? pad("0" + str, max) : str;
43 | }
44 |
45 | function sidebar(type) {
46 | const template_source = "/views/user-contributions/tpl/user-contributions.tpl";
47 | const target = "#right_sidebar_list";
48 |
49 | $.get(template_source, function (tpl) {
50 | $.getJSON(getUrl(), function (data) {
51 | data.forEach(function (d) {
52 | let total = 0;
53 |
54 | d.files.forEach(function (d) {
55 | total += +d.count;
56 | });
57 | d.total = total;
58 |
59 | d.user_id = d.user.replace(/\s/g, "_");
60 | });
61 |
62 | if (type === "by_num") {
63 | data = data.sort(function (a, b) {
64 | return b.total - a.total;
65 | });
66 | } else {
67 | data = data.sort(function (a, b) {
68 | if (a.user < b.user) {
69 | return -1;
70 | }
71 | if (a.user > b.user) {
72 | return 1;
73 | }
74 | return 0;
75 | });
76 | }
77 |
78 | data.forEach(function (d) {
79 | d.total = nFormatter(d.total);
80 | });
81 |
82 | let template = Handlebars.compile(tpl);
83 | $(target).html(template({ users: data }));
84 |
85 | highlight();
86 | });
87 | });
88 | }
89 |
90 | function sorting_sidebar() {
91 | let byNumEl = $("#by_num");
92 | let byNameEl = $("#by_name");
93 | byNumEl.on("click", function () {
94 | if (byNumEl.hasClass("active_order")) {
95 | //console.log("già selezionato")
96 | } else {
97 | byNameEl.toggleClass("active_order");
98 | byNumEl.toggleClass("active_order");
99 | sidebar("by_num");
100 | byNumEl.css("cursor", "default");
101 | byNameEl.css("cursor", "pointer");
102 | }
103 | });
104 |
105 | byNameEl.on("click", function () {
106 | if (byNameEl.hasClass("active_order")) {
107 | //console.log("già selezionato")
108 | } else {
109 | byNameEl.toggleClass("active_order");
110 | byNumEl.toggleClass("active_order");
111 | sidebar("by_name");
112 | byNameEl.css("cursor", "default");
113 | byNumEl.css("cursor", "pointer");
114 | }
115 | });
116 | }
117 |
118 | function setCategoryCb(category) {
119 | SUBCATEGORY = category;
120 | download(category);
121 | }
122 |
123 | function download(category) {
124 | // remove old link
125 | $("#download_dataset a").remove();
126 | // recreate download link based on timespan
127 | $(
128 | '§[messages.download-dataset]§'
133 | ).appendTo("#download_dataset");
134 | }
135 |
136 | function highlight() {
137 | if (ACTIVE_ITEM_ID !== undefined) {
138 | $("#" + ACTIVE_ITEM_ID)
139 | .closest(".list_item")
140 | .addClass("list_item_active");
141 | }
142 |
143 | $(".list_item").on("click", function () {
144 | let element = $(this).find(".item").attr("id");
145 |
146 | // highlight Sidebar and show bars
147 | if ($(this).hasClass("list_item_active")) {
148 | hideUserContributionsBars();
149 | $(".list_item").removeClass("list_item_active");
150 | ACTIVE_ITEM_ID = undefined;
151 | } else {
152 | showUserContributionsBars(element);
153 | $(".list_item").removeClass("list_item_active");
154 | ACTIVE_ITEM_ID = element;
155 | $(this).addClass("list_item_active");
156 | }
157 | });
158 | }
159 |
160 | $(document).ready(function () {
161 | setCategory(setCategoryCb);
162 | sidebar("by_num");
163 | dataviz();
164 | how_to_read();
165 | switch_page();
166 | sorting_sidebar();
167 | $("#groupby-select").change(function () {
168 | dataviz();
169 | download(SUBCATEGORY);
170 | });
171 | });
172 |
--------------------------------------------------------------------------------
/app/pages/views/user-contributions/tpl/user-contributions.tpl:
--------------------------------------------------------------------------------
1 | {{#each users}}
2 |
4 |
61 |
5 |
6 | {{image_name}}
7 |
8 |
31 |
32 |
33 |
60 |
34 |
46 |
35 |
36 | §[messages.usage]§
37 |
38 |
39 |
43 | {{usage}}
44 |
45 |
47 |
59 |
48 |
49 | §[messages.projects]§
50 |
51 |
52 |
56 | {{projects}}
57 |
58 |
62 |
92 |
63 |
91 |
64 |
65 |
66 | {{#each wikis}}
67 |
89 |
90 | | 69 | 72 | {{wiki_name}} 73 | 74 | | 75 |76 | {{#each wiki_links}} 77 | 81 | {{wiki_page}} 82 | 83 | {{/each}} 84 | | 85 |
3 |
38 | {{/each}}
--------------------------------------------------------------------------------
/app/server.js:
--------------------------------------------------------------------------------
1 | var express = require('express');
2 | var apicache = require('apicache').options({ debug: false }).middleware;
3 | var morgan = require('morgan');
4 | var cookieParser = require('cookie-parser');
5 | var Sentry = require('@sentry/node');
6 | var config = require('../config/config.json');
7 |
8 | var app = express();
9 |
10 | if (typeof config.raven !== 'undefined') {
11 | Sentry.init({dsn: config.raven.glamtoolsweb.DSN});
12 | app.use(Sentry.Handlers.requestHandler());
13 | }
14 |
15 | app.use(morgan('common'));
16 | app.use(express.raw({type: 'image/svg+xml', limit: '1mb'}));
17 | app.use(express.json());
18 | app.use(cookieParser());
19 |
20 | require('./routes.js')(app, apicache);
21 |
22 | if (typeof config.raven !== 'undefined') {
23 | app.use(Sentry.Handlers.errorHandler());
24 | }
25 |
26 | var port = process.argv[2] ? parseInt(process.argv[2]) : 8081;
27 |
28 | var server = app.listen(port, function() {
29 | var host = server.address().address;
30 | var port = server.address().port;
31 | console.log('Server listening at http://%s:%s', host, port);
32 | });
33 |
--------------------------------------------------------------------------------
/config/config.example.json:
--------------------------------------------------------------------------------
1 | {
2 | "postgres": {
3 | "user": "cassandra",
4 | "password": "MYPOSTGRESPASSWORD",
5 | "host": "localhost",
6 | "port": 5432
7 | },
8 | "metabase": {
9 | "url": "http://localhost:3000",
10 | "proxy": "http://localhost:8081",
11 | "username": "",
12 | "password": "",
13 | "database": {
14 | "user": "metabase",
15 | "password": "MYPOSTGRESPASSWORD",
16 | "host": "localhost",
17 | "port": 5432
18 | },
19 | "secret": ""
20 | },
21 | "mongodb": {
22 | "url": "mongodb://localhost:27017",
23 | "database": "cassandra",
24 | "collection": "glams"
25 | },
26 | "wmflabs": {
27 | "host": "localhost",
28 | "port": 3306,
29 | "user": "MYWMFUSER",
30 | "password": "MYWMFPASSWORD",
31 | "database": "commonswiki_p"
32 | },
33 | "admin": {
34 | "username": "admin",
35 | "password": "MYADMINPASSWORD"
36 | },
37 | "glamUser": {
38 | "users": [
39 | {
40 | "username": "glam",
41 | "password": "MYUSERPASSWORD"
42 | }
43 | ]
44 | },
45 | "limits": {
46 | "categories": 1000,
47 | "images": 500000
48 | },
49 | "mailgun": {
50 | "auth": {
51 | "api_key": "",
52 | "domain": ""
53 | },
54 | "host": "api.eu.mailgun.net",
55 | "mailFrom": "",
56 | "mailTo": ""
57 | },
58 | "recaptcha": {
59 | "siteKey": "",
60 | "secretKey": ""
61 | }
62 | }
63 |
--------------------------------------------------------------------------------
/config/config.js:
--------------------------------------------------------------------------------
1 | var MongoClient = require('mongodb').MongoClient;
2 | var { Pool, Client } = require('pg');
3 | var fs = require('fs');
4 |
5 | var config = JSON.parse(fs.readFileSync("../config/config.json"));
6 |
7 | config.glamUser['realm'] = 'User area';
8 | var glamUser = config.glamUser;
9 | glamUser.users.push(config.admin);
10 | config.admin['realm'] = 'Admin area';
11 |
12 | exports.admin = config.admin;
13 | exports.glamUser = glamUser;
14 | exports.limits = config.limits;
15 | exports.wmflabs = config.wmflabs;
16 | exports.metabase = config.metabase;
17 | exports.recaptcha = config.recaptcha;
18 |
19 | const client = new MongoClient(config['mongodb']['url'], { useNewUrlParser: true });
20 |
21 | var glams = {};
22 |
23 | function isConnected(client) {
24 | return !!client && !!client.topology && client.topology.isConnected();
25 | }
26 |
27 | function loadGlams(callback) {
28 | let query = () => {
29 | const db = client.db(config['mongodb']['database']);
30 | const collection = db.collection(config['mongodb']['collection']);
31 | collection.find({}).toArray((err, docs) => {
32 | docs.forEach((element) => {
33 | let glam = {
34 | 'name': element['name'],
35 | 'fullname': element['fullname'],
36 | 'category': element['category'],
37 | 'image': element['image'],
38 | connection: new Pool({
39 | 'user': config['postgres']['user'],
40 | 'password': config['postgres']['password'],
41 | 'host': config['postgres']['host'],
42 | 'port': config['postgres']['port'],
43 | 'database': element['database']
44 | })
45 | };
46 |
47 | if (element['lastrun']) {
48 | glam['lastrun'] = element['lastrun'];
49 | } else {
50 | glam['lastrun'] = null;
51 | }
52 |
53 | if (element['status']) {
54 | glam['status'] = element['status'];
55 | } else {
56 | glam['status'] = null;
57 | }
58 |
59 | if (element['dashboard_id']) {
60 | glam['dashboard_id'] = element['dashboard_id'];
61 | } else {
62 | glam['dashboard_id'] = null;
63 | }
64 |
65 | if (element['http-auth']) {
66 | glam['http-auth'] = element['http-auth'];
67 | glam['http-auth']['realm'] = element['name'] + " stats";
68 | }
69 |
70 | // Glams are never deleted
71 | glams[glam['name']] = glam;
72 | });
73 |
74 | if (callback !== undefined)
75 | callback();
76 | });
77 | };
78 |
79 | if (!isConnected(client)) {
80 | client.connect(query);
81 | } else {
82 | query();
83 | }
84 | }
85 |
86 | function insertGlam(glam) {
87 | let query = () => {
88 | const db = client.db(config['mongodb']['database']);
89 | const collection = db.collection(config['mongodb']['collection']);
90 | collection.insertOne(glam, () => {
91 | loadGlams();
92 | });
93 | };
94 |
95 | if (!isConnected(client)) {
96 | client.connect(query);
97 | } else {
98 | query();
99 | }
100 | }
101 |
102 | function updateGlam(glam) {
103 | let query = () => {
104 | const db = client.db(config['mongodb']['database']);
105 | const collection = db.collection(config['mongodb']['collection']);
106 | collection.updateOne({ name: glam['name'] }, { $set: glam }, () => {
107 | loadGlams();
108 | });
109 | };
110 |
111 | if (!isConnected(client)) {
112 | client.connect(query);
113 | } else {
114 | query();
115 | }
116 | }
117 |
118 | exports.glams = glams;
119 | exports.loadGlams = loadGlams;
120 | exports.insertGlam = insertGlam;
121 | exports.updateGlam = updateGlam;
--------------------------------------------------------------------------------
/deploy/.gitignore:
--------------------------------------------------------------------------------
1 | id_rsa_cassandra
2 | inventory.ini
--------------------------------------------------------------------------------
/deploy/cassandra-autossh.service:
--------------------------------------------------------------------------------
1 | [Unit]
2 | Description=AutoSSH for Cassandra
3 |
4 | [Service]
5 | User=glam
6 | Group=glam
7 | Restart=always
8 | RestartSec=15s
9 | ExecStart=/usr/bin/autossh -N wmflabs
10 |
11 | [Install]
12 | WantedBy=multi-user.target
13 |
--------------------------------------------------------------------------------
/deploy/cassandra.conf:
--------------------------------------------------------------------------------
1 | [program:cassandra]
2 | directory=/home/glam/cassandra-GLAM-tools/app
3 | command=node server.js
4 | priority=10
5 | exitcodes=0,2
6 | stopsignal=QUIT
7 | user=glam
8 | log_stdout=true
9 | log_stderr=true
10 | logfile_maxbytes=10MB
11 | logfile_backups=10
--------------------------------------------------------------------------------
/deploy/config:
--------------------------------------------------------------------------------
1 | Host wmflabs
2 | HostName login.toolforge.org
3 | User MYWMFLOGIN
4 | Port 22
5 | IdentityFile ~/.ssh/id_rsa_cassandra
6 | LocalForward 3306 commonswiki.analytics.db.svc.wikimedia.cloud:3306
7 |
--------------------------------------------------------------------------------
/deploy/crontab:
--------------------------------------------------------------------------------
1 | # GLAM tool crontab
2 | */5 * * * * glam python3 /home/glam/cassandra-GLAM-tools/etl/run.py >> /var/log/cassandra/etl.log 2>&1
3 | 0 18 * * * glam python3 /home/glam/cassandra-GLAM-tools/etl/run_views.py >> /var/log/cassandra/views.log 2>&1
4 | 30 3 * * 6 glam python3 /home/glam/cassandra-GLAM-tools/recommender/run.py >> /var/log/cassandra/recommender.log 2>&1
5 |
--------------------------------------------------------------------------------
/deploy/metabase.conf:
--------------------------------------------------------------------------------
1 | [program:metabase]
2 | directory=/home/glam/metabase
3 | command=java -jar metabase.jar
4 | priority=20
5 | exitcodes=0,2
6 | stopsignal=QUIT
7 | user=glam
8 | log_stdout=true
9 | log_stderr=true
10 | logfile_maxbytes=10MB
11 | logfile_backups=10
--------------------------------------------------------------------------------
/deploy/pontoon.conf:
--------------------------------------------------------------------------------
1 | [program:pontoon]
2 | directory=/home/glam/pontoon
3 | command=pipenv run gunicorn pontoon.wsgi
4 | priority=20
5 | exitcodes=0,2
6 | stopsignal=QUIT
7 | user=glam
8 | log_stdout=true
9 | log_stderr=true
10 | logfile_maxbytes=10MB
11 | logfile_backups=10
--------------------------------------------------------------------------------
/deploy/pontoon.env:
--------------------------------------------------------------------------------
1 | SECRET_KEY=MYRANDOMKEY
2 | DATABASE_URL=postgres://pontoon:MYPOSTGRESPASSWORD@localhost/pontoon
3 | SITE_URL=https://host.example.com
--------------------------------------------------------------------------------
/deploy/pontoon.yml:
--------------------------------------------------------------------------------
1 | ---
2 | - hosts: pontoon
3 | remote_user: root
4 | vars:
5 | install_path: /home/glam/pontoon
6 | postgres_password: supersecret
7 |
8 | tasks:
9 | - name: Create user glam
10 | user:
11 | name: glam
12 | shell: /bin/bash
13 | password: '!'
14 |
15 | - name: Update apt cache
16 | apt: update_cache=yes
17 |
18 | - name: Install dependencies
19 | package:
20 | name: "{{ item }}"
21 | state: present
22 | loop:
23 | - apt-transport-https
24 | - gnupg2
25 | - git
26 | - python3-pip
27 | - postgresql-12
28 | - autossh
29 | - supervisor
30 | - libxml2-dev
31 | - libxslt1-dev
32 | - python3-dev
33 | - libmemcached-dev
34 | - build-essential
35 | - libpq-dev
36 |
37 | - name: Add NodeSource Node.js repository
38 | shell: "curl -fsSL https://deb.nodesource.com/setup_12.x | bash -"
39 |
40 | - name: Update apt cache
41 | apt: update_cache=yes
42 |
43 | - name: Install Node.js
44 | package:
45 | name: nodejs
46 | state: present
47 |
48 | - name: Install Pipenv
49 | shell: "pip3 install pipenv"
50 |
51 | - name: Download Pontoon
52 | git:
53 | repo: "https://github.com/mozilla/pontoon.git"
54 | dest: "{{ install_path }}"
55 | become: yes
56 | become_user: glam
57 |
58 | - name: Install Python dependencies
59 | command: "pipenv run pip install -r requirements.txt"
60 | args:
61 | chdir: "{{ install_path }}"
62 | become: yes
63 | become_user: glam
64 |
65 | - name: Install Node.js dependencies
66 | command: "npm install"
67 | args:
68 | chdir: "{{ install_path }}"
69 | become: yes
70 | become_user: glam
71 |
72 | - name: Install Node.js frontend dependencies
73 | command: "npm install"
74 | args:
75 | chdir: "{{ install_path }}/frontend"
76 | become: yes
77 | become_user: glam
78 |
79 | - name: Run webpack
80 | command: "./node_modules/.bin/webpack"
81 | args:
82 | chdir: "{{ install_path }}"
83 | become: yes
84 | become_user: glam
85 |
86 | - name: Build Node.js frontend dependencies
87 | command: "npm run build"
88 | args:
89 | chdir: "{{ install_path }}/frontend"
90 | become: yes
91 | become_user: glam
92 |
93 | - name: Create PostgreSQL user
94 | command: psql -c "CREATE USER pontoon WITH PASSWORD '{{ postgres_password }}' SUPERUSER;"
95 | become: yes
96 | become_user: postgres
97 |
98 | - name: Create PostgreSQL database
99 | command: psql -c "CREATE DATABASE pontoon;"
100 | become: yes
101 | become_user: postgres
102 |
103 | - name: Create PostgreSQL privileges
104 | command: psql -c "GRANT ALL PRIVILEGES ON DATABASE pontoon to pontoon;"
105 | become: yes
106 | become_user: postgres
107 |
108 | - name: Copy env file
109 | copy:
110 | src: pontoon.env
111 | dest: "{{ install_path }}/.env"
112 | become: yes
113 | become_user: glam
114 |
115 | - name: Set PostgreSQL password
116 | replace:
117 | path: "{{ install_path }}/.env"
118 | regexp: 'MYPOSTGRESPASSWORD'
119 | replace: "{{ postgres_password }}"
120 |
121 | - name: Set random key
122 | replace:
123 | path: "{{ install_path }}/.env"
124 | regexp: 'MYRANDOMKEY'
125 | replace: "{{ lookup('password', '/dev/null chars=ascii_lowercase,digits length=32') }}"
126 |
127 | - name: Move static files
128 | command: "pipenv run python manage.py collectstatic --noinput"
129 | args:
130 | chdir: "{{ install_path }}"
131 | become: yes
132 | become_user: glam
133 |
134 | - name: Run migrations
135 | command: "pipenv run python manage.py migrate"
136 | args:
137 | chdir: "{{ install_path }}"
138 | become: yes
139 | become_user: glam
140 |
141 | - name: Copy supervisor config
142 | copy:
143 | src: pontoon.conf
144 | dest: /etc/supervisor/conf.d/pontoon.conf
145 |
146 | - name: Restart supervisor
147 | shell: "supervisorctl reread && supervisorctl update"
148 |
--------------------------------------------------------------------------------
/docs/architecture.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/synapta/cassandra-GLAM-tools/613cc88f0570a09146e1d4e70ab4509050280139/docs/architecture.png
--------------------------------------------------------------------------------
/docs/presentations/2016-09-01-How-libraries-fall-in-love-with-Wikidata.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/synapta/cassandra-GLAM-tools/613cc88f0570a09146e1d4e70ab4509050280139/docs/presentations/2016-09-01-How-libraries-fall-in-love-with-Wikidata.pdf
--------------------------------------------------------------------------------
/docs/presentations/2017-11-08-Cassandra_Backend.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/synapta/cassandra-GLAM-tools/613cc88f0570a09146e1d4e70ab4509050280139/docs/presentations/2017-11-08-Cassandra_Backend.pdf
--------------------------------------------------------------------------------
/docs/presentations/2019-11-03-Wikidata_Zurich_Training.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/synapta/cassandra-GLAM-tools/613cc88f0570a09146e1d4e70ab4509050280139/docs/presentations/2019-11-03-Wikidata_Zurich_Training.pdf
--------------------------------------------------------------------------------
/etl/.gitignore:
--------------------------------------------------------------------------------
1 | temp/*
2 |
--------------------------------------------------------------------------------
/etl/SQL/dailyInsert.sql:
--------------------------------------------------------------------------------
1 | create or replace function dailyInsert(text, date,integer, integer, integer)
2 | returns integer as $$
3 | declare id integer;
4 | begin
5 | select media_id into id from images where img_name=$1;
6 | insert into visualizations values(id,$2,$3,$4,$5);
7 | return id;
8 | EXCEPTION
9 | WHEN OTHERS THEN RETURN -1;
10 | end;
11 | $$language plpgsql;
12 |
--------------------------------------------------------------------------------
/etl/SQL/db_init.sql:
--------------------------------------------------------------------------------
1 | CREATE table IF NOT EXISTS categories (page_title varchar(255) primary key, cat_subcats int, cat_files int, cl_to varchar(255)[], cat_level int[]);
2 | CREATE table IF NOT EXISTS images (img_name varchar(255) primary key, img_user_text varchar(255), img_timestamp timestamp without time zone,img_size bigint,cl_to varchar(255)[],media_id serial unique,is_alive boolean);
3 | CREATE table IF NOT EXISTS visualizations (media_id int references images(media_id), access_date date, accesses bigint, wm_accesses bigint, nwm_accesses bigint, primary key(media_id,access_date));
4 | CREATE table IF NOT EXISTS annotations (annotation_date date primary key, annotation_value varchar(255), annotation_position varchar(255));
5 | CREATE table IF NOT EXISTS usages (gil_wiki varchar(20),gil_page_title varchar(255),gil_to varchar(255), first_seen date,last_seen date, is_alive boolean, primary key(gil_to,gil_page_title,first_seen,gil_wiki));
6 | CREATE INDEX IF NOT EXISTS ad_GBy on visualizations(access_date);
7 | create materialized view if not exists visualizations_sum as select sum(accesses) as accesses_sum, access_date from visualizations group by access_date;
8 | create materialized view if not exists visualizations_stats as select i.img_name as img_name, sum(v.accesses) as tot, avg(v.accesses) as avg, PERCENTILE_CONT(0.5) WITHIN GROUP(ORDER by v.accesses) as median from images as i, visualizations as v where i.media_id = v.media_id and i.is_alive = true group by i.img_name;
9 | CREATE TABLE IF NOT EXISTS recommendations (img_name varchar(255) NOT NULL, site varchar(20) NOT NULL, title varchar(255) NOT NULL, url varchar(255) NOT NULL, score float4 NULL, last_update date NOT NULL, hidden bool NOT NULL DEFAULT false, CONSTRAINT recommendations_fk FOREIGN KEY (img_name) REFERENCES images(img_name));
10 | CREATE UNIQUE INDEX recommendations_img_name_idx ON recommendations USING btree (img_name, site, title);
11 | GRANT USAGE ON SCHEMA public TO metabase;
12 | GRANT SELECT ON ALL TABLES IN SCHEMA public TO metabase;
--------------------------------------------------------------------------------
/etl/SQL/functions.sql:
--------------------------------------------------------------------------------
1 | create or replace function AddCategory(title varchar(255),subcats int ,files int,_to varchar(255),_level int)
2 | returns void as $$
3 | declare
4 | t varchar(255)[];
5 | l int[];
6 | begin
7 | IF EXISTS (SELECT 1 FROM categories WHERE page_title = title) THEN
8 | update categories set cl_to=cl_to||_to,cat_level=cat_level||_level where page_title=title;
9 | else
10 | t[0]:=_to;
11 | l[0]:=_level;
12 | insert into categories values(title,subcats,files,t,l);
13 | end if;
14 | EXCEPTION
15 | WHEN others THEN
16 | RAISE NOTICE 'Error on %',title;
17 | end;
18 | $$language plpgsql;
19 |
20 | create or replace function AddImage(_name varchar(255),_user varchar(255) ,_timestamp timestamp without time zone,size bigint,_to varchar(255))
21 | returns void as $$
22 | declare
23 | t varchar(255)[];
24 | opened boolean;
25 | begin
26 | SELECT is_alive into opened FROM images WHERE img_name = _name;
27 | t[0]:=_to;
28 | IF FOUND THEN
29 | if opened then
30 | update images set cl_to=cl_to||_to where img_name=_name;
31 | else
32 | update images set cl_to=t, is_alive=true where img_name=_name;
33 | end if;
34 | else
35 | insert into images(img_name,img_user_text,img_timestamp,img_size,cl_to,is_alive) values(_name,_user,_timestamp,size,t,true);
36 | end if;
37 | EXCEPTION
38 | WHEN others THEN
39 | RAISE NOTICE 'Error on %',_name;
40 | end;
41 | $$language plpgsql;
42 |
43 |
44 | create or replace function AddUsage(_wiki varchar(20),_page varchar(255) ,_to varchar(255))
45 | returns void as $$
46 | declare
47 | appeared date;
48 | begin
49 | SELECT first_seen into appeared FROM usages WHERE gil_page_title = _page and gil_to=_to and gil_wiki=_wiki and is_alive=true;
50 | IF FOUND THEN
51 | update usages set last_seen=CURRENT_DATE where gil_page_title=_page and gil_to=_to and gil_wiki=_wiki and first_seen=appeared;
52 | else
53 | insert into usages(gil_wiki,gil_page_title,gil_to,first_seen,last_seen,is_alive) values(_wiki,_page,_to,CURRENT_DATE,CURRENT_DATE,true);
54 | end if;
55 | end;
56 | $$language plpgsql;
--------------------------------------------------------------------------------
/etl/SQL/maintenance.sql:
--------------------------------------------------------------------------------
1 | create table if not exists dailyImageUsage(img_name varchar(255),count_date date, count bigint, primary key (img_name, count_date));
2 | grant select on dailyimageusage to metabase;
3 | create index if not exists dailyimageusage_img_name_idx on dailyimageusage (img_name);
4 | create or replace function doMaintenance()
5 | returns void as $$
6 | begin
7 | insert into dailyImageUsage(img_name, count_date, count) select gil_to, CURRENT_DATE, count(*) from usages where last_seen=current_date group by gil_to on conflict do nothing;
8 | update usages set is_alive=false where last_seen<(current_date-5);
9 | end;
10 | $$language plpgsql;
11 |
--------------------------------------------------------------------------------
/etl/cards/10.json:
--------------------------------------------------------------------------------
1 | {"description": "§[metabase.corr-old-desc]§", "archived": false, "collection_position": null, "table_id": null, "result_metadata": [{"base_type": "type/Float", "display_name": "Corr Coef Using PGSQL Func", "name": "Corr Coef Using PGSQL Func", "special_type": null, "fingerprint": {"global": {"distinct-count": 1, "nil%": 0.0}, "type": {"type/Number": {"min": 0.0776270546580604, "max": 0.0776270546580604, "avg": 0.0776270546580604, "sd": null, "q1": 0.0776270546580604, "q3": 0.0776270546580604}}}}], "creator": {"email": "alessio@synapta.it", "first_name": "Alessio", "last_login": "2021-01-25T09:56:39.29734Z", "is_qbnewb": false, "is_superuser": true, "id": 2, "last_name": "Melandri", "date_joined": "2020-05-19T09:02:11.385428Z", "common_name": "Alessio Melandri"}, "can_write": true, "database_id": 3, "enable_embedding": false, "collection_id": 16, "query_type": "native", "name": "§[& metabase.corr-old]§", "dashboard_count": 1, "read_permissions": null, "creator_id": 2, "updated_at": "2021-05-23T06:12:44.313438Z", "made_public_by_id": null, "embedding_params": null, "cache_ttl": null, "dataset_query": {"type": "native", "native": {"query": "SELECT corr(average, oldness) as \"Corr Coef Using PGSQL Func\" FROM(\nSELECT AVG(accesses) as average, EXTRACT(day from NOW()-img_timestamp) as oldness\nFROM images i, visualizations v\nWHERE i.media_id = v.media_id\ngroup by i.img_name\n) as a\n\n \n"}, "database": 3}, "id": 10, "display": "scalar", "visualization_settings": {"graph.dimensions": ["average"], "graph.metrics": ["img_size"], "graph.x_axis.scale": "pow", "graph.y_axis.scale": "pow"}, "collection": {"id": 16, "name": "Correlations", "description": null, "color": "#509EE3", "archived": false, "location": "/7/", "personal_owner_id": null, "slug": "correlations"}, "created_at": "2020-06-03T14:04:18.236465Z", "public_uuid": null}
--------------------------------------------------------------------------------
/etl/cards/18.json:
--------------------------------------------------------------------------------
1 | {"description": "§[metabase.uploaded-last-desc]§", "archived": false, "collection_position": null, "table_id": null, "result_metadata": [{"base_type": "type/Decimal", "display_name": "?column?", "name": "?column?", "special_type": null, "fingerprint": {"global": {"distinct-count": 1, "nil%": 0.0}, "type": {"type/Number": {"min": 0.03210463733650416, "max": 0.03210463733650416, "avg": 0.03210463733650416, "sd": null, "q1": 0.03210463733650416, "q3": 0.03210463733650416}}}}], "creator": {"email": "alessio@synapta.it", "first_name": "Alessio", "last_login": "2021-01-25T09:56:39.29734Z", "is_qbnewb": false, "is_superuser": true, "id": 2, "last_name": "Melandri", "date_joined": "2020-05-19T09:02:11.385428Z", "common_name": "Alessio Melandri"}, "can_write": true, "database_id": 3, "enable_embedding": false, "collection_id": 14, "query_type": "native", "name": "§[metabase.uploaded-last]§", "dashboard_count": 1, "read_permissions": null, "creator_id": 2, "updated_at": "2021-05-23T06:12:41.572739Z", "made_public_by_id": null, "embedding_params": null, "cache_ttl": null, "dataset_query": {"type": "native", "native": {"query": "SELECT (COUNT(*) FILTER(WHERE img_timestamp > (now() - interval '1 year')))/COUNT(*)::numeric\nFROM images"}, "database": 3}, "id": 18, "display": "scalar", "visualization_settings": {"column_settings": {"[\"name\",\"?column?\"]": {"scale": 100, "suffix": "%", "prefix": "+"}}}, "collection": {"id": 14, "name": "Upload", "description": null, "color": "#509EE3", "archived": false, "location": "/7/", "personal_owner_id": null, "slug": "upload"}, "created_at": "2020-12-07T17:34:53.572179Z", "public_uuid": null}
--------------------------------------------------------------------------------
/etl/cards/19.json:
--------------------------------------------------------------------------------
1 | {"description": "§[metabase.perc-used-desc]§", "archived": false, "collection_position": null, "table_id": null, "result_metadata": [{"base_type": "type/Decimal", "display_name": "?column?", "name": "?column?", "special_type": null, "fingerprint": {"global": {"distinct-count": 1, "nil%": 0.0}, "type": {"type/Number": {"min": 0.45422116527942924, "max": 0.45422116527942924, "avg": 0.45422116527942924, "sd": null, "q1": 0.45422116527942924, "q3": 0.45422116527942924}}}}], "creator": {"email": "alessio@synapta.it", "first_name": "Alessio", "last_login": "2021-01-25T09:56:39.29734Z", "is_qbnewb": false, "is_superuser": true, "id": 2, "last_name": "Melandri", "date_joined": "2020-05-19T09:02:11.385428Z", "common_name": "Alessio Melandri"}, "can_write": true, "database_id": 3, "enable_embedding": false, "collection_id": 13, "query_type": "native", "name": "§[metabase.perc-used]§", "dashboard_count": 1, "read_permissions": null, "creator_id": 2, "updated_at": "2021-05-23T06:12:41.34413Z", "made_public_by_id": null, "embedding_params": null, "cache_ttl": null, "dataset_query": {"type": "native", "native": {"query": "SELECT COUNT(DISTINCT gil_to)/(SELECT COUNT(*) from images)::numeric\nFROM usages"}, "database": 3}, "id": 19, "display": "scalar", "visualization_settings": {"column_settings": {"[\"name\",\"?column?\"]": {"scale": 100, "suffix": "%"}}}, "collection": {"id": 13, "name": "Usage", "description": null, "color": "#509EE3", "archived": false, "location": "/7/", "personal_owner_id": null, "slug": "usage"}, "created_at": "2020-12-07T17:38:52.198419Z", "public_uuid": null}
--------------------------------------------------------------------------------
/etl/cards/20.json:
--------------------------------------------------------------------------------
1 | {"description": "§[metabase.views-change-desc]§", "archived": false, "collection_position": null, "table_id": null, "result_metadata": [{"base_type": "type/Decimal", "display_name": "?column?", "name": "?column?", "special_type": null, "fingerprint": {"global": {"distinct-count": 1, "nil%": 0.0}, "type": {"type/Number": {"min": -0.006787059042099267, "max": -0.006787059042099267, "avg": -0.006787059042099267, "sd": null, "q1": -0.006787059042099267, "q3": -0.006787059042099267}}}}], "creator": {"email": "alessio@synapta.it", "first_name": "Alessio", "last_login": "2021-01-25T09:56:39.29734Z", "is_qbnewb": false, "is_superuser": true, "id": 2, "last_name": "Melandri", "date_joined": "2020-05-19T09:02:11.385428Z", "common_name": "Alessio Melandri"}, "can_write": true, "database_id": 3, "enable_embedding": false, "collection_id": 12, "query_type": "native", "name": "§[metabase.views-change]§", "dashboard_count": 1, "read_permissions": null, "creator_id": 2, "updated_at": "2021-05-23T06:12:46.273915Z", "made_public_by_id": null, "embedding_params": null, "cache_ttl": null, "dataset_query": {"database": 3, "native": {"query": "with a as (\n select sum(accesses) prev_year\n from visualizations\n where date_trunc('year', access_date) = date_trunc('year', now() - interval '2 year')\n group by date_trunc('year', access_date)\n), b as (\n select sum(accesses) as current_year\n from visualizations\n where date_trunc('year', access_date) = date_trunc('year', now() - interval '1 year')\n group by date_trunc('year', access_date)\n)\nselect (current_year- prev_year)/prev_year::numeric\nfrom a,b"}, "type": "native"}, "id": 20, "display": "scalar", "visualization_settings": {"column_settings": {"[\"name\",\"?column?\"]": {"scale": 100, "suffix": "%"}}}, "collection": {"id": 12, "name": "§[metabase.views]§", "description": null, "color": "#509EE3", "archived": false, "location": "/7/", "personal_owner_id": null, "slug": "views"}, "created_at": "2020-12-07T17:54:02.190531Z", "public_uuid": null}
--------------------------------------------------------------------------------
/etl/cards/37.json:
--------------------------------------------------------------------------------
1 | {"description": "§[metabase.total-views-desc]§", "archived": false, "collection_position": null, "table_id": null, "result_metadata": [{"base_type": "type/Decimal", "display_name": "sum", "name": "sum", "special_type": null, "fingerprint": {"global": {"distinct-count": 1, "nil%": 0.0}, "type": {"type/Number": {"min": 33151052.0, "max": 33151052.0, "avg": 33151052.0, "sd": null, "q1": 33151052.0, "q3": 33151052.0}}}}], "creator": {"email": "alessio@synapta.it", "first_name": "Alessio", "last_login": "2021-01-25T09:56:39.29734Z", "is_qbnewb": false, "is_superuser": true, "id": 2, "last_name": "Melandri", "date_joined": "2020-05-19T09:02:11.385428Z", "common_name": "Alessio Melandri"}, "can_write": true, "database_id": 3, "enable_embedding": false, "collection_id": 12, "query_type": "native", "name": "§[metabase.total-views]§", "dashboard_count": 1, "read_permissions": null, "creator_id": 2, "updated_at": "2021-05-23T06:12:44.09671Z", "made_public_by_id": null, "embedding_params": null, "cache_ttl": null, "dataset_query": {"database": 3, "native": {"query": "SELECT SUM(accesses)\nFROM visualizations"}, "type": "native"}, "id": 37, "display": "scalar", "visualization_settings": {}, "collection": {"id": 12, "name": "Views", "description": null, "color": "#509EE3", "archived": false, "location": "/7/", "personal_owner_id": null, "slug": "views"}, "created_at": "2021-01-13T09:40:38.895889Z", "public_uuid": null}
--------------------------------------------------------------------------------
/etl/cards/38.json:
--------------------------------------------------------------------------------
1 | {"description": "§[metabase.views-year-desc]§", "archived": false, "collection_position": null, "table_id": null, "result_metadata": [{"base_type": "type/DateTimeWithLocalTZ", "display_name": "§[metabase.year]§", "name": "§[metabase.year]§", "special_type": null, "fingerprint": {"global": {"distinct-count": 7, "nil%": 0.0}, "type": {"type/DateTime": {"earliest": "2015-01-01T00:00:00+01:00", "latest": "2021-01-01T00:00:00+01:00"}}}}, {"base_type": "type/Decimal", "display_name": "§[visits-per-year]§", "name": "§[metabase.visits-per-year]§", "special_type": null, "fingerprint": {"global": {"distinct-count": 7, "nil%": 0.0}, "type": {"type/Number": {"min": 1840097.0, "max": 6792721.0, "avg": 4735864.571428572, "sd": 1504320.9550392781, "q1": 4530486.5, "q3": 5409653.5}}}}, {"base_type": "type/Decimal", "display_name": "§[metabase.cumulative-usage]§", "name": "§[metabase.cumulative-usage]§", "special_type": null, "fingerprint": {"global": {"distinct-count": 7, "nil%": 0.0}, "type": {"type/Number": {"min": 6792721.0, "max": 33151052.0, "avg": 21432633.85714286, "sd": 9825542.78939938, "q1": 13519908.25, "q3": 30180265.0}}}}], "creator": {"email": "alessio@synapta.it", "first_name": "Alessio", "last_login": "2021-01-25T09:56:39.29734Z", "is_qbnewb": false, "is_superuser": true, "id": 2, "last_name": "Melandri", "date_joined": "2020-05-19T09:02:11.385428Z", "common_name": "Alessio Melandri"}, "can_write": true, "database_id": 3, "enable_embedding": false, "collection_id": 12, "query_type": "native", "name": "§[metabase.views-year]§", "dashboard_count": 1, "read_permissions": null, "creator_id": 2, "updated_at": "2021-05-23T06:12:44.105668Z", "made_public_by_id": null, "embedding_params": null, "cache_ttl": null, "dataset_query": {"database": 3, "native": {"query": "SELECT d_year as \"§[metabase.year]§\", visits_per_year as \"§[metabase.visits-per-year]§\", sum(visits_per_year) OVER (ORDER BY d_year) as \"§[metabase.cumulative-usage]§\"\nfrom (\nSELECT date_trunc('year', access_date) AS d_year, SUM(accesses) AS visits_per_year\nFROM visualizations\ngroup by d_year) as a\ngroup by d_year, visits_per_year"}, "type": "native"}, "id": 38, "display": "line", "visualization_settings": {"graph.dimensions": ["§[metabase.year]§"], "graph.metrics": ["§[metabase.visits-per-year]§", "§[metabase.cumulative-usage]§"], "graph.show_values": false, "series_settings": {"§[metabase.cumulative-usage]§": {"color": "#A989C5"}, "§[metabase.visits-per-year]§": {"color": "#509EE3"}}}, "collection": {"id": 12, "name": "§[metabase.views]§", "description": null, "color": "#509EE3", "archived": false, "location": "/7/", "personal_owner_id": null, "slug": "views"}, "created_at": "2021-01-13T20:49:13.257604Z", "public_uuid": null}
--------------------------------------------------------------------------------
/etl/cards/39.json:
--------------------------------------------------------------------------------
1 | {"description": "§[metabase.enanched-desc]§", "archived": false, "collection_position": null, "table_id": null, "result_metadata": [{"base_type": "type/BigInteger", "display_name": "count", "name": "count", "special_type": "type/Quantity", "fingerprint": {"global": {"distinct-count": 1, "nil%": 0.0}, "type": {"type/Number": {"min": 814.0, "max": 814.0, "avg": 814.0, "sd": null, "q1": 814.0, "q3": 814.0}}}}], "creator": {"email": "alessio@synapta.it", "first_name": "Alessio", "last_login": "2021-01-25T09:56:39.29734Z", "is_qbnewb": false, "is_superuser": true, "id": 2, "last_name": "Melandri", "date_joined": "2020-05-19T09:02:11.385428Z", "common_name": "Alessio Melandri"}, "can_write": true, "database_id": 3, "enable_embedding": false, "collection_id": 13, "query_type": "native", "name": "§[metabase.enanched]§", "dashboard_count": 1, "read_permissions": null, "creator_id": 2, "updated_at": "2021-05-23T06:12:41.32512Z", "made_public_by_id": null, "embedding_params": null, "cache_ttl": null, "dataset_query": {"type": "native", "native": {"query": "select count(*)\nfrom usages\nwhere is_alive is true"}, "database": 3}, "id": 39, "display": "scalar", "visualization_settings": {}, "collection": {"id": 13, "name": "Usage", "description": null, "color": "#509EE3", "archived": false, "location": "/7/", "personal_owner_id": null, "slug": "usage"}, "created_at": "2021-01-18T15:18:59.673031Z", "public_uuid": null}
--------------------------------------------------------------------------------
/etl/cards/40.json:
--------------------------------------------------------------------------------
1 | {"description": "§[metabase.media-used-desc]§", "archived": false, "collection_position": null, "table_id": null, "result_metadata": [{"base_type": "type/DateTimeWithLocalTZ", "display_name": "§[metabase.year]§", "name": "§[metabase.year]§", "special_type": null, "fingerprint": {"global": {"distinct-count": 4, "nil%": 0.0}, "type": {"type/DateTime": {"earliest": "2017-01-01T00:00:00+01:00", "latest": "2020-01-01T00:00:00+01:00"}}}}, {"base_type": "type/BigInteger", "display_name": "§[metabase.new-usages]§", "name": "§[metabase.new-usages]§", "special_type": null, "fingerprint": {"global": {"distinct-count": 4, "nil%": 0.0}, "type": {"type/Number": {"min": 5.0, "max": 736.0, "avg": 220.75, "sd": 347.2188310945515, "q1": 13.5, "q3": 428.0}}}}, {"base_type": "type/Decimal", "display_name": "§[metabase.cumulative-usage]§", "name": "§[metabase.cumulative-usage]§", "special_type": null, "fingerprint": {"global": {"distinct-count": 4, "nil%": 0.0}, "type": {"type/Number": {"min": 736.0, "max": 883.0, "avg": 805.25, "sd": 77.62463097067408, "q1": 738.5, "q3": 872.0}}}}], "creator": {"email": "alessio@synapta.it", "first_name": "Alessio", "last_login": "2021-01-25T09:56:39.29734Z", "is_qbnewb": false, "is_superuser": true, "id": 2, "last_name": "Melandri", "date_joined": "2020-05-19T09:02:11.385428Z", "common_name": "Alessio Melandri"}, "can_write": true, "database_id": 3, "enable_embedding": false, "collection_id": 13, "query_type": "native", "name": "§[metabase.media-used-year]§", "dashboard_count": 1, "read_permissions": null, "creator_id": 2, "updated_at": "2021-05-23T06:12:41.309301Z", "made_public_by_id": null, "embedding_params": null, "cache_ttl": null, "dataset_query": {"database": 3, "native": {"query": "SELECT d_year as \"§[metabase.year]§\", new_usage as \"§[metabase.new-usages]§\", sum(new_usage) OVER (ORDER BY d_year) as \"§[metabase.cumulative-usage]§\"\nfrom (\nSELECT date_trunc('year', first_seen) AS d_year, COUNT(*) AS new_usage\nFROM usages\ngroup by d_year) as a\ngroup by d_year, new_usage"}, "type": "native"}, "id": 40, "display": "line", "visualization_settings": {"graph.dimensions": ["§[metabase.year]§"], "graph.metrics": ["§[metabase.new-usages]§", "§[metabase.cumulative-usage]§"], "graph.show_values": false, "series_settings": {"§[metabase.new-usages]§": {"color": "#509EE3"}, "§[metabase.cumulative-usage]§": {"color": "#A989C5"}}}, "collection": {"id": 13, "name": "Usage", "description": null, "color": "#509EE3", "archived": false, "location": "/7/", "personal_owner_id": null, "slug": "usage"}, "created_at": "2021-01-18T15:32:16.244733Z", "public_uuid": null}
--------------------------------------------------------------------------------
/etl/cards/44.json:
--------------------------------------------------------------------------------
1 | {"description": "§[metabase.total-media-desc]§", "archived": false, "collection_position": null, "table_id": null, "result_metadata": [{"base_type": "type/BigInteger", "display_name": "count", "name": "count", "special_type": "type/Quantity", "fingerprint": {"global": {"distinct-count": 1, "nil%": 0.0}, "type": {"type/Number": {"min": 841.0, "max": 841.0, "avg": 841.0, "sd": null, "q1": 841.0, "q3": 841.0}}}}], "creator": {"email": "alessio@synapta.it", "first_name": "Alessio", "last_login": "2021-01-25T09:56:39.29734Z", "is_qbnewb": false, "is_superuser": true, "id": 2, "last_name": "Melandri", "date_joined": "2020-05-19T09:02:11.385428Z", "common_name": "Alessio Melandri"}, "can_write": true, "database_id": 3, "enable_embedding": false, "collection_id": 14, "query_type": "native", "name": "§[metabase.total-media]§", "dashboard_count": 1, "read_permissions": null, "creator_id": 2, "updated_at": "2021-05-23T06:12:41.564956Z", "made_public_by_id": null, "embedding_params": null, "cache_ttl": null, "dataset_query": {"type": "native", "native": {"query": "SELECT COUNT(*)\nFROM images"}, "database": 3}, "id": 44, "display": "scalar", "visualization_settings": {}, "collection": {"id": 14, "name": "Upload", "description": null, "color": "#509EE3", "archived": false, "location": "/7/", "personal_owner_id": null, "slug": "upload"}, "created_at": "2021-01-19T10:03:24.77234Z", "public_uuid": null}
--------------------------------------------------------------------------------
/etl/cards/45.json:
--------------------------------------------------------------------------------
1 | {"description": "§[metabase.media-added-desc]§", "archived": false, "collection_position": null, "table_id": null, "result_metadata": [{"base_type": "type/DateTime", "display_name": "§[metabase.year]§", "name": "§[metabase.year]§", "special_type": null, "fingerprint": {"global": {"distinct-count": 16, "nil%": 0.0}, "type": {"type/DateTime": {"earliest": "2005-01-01T00:00:00+01:00", "latest": "2020-01-01T00:00:00+01:00"}}}}, {"base_type": "type/BigInteger", "display_name": "§[metabase.media-per-year]§", "name": "§[metabase.media-per-year]§", "special_type": null, "fingerprint": {"global": {"distinct-count": 16, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "max": 233.0, "avg": 52.5625, "sd": 53.95920526966027, "q1": 25.5, "q3": 61.0}}}}, {"base_type": "type/Decimal", "display_name": "§[metabase.media-cumulated]§", "name": "§[metabase.media-cumulated]§", "special_type": null, "fingerprint": {"global": {"distinct-count": 16, "nil%": 0.0}, "type": {"type/Number": {"min": 1.0, "max": 841.0, "avg": 471.4375, "sd": 286.42066237150794, "q1": 285.0, "q3": 711.0}}}}], "creator": {"email": "alessio@synapta.it", "first_name": "Alessio", "last_login": "2021-01-25T09:56:39.29734Z", "is_qbnewb": false, "is_superuser": true, "id": 2, "last_name": "Melandri", "date_joined": "2020-05-19T09:02:11.385428Z", "common_name": "Alessio Melandri"}, "can_write": true, "database_id": 3, "enable_embedding": false, "collection_id": 14, "query_type": "native", "name": "§[metabase.media-added]§", "dashboard_count": 1, "read_permissions": null, "creator_id": 2, "updated_at": "2021-05-23T06:12:41.452581Z", "made_public_by_id": null, "embedding_params": null, "cache_ttl": null, "dataset_query": {"database": 3, "native": {"query": "SELECT d_year as \"§[metabase.year]§\", media_year as \"§[metabase.media-per-year]§\", sum(media_year) OVER (ORDER BY d_year) as \"§[metabase.media-cumulated]§\"\nfrom (\nSELECT date_trunc('year', img_timestamp) AS d_year, COUNT(*) AS media_year\nFROM images\ngroup by d_year) as a\ngroup by d_year, media_year"}, "type": "native"}, "id": 45, "display": "line", "visualization_settings": {"graph.dimensions": ["§[metabase.year]§"], "graph.metrics": ["§[metabase.media-per-year]§", "§[metabase.media-cumulated]§"], "graph.show_values": false, "series_settings": {"§[metabase.media-per-year]§": {"color": "#509EE3"}, "§[metabase.media-cumulated]§": {"color": "#A989C5"}}}, "collection": {"id": 14, "name": "Upload", "description": null, "color": "#509EE3", "archived": false, "location": "/7/", "personal_owner_id": null, "slug": "upload"}, "created_at": "2021-01-19T10:13:27.009467Z", "public_uuid": null}
--------------------------------------------------------------------------------
/etl/cards/7.json:
--------------------------------------------------------------------------------
1 | {"description": "§[metabase.corr-size-desc]§", "archived": false, "collection_position": null, "table_id": null, "result_metadata": [{"base_type": "type/Float", "display_name": "Corr Coef Using PGSQL Func", "name": "Corr Coef Using PGSQL Func", "special_type": null, "fingerprint": {"global": {"distinct-count": 1, "nil%": 0.0}, "type": {"type/Number": {"min": -0.03958996396310031, "max": -0.03958996396310031, "avg": -0.03958996396310031, "sd": null, "q1": -0.03958996396310031, "q3": -0.03958996396310031}}}}], "creator": {"email": "alessio@synapta.it", "first_name": "Alessio", "last_login": "2021-01-25T09:56:39.29734Z", "is_qbnewb": false, "is_superuser": true, "id": 2, "last_name": "Melandri", "date_joined": "2020-05-19T09:02:11.385428Z", "common_name": "Alessio Melandri"}, "can_write": true, "database_id": 3, "enable_embedding": false, "collection_id": 16, "query_type": "native", "name": "§[& metabase.corr-size]§", "dashboard_count": 1, "read_permissions": null, "creator_id": 2, "updated_at": "2021-05-23T06:12:44.61295Z", "made_public_by_id": null, "embedding_params": null, "cache_ttl": null, "dataset_query": {"type": "native", "native": {"query": "SELECT corr(average, img_size) as \"Corr Coef Using PGSQL Func\" FROM(\nSELECT AVG(accesses) as average, img_size\nFROM images i, visualizations v\nWHERE i.media_id = v.media_id\ngroup by img_name\n) as a\n\n \n"}, "database": 3}, "id": 7, "display": "scalar", "visualization_settings": {"graph.dimensions": ["average"], "graph.metrics": ["img_size"], "graph.x_axis.scale": "pow", "graph.y_axis.scale": "pow"}, "collection": {"id": 16, "name": "Correlations", "description": null, "color": "#509EE3", "archived": false, "location": "/7/", "personal_owner_id": null, "slug": "correlations"}, "created_at": "2020-06-03T13:51:16.813633Z", "public_uuid": null}
--------------------------------------------------------------------------------
/etl/cards/9.json:
--------------------------------------------------------------------------------
1 | {"description": "§[metabase.corr-usage-desc]§", "archived": false, "collection_position": null, "table_id": null, "result_metadata": [{"base_type": "type/Float", "display_name": "Corr Coef Using PGSQL Func", "name": "Corr Coef Using PGSQL Func", "special_type": null, "fingerprint": {"global": {"distinct-count": 1, "nil%": 0.0}, "type": {"type/Number": {"min": 0.6663149842984744, "max": 0.6663149842984744, "avg": 0.6663149842984744, "sd": null, "q1": 0.6663149842984744, "q3": 0.6663149842984744}}}}], "creator": {"email": "alessio@synapta.it", "first_name": "Alessio", "last_login": "2021-01-25T09:56:39.29734Z", "is_qbnewb": false, "is_superuser": true, "id": 2, "last_name": "Melandri", "date_joined": "2020-05-19T09:02:11.385428Z", "common_name": "Alessio Melandri"}, "can_write": true, "database_id": 3, "enable_embedding": false, "collection_id": 16, "query_type": "native", "name": "§[& metabase.corr-usage]§", "dashboard_count": 1, "read_permissions": null, "creator_id": 2, "updated_at": "2021-05-23T06:14:31.679824Z", "made_public_by_id": null, "embedding_params": null, "cache_ttl": null, "dataset_query": {"type": "native", "native": {"query": "SELECT corr(average, usage) as \"Corr Coef Using PGSQL Func\" FROM(\nSELECT MIN(\"avg\") as average, AVG(u.\"count\") as usage\nFROM visualizations_stats v, dailyimageusage u\nWHERE u.img_name = v.img_name\ngroup by u.img_name\n) as a\n"}, "database": 3}, "id": 9, "display": "scalar", "visualization_settings": {"graph.dimensions": ["average"], "graph.metrics": ["img_size"], "graph.x_axis.scale": "pow", "graph.y_axis.scale": "pow"}, "collection": {"id": 16, "name": "Correlations", "description": null, "color": "#509EE3", "archived": false, "location": "/7/", "personal_owner_id": null, "slug": "correlations"}, "created_at": "2020-06-03T13:58:33.081456Z", "public_uuid": null}
--------------------------------------------------------------------------------
/etl/dashboard.py:
--------------------------------------------------------------------------------
1 | import json
2 | import pymongo
3 | import requests
4 | import sys
5 |
6 | config_file = '../config/config.json'
7 |
8 | config = json.load(open(config_file))
9 |
10 | try:
11 | sys.argv[1]
12 | except IndexError:
13 | print('Please provide a GLAM')
14 | sys.exit(1)
15 |
16 | mongo_client = pymongo.MongoClient(config['mongodb']['url'])
17 | mongo_db = mongo_client[config['mongodb']['database']]
18 | mongo_collection = mongo_db[config['mongodb']['collection']]
19 |
20 | # Find GLAM
21 | glam = mongo_collection.find_one({"name": sys.argv[1]})
22 |
23 | if glam is None:
24 | print('Cannot find GLAM', sys.argv[1])
25 | sys.exit(1)
26 |
27 | # Get Metabase session
28 | r = requests.post(config['metabase']['url'] + '/api/session', json={
29 | "username": config['metabase']['username'],
30 | "password": config['metabase']['password']
31 | })
32 | headers = {"X-Metabase-Session": r.json()['id']}
33 |
34 | # Create database
35 | database = requests.post(config['metabase']['url'] + '/api/database', headers=headers, json={
36 | "name": glam['fullname'],
37 | "engine": "postgres",
38 | "details": {
39 | "dbname": glam['database'],
40 | "host": config['metabase']['database']['host'],
41 | "port": config['metabase']['database']['port'],
42 | "user": config['metabase']['database']['user'],
43 | "password": config['metabase']['database']['password'],
44 | "ssl": True
45 | }
46 | })
47 | database_id = database.json()['id']
48 |
49 | # Create collection
50 | collection = requests.post(config['metabase']['url'] + '/api/collection', headers=headers, json={
51 | "name": glam['fullname'],
52 | "color": "#509EE3"
53 | })
54 | collection_id = collection.json()['id']
55 |
56 | # Create dashboard
57 | new_dashboard = requests.post(config['metabase']['url'] + '/api/dashboard', headers=headers, json={
58 | "name": glam['fullname'],
59 | "description": None,
60 | "parameters": [],
61 | "collection_id": collection_id
62 | })
63 | dashboard_id = new_dashboard.json()['id']
64 |
65 | # Get reference dashboard
66 | with open('dashboard.json', 'r') as f:
67 | dashboard = json.load(f)
68 |
69 | # For all the cards
70 | for card in dashboard['ordered_cards']:
71 |
72 | # Create a new card
73 | if card['card_id'] is not None:
74 | with open('cards/' + str(card['card_id']) + '.json', 'r') as f:
75 | new_card_dict = json.load(f)
76 |
77 | new_card_dict['dataset_query']['database'] = database_id
78 |
79 | new_card = requests.post(config['metabase']['url'] + '/api/card', headers=headers, json={
80 | "visualization_settings": new_card_dict['visualization_settings'],
81 | "collection_id": collection_id,
82 | "name": new_card_dict['name'],
83 | "description": new_card_dict['description'],
84 | "dataset_query": new_card_dict['dataset_query'],
85 | "display": new_card_dict['display']
86 | })
87 |
88 | card_id = new_card.json()['id']
89 |
90 | else:
91 | card_id = None
92 |
93 | # Add card to dashboard
94 | requests.post(config['metabase']['url'] + '/api/dashboard/' + str(dashboard_id) + '/cards', headers=headers, json={
95 | "sizeX": card['sizeX'],
96 | "sizeY": card['sizeY'],
97 | "row": card['row'],
98 | "col": card['col'],
99 | "series": card['series'],
100 | "parameter_mappings": card['parameter_mappings'],
101 | "visualization_settings": card['visualization_settings'],
102 | "cardId": card_id
103 | })
104 |
105 | # Enable embedding
106 | requests.put(config['metabase']['url'] + '/api/dashboard/' + str(dashboard_id), headers=headers, json={
107 | "enable_embedding": True
108 | })
109 |
110 | # Save dashboard_id
111 | mongo_collection.update_one({'_id': glam['_id']}, {
112 | '$set': {'dashboard_id': dashboard_id}})
113 |
--------------------------------------------------------------------------------
/etl/run.py:
--------------------------------------------------------------------------------
1 | import fcntl
2 | import json
3 | import logging
4 | import os
5 | import subprocess
6 | import sys
7 | import time
8 | from datetime import datetime, timedelta
9 | from subprocess import SubprocessError
10 |
11 | import psycopg2
12 | import pymongo
13 | from psycopg2 import ProgrammingError
14 |
15 | import sentry_sdk
16 | from sentry_sdk.integrations.logging import LoggingIntegration
17 |
18 | config_file = '../config/config.json'
19 |
20 | logging.basicConfig(level=logging.INFO,
21 | format='%(asctime)s %(levelname)s %(message)s')
22 |
23 |
24 | def update(collection, glam):
25 | collection.update_one({'_id': glam['_id']}, {
26 | '$set': {'lastrun': datetime.utcnow(),
27 | 'status': 'running'}})
28 |
29 |
30 | def fail(collection, glam):
31 | collection.update_one({'_id': glam['_id']}, {
32 | '$set': {'status': 'failed'}})
33 |
34 |
35 | def views_date():
36 | date = datetime.utcnow() - timedelta(days=1)
37 | return date.strftime("%Y-%m-%d")
38 |
39 |
40 | def setup(name):
41 | logging.info('Running setup.js for %s', name)
42 | subprocess.run(['node', 'setup.js', name], check=True)
43 | logging.info('Subprocess setup.js completed')
44 |
45 |
46 | def dashboard(name):
47 | logging.info('Running dashboard.py for %s', name)
48 | subprocess.run(['python3', 'dashboard.py', name], check=True)
49 | logging.info('Subprocess dashboard.py completed')
50 |
51 |
52 | def etl(name):
53 | logging.info('Running etl.js for %s', name)
54 | subprocess.run(['node', 'etl.js', name], check=True)
55 | logging.info('Subprocess etl.js completed')
56 |
57 |
58 | def process_glam(collection, glam):
59 | if datetime.utcnow() < glam['lastrun'] + timedelta(days=1):
60 | logging.info('Glam %s is already updated', glam['name'])
61 | return
62 |
63 | # mediacounts are available around 2:00 UTC
64 | if datetime.utcnow().hour <= 2:
65 | logging.info('Glam %s update delayed', glam['name'])
66 | return
67 |
68 | success = True
69 | logging.info('Running scheduler for %s', glam['name'])
70 |
71 | # Run etl.js
72 | try:
73 | etl(glam['name'])
74 | except SubprocessError as e:
75 | success = False
76 | logging.error('Subprocess etl.js failed')
77 |
78 | if e.returncode == 65:
79 | logging.error('Glam %s is now failed', glam['name'])
80 | fail(collection, glam)
81 | return
82 |
83 | if success:
84 | logging.info('Completed scheduler for %s', glam['name'])
85 | update(collection, glam)
86 | else:
87 | logging.error('Failed scheduler for %s', glam['name'])
88 |
89 |
90 | def create_database(config, database):
91 | connstring = "dbname=template1 user=" + config['postgres']['user'] + \
92 | " password=" + config['postgres']['password'] + \
93 | " host=" + config['postgres']['host'] + \
94 | " port=" + str(config['postgres']['port'])
95 | conn = psycopg2.connect(connstring)
96 | conn.autocommit = True
97 | curse = conn.cursor()
98 | try:
99 | curse.execute("CREATE DATABASE " + database + " WITH OWNER = " + config['postgres']['user'] + " " +
100 | "ENCODING = 'UTF8' " +
101 | "CONNECTION LIMIT = -1 TEMPLATE template0;")
102 | curse.execute("GRANT CONNECT ON DATABASE " + database + " TO metabase;")
103 | except ProgrammingError:
104 | # the database is already available
105 | pass
106 | finally:
107 | conn.close()
108 |
109 |
110 | def main():
111 | config = json.load(open(config_file))
112 |
113 | try:
114 | sentry_logging = LoggingIntegration(
115 | level=logging.INFO,
116 | event_level=logging.ERROR
117 | )
118 | sentry_sdk.init(
119 | dsn=config['raven']['glamtoolsetl']['DSN'],
120 | integrations=[sentry_logging]
121 | )
122 | logging.info('External error reporting ENABLED')
123 | except KeyError:
124 | logging.info('External error reporting DISABLED')
125 |
126 | client = pymongo.MongoClient(config['mongodb']['url'])
127 | db = client[config['mongodb']['database']]
128 | collection = db[config['mongodb']['collection']]
129 |
130 | for glam in collection.find():
131 | if 'status' in glam:
132 | if glam['status'] == 'paused':
133 | logging.info('Glam %s is paused', glam['name'])
134 | continue
135 |
136 | if glam['status'] == 'failed':
137 | logging.info('Glam %s is failed', glam['name'])
138 | continue
139 |
140 | if 'lastrun' in glam:
141 | process_glam(collection, glam)
142 | else:
143 | # this is the first run
144 | glam['lastrun'] = datetime.fromtimestamp(0)
145 |
146 | create_database(config, glam['database'])
147 |
148 | try:
149 | setup(glam['name'])
150 | dashboard(glam['name'])
151 | except SubprocessError:
152 | logging.error('Subprocess setup.js or dashboard.py failed')
153 | continue
154 |
155 | process_glam(collection, glam)
156 |
157 |
158 | if __name__ == '__main__':
159 | # change the working directory to the script's own directory
160 | script_dir = os.path.dirname(sys.argv[0])
161 | if script_dir != '':
162 | os.chdir(script_dir)
163 |
164 | try:
165 | lockfile = open('/tmp/cassandra.lock', 'w')
166 | fcntl.flock(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
167 | main()
168 | fcntl.flock(lockfile, fcntl.LOCK_UN)
169 | except IOError:
170 | raise SystemExit('Scheduler is already running')
171 |
--------------------------------------------------------------------------------
/etl/run_views.py:
--------------------------------------------------------------------------------
1 | import fcntl
2 | import json
3 | import logging
4 | import os
5 | import subprocess
6 | import sys
7 | from datetime import date, datetime, timedelta
8 | from subprocess import SubprocessError
9 |
10 | import psycopg2
11 | import pymongo
12 |
13 | import sentry_sdk
14 | from sentry_sdk.integrations.logging import LoggingIntegration
15 |
16 | config_file = '../config/config.json'
17 |
18 | global_min_date = date(2015, 1, 1)
19 | global_max_date = date.today() - timedelta(days=2)
20 | views_dir = 'temp'
21 |
22 | logging.basicConfig(level=logging.INFO,
23 | format='%(asctime)s %(levelname)s %(message)s')
24 |
25 |
26 | def add_missing_dates(config, glam):
27 | logging.info('Processing glam %s', glam['name'])
28 |
29 | connstring = "dbname=" + glam['database'] + " user=" + config['postgres']['user'] + \
30 | " password=" + config['postgres']['password'] + \
31 | " host=" + config['postgres']['host'] + \
32 | " port=" + str(config['postgres']['port'])
33 | conn = psycopg2.connect(connstring)
34 | conn.autocommit = True
35 | curse = conn.cursor()
36 |
37 | # Find the dates already in the database
38 | curse.execute(
39 | "select distinct access_date from visualizations order by access_date")
40 | current_dates = list(map(lambda x: x[0], curse.fetchall()))
41 |
42 | # Find the date of the first image, if any
43 | curse.execute("SELECT min(img_timestamp) FROM images")
44 | try:
45 | first_image = curse.fetchone()[0].date()
46 | except TypeError:
47 | first_image = global_min_date
48 | conn.close()
49 |
50 | try:
51 | min_date = datetime.strptime(glam['min_date'], "%Y-%m-%d").date()
52 | except KeyError:
53 | min_date = max([first_image, date.today() - timedelta(days=10)])
54 |
55 | candidate_dates = [min_date + timedelta(days=x)
56 | for x in range(0, (global_max_date - min_date).days)]
57 |
58 | glam['missing_dates'] = []
59 |
60 | for date_value in candidate_dates:
61 | if date_value not in current_dates:
62 | glam['missing_dates'].append(date_value)
63 |
64 |
65 | def update_min_date(collection, glam, date_str):
66 | collection.update_one({'_id': glam['_id']}, {
67 | '$set': {'min_date': date_str}})
68 |
69 |
70 | def main():
71 | config = json.load(open(config_file))
72 |
73 | try:
74 | sentry_logging = LoggingIntegration(
75 | level=logging.INFO,
76 | event_level=logging.ERROR
77 | )
78 | sentry_sdk.init(
79 | dsn=config['raven']['glamtoolsetl']['DSN'],
80 | integrations=[sentry_logging]
81 | )
82 | logging.info('External error reporting ENABLED')
83 | except KeyError:
84 | logging.info('External error reporting DISABLED')
85 |
86 | client = pymongo.MongoClient(config['mongodb']['url'])
87 | db = client[config['mongodb']['database']]
88 | collection = db[config['mongodb']['collection']]
89 |
90 | glams = []
91 |
92 | for glam in collection.find():
93 | if 'status' in glam:
94 | if glam['status'] == 'paused':
95 | logging.info('Glam %s is paused', glam['name'])
96 | continue
97 |
98 | if glam['status'] == 'failed':
99 | logging.info('Glam %s is failed', glam['name'])
100 | continue
101 |
102 | add_missing_dates(config, glam)
103 | glams.append(glam)
104 |
105 | date_interval = [global_min_date + timedelta(days=x)
106 | for x in range(0, (global_max_date - global_min_date).days)]
107 |
108 | # for all dates
109 | for date_value in date_interval:
110 | logging.info('Working with date %s', date_value)
111 |
112 | for glam in glams:
113 | logging.info('Working with GLAM %s', glam['name'])
114 | date_str = date_value.strftime("%Y-%m-%d")
115 |
116 | if date_value in glam['missing_dates']:
117 | try:
118 | subprocess.run(['python3', 'views.py', glam['name'], date_str, '--dir', views_dir], check=True)
119 | update_min_date(collection, glam, date_str)
120 | except SubprocessError:
121 | logging.error('Subprocess views.py failed')
122 |
123 | views_path = os.path.join(views_dir, date_str + '.tsv.bz2')
124 | if os.path.isfile(views_path):
125 | os.remove(views_path)
126 |
127 |
128 | if __name__ == '__main__':
129 | # change the working directory to the script's own directory
130 | script_dir = os.path.dirname(sys.argv[0])
131 | if script_dir != '':
132 | os.chdir(script_dir)
133 |
134 | try:
135 | lockfile = open('/tmp/cassandra_views.lock', 'w')
136 | fcntl.flock(lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
137 | main()
138 | fcntl.flock(lockfile, fcntl.LOCK_UN)
139 | except IOError:
140 | raise SystemExit('Views is already running')
141 |
--------------------------------------------------------------------------------
/etl/setup.js:
--------------------------------------------------------------------------------
1 | var config = require("../config/config.js");
2 | var fs = require("fs");
3 |
4 | var endSetup = function () {
5 | console.log("Setup ended");
6 | process.exit(0);
7 | };
8 |
9 | var processQuery = function (db, files, step) {
10 | if (step >= files.length) {
11 | endSetup();
12 | return;
13 | }
14 | filetoRead = "SQL/" + files[step];
15 | console.log(filetoRead);
16 | fs.readFile(filetoRead, 'ascii', function (_, read) {
17 | // console.log("Executing: ");
18 | read = read.toString('ascii');
19 | // console.log(read);
20 | db.query(read, function (err, b) {
21 | if (!err) {
22 | processQuery(db, files, step+1);
23 | }
24 | else {
25 | console.log("Error in step " + step);
26 | console.log(err);
27 | endSetup();
28 | return;
29 | }
30 | })
31 | })
32 | };
33 |
34 | if (process.argv.length != 3) {
35 | console.log('Missing GLAM name');
36 | process.exit(1);
37 | }
38 |
39 | config.loadGlams(() => {
40 | var glam = config.glams[process.argv[2]];
41 |
42 | if (glam === undefined) {
43 | console.log('Unknown GLAM name');
44 | process.exit(1);
45 | }
46 |
47 | var files = ["db_init.sql", "dailyInsert.sql", "functions.sql", "maintenance.sql"];
48 |
49 | console.log("Setup started");
50 | processQuery(glam.connection, files, 0);
51 | });
52 |
--------------------------------------------------------------------------------
/etl/views.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | import bz2
3 | import json
4 | import os
5 | import sys
6 | import urllib.parse
7 | import urllib.request
8 |
9 | import psycopg2
10 | import pymongo
11 |
12 | watched = set()
13 |
14 |
15 | def reporter(first, second, third):
16 | if first % 1000 == 0:
17 | print("Download progress: " + str(first * second * 100 // third) + "%")
18 |
19 |
20 | def download(date, folder):
21 | if not os.path.exists(folder):
22 | os.makedirs(folder)
23 |
24 | filename = os.path.join(folder, date + '.tsv.bz2')
25 |
26 | year, month, day = date.split("-")
27 | baseurl = "https://dumps.wikimedia.org/other/mediacounts/daily/"
28 | finalurl = baseurl + year + "/mediacounts." + \
29 | year + "-" + month + "-" + day + ".v00.tsv.bz2"
30 |
31 | # check file size
32 | if os.path.isfile(filename):
33 | remote_size = urllib.request.urlopen(finalurl).length
34 | local_size = os.stat(filename).st_size
35 | if remote_size == local_size:
36 | return filename
37 | else:
38 | os.remove(filename)
39 |
40 | print("Retrieving " + finalurl + "...")
41 | urllib.request.urlretrieve(finalurl, filename, reporter)
42 | print("Download completed.")
43 |
44 | return filename
45 |
46 |
47 | def process(conn, date, folder):
48 | filename = download(date, folder)
49 | print("Loading visualizations from file", filename)
50 |
51 | source_file = bz2.BZ2File(filename, "r")
52 | curse = conn.cursor()
53 | counter = 0
54 |
55 | for line in source_file:
56 | if counter == len(watched):
57 | break
58 | arr = line.decode().split("\t")
59 | keysX = arr[0].split("/")
60 | # count only files from commons
61 | if len(keysX) < 6 or keysX[2] != 'commons':
62 | continue
63 | key = keysX[len(keysX) - 1]
64 | key = urllib.parse.unquote(key)
65 | if key in watched:
66 | counter += 1
67 | if counter % 100 == 0:
68 | print("Loading progress: " +
69 | str(counter * 100 // len(watched)) + "%")
70 | query = "select * from dailyinsert('" + key.replace(
71 | "'", "''") + "','" + date + "'," + arr[2] + "," + arr[22] + "," + arr[23] + ")"
72 | curse.execute(query)
73 |
74 | curse.execute('refresh materialized view visualizations_sum')
75 | curse.execute('refresh materialized view visualizations_stats')
76 | curse.close()
77 | source_file.close()
78 |
79 |
80 | def loadImages(conn):
81 | curse = conn.cursor()
82 | curse.execute("SELECT img_name FROM images;")
83 | w = 0
84 | while w < curse.rowcount:
85 | w += 1
86 | file = curse.fetchone()
87 | file = file[0]
88 | # print(file)
89 | if file not in watched:
90 | watched.add(file)
91 | curse.close()
92 |
93 |
94 | def main():
95 | parser = argparse.ArgumentParser()
96 | parser.add_argument('glam', type=str)
97 | parser.add_argument('date', type=str)
98 | parser.add_argument('--dir', type=str, required=False, default='temp')
99 | args = parser.parse_args()
100 |
101 | # read settings
102 | config = json.load(open('../config/config.json'))
103 | client = pymongo.MongoClient(config['mongodb']['url'])
104 | db = client[config['mongodb']['database']]
105 | collection = db[config['mongodb']['collection']]
106 | glam = collection.find_one({"name": args.glam})
107 |
108 | if glam == None:
109 | print("Unknown Glam name", args.glam)
110 | sys.exit(1)
111 |
112 | connstring = "dbname=" + glam['database'] + " user=" + config['postgres']['user'] + \
113 | " password=" + config['postgres']['password'] + \
114 | " host=" + config['postgres']['host'] + \
115 | " port=" + str(config['postgres']['port'])
116 | pgconnection = psycopg2.connect(connstring)
117 | # print(pgconnection.encoding)
118 | pgconnection.autocommit = True
119 |
120 | loadImages(pgconnection)
121 | process(pgconnection, args.date, args.dir)
122 |
123 | pgconnection.close()
124 | print("Process completed")
125 |
126 |
127 | if __name__ == "__main__":
128 | main()
129 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "cassandra",
3 | "version": "2.0.0",
4 | "author": "Synapta Srl
4 |
36 |
37 |
5 |
6 | {{user}}
7 |
8 |
19 |
20 |
21 |
35 |
22 |
34 |
23 |
24 | §[messages.files]§
25 |
26 |
27 |
31 | {{total}}
32 |
33 |