├── .gitignore ├── composer.json ├── clean_old_tmp_files.sh ├── start_bot.sh ├── README ├── cronfile ├── new_job_from_qs.php ├── public_html ├── config.json.template ├── vue.js ├── vue_components │ ├── batch_access_mixin.html │ ├── user.html │ ├── batches.html │ ├── user-page.html │ ├── main-page.html │ ├── batch-commands.html │ ├── command.html │ └── batch.html ├── index.html ├── index_old.html ├── api.php └── quickstatements.js ├── schema.sql ├── bot.php └── COPYING /.gitignore: -------------------------------------------------------------------------------- 1 | # autogenerated 2 | /composer.lock 3 | /vendor 4 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "require": { 3 | "addwiki/mediawiki-api": "~2.0" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /clean_old_tmp_files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | find /data/project/quickstatements/public_html/tmp/ -mindepth 1 -mtime +1 -delete 3 | 4 | -------------------------------------------------------------------------------- /start_bot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | qdel bot 3 | sleep 10 4 | jsub -mem 4g -N bot -once -continuous /data/project/quickstatements/bot.php 5 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Using addwiki: 2 | https://github.com/addwiki/mediawiki-api 3 | 4 | Copy public_html/config.json.template to config.json and modify for your needs 5 | -------------------------------------------------------------------------------- /cronfile: -------------------------------------------------------------------------------- 1 | #17 * * * * jsub -mem 2g -N bot1 -quiet /data/project/quickstatements/bot.php single_batch 2 | #0,5,10,15,20,25,30,35,40,45,50,55 * * * * jsub -mem 2g -N bot1 -quiet /data/project/quickstatements/bot.php single_batch 3 | #3,8,13,18,23,28,33,38,43,48,53,58 * * * * jsub -mem 2g -N bot2 -quiet /data/project/quickstatements/bot.php single_batch 4 | * 2 * * * jsub -N old_tmp -once -quiet /data/project/quickstatements/clean_old_tmp_files.sh 5 | 6 | # THis now runs on the petscan vm 7 | #15,45 * * * * jsub -mem 4g -once -quiet -N qs_rust /data/project/quickstatements/rust/quickstatements_rs/target/release/main bot 8 | -------------------------------------------------------------------------------- /new_job_from_qs.php: -------------------------------------------------------------------------------- 1 | #!/usr/bin/php 2 | use_command_compression = true ; 18 | $j = $qs->importData ( $commands , 'V1' , true ) ; 19 | #$j['data']['commands'] = $qs->compressCommands ( $j['data']['commands'] ) ; 20 | #print_r ( $j ) ; 21 | $batch_id = $qs->addBatch ( $j['data']['commands'] , $user_id_magnus , $batch_name ) ; 22 | print "Now as batch #$batch_id with " . count($j['data']['commands']) . " commands\n" ; 23 | 24 | ?> 25 | -------------------------------------------------------------------------------- /public_html/config.json.template: -------------------------------------------------------------------------------- 1 | { 2 | "site" : "wikidata" , 3 | "bot_config_file" : "/data/project/quickstatements/bot.ini" , 4 | "logfile" : "/data/project/quickstatements/tool.log" , 5 | "valid_origin" : "https://quickstatements.toolforge.org" , 6 | "sites" : { 7 | "wikidata" : { 8 | "oauth" : { "language":"wikidata" , "project":"wikidata" , "ini_file":"/data/project/quickstatements/oauth.ini" , "mwOAuthUrl":"https://www.mediawiki.org/w/index.php?title=Special:OAuth" , "mwOAuthIW":"mw" } , 9 | "server" : "www.wikidata.org" , 10 | "api" : "https://www.wikidata.org/w/api.php" , 11 | "pageBase" : "https://www.wikidata.org/wiki/" , 12 | "entityBase" : "http://www.wikidata.org/entity/" , 13 | "toolBase" : "https://quickstatements.toolforge.org" , 14 | "types" : { 15 | "P" : { "type":"property" , "ns":120 , "ns_prefix":"Property:" } , 16 | "Q" : { "type":"item" , "ns":0 , "ns_prefix":"" }, 17 | "L" : { "type":"lexeme", "ns" 146, "ns_prefix":"Lexeme:" } 18 | } 19 | } , 20 | "factgrid" : { 21 | "oauth" : { "language":"wikidata" , "project":"wikidata" , "ini_file":"/home/factgrid/quickstatements/oauth.ini" , "mwOAuthUrl":"https://database.factgrid.de/index.php?title=Special:OAuth" , "mwOAuthIW":"" } , 22 | "api" : "https://database.factgrid.de/api.php" , 23 | "pageBase" : "https://database.factgrid.de/index.php/" , 24 | "entityBase" : "https://database.factgrid.de/entity/" , 25 | "toolBase" : "https://database.factgrid.de/quickstatements" , 26 | "types" : { 27 | "P" : { "type":"property" , "ns":122 , "ns_prefix":"Property:" } , 28 | "Q" : { "type":"item" , "ns":120 , "ns_prefix":"Item:" } 29 | } 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /schema.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE `batch` ( 2 | `id` int(11) unsigned NOT NULL AUTO_INCREMENT, 3 | `name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL, 4 | `user` int(11) NOT NULL DEFAULT '', 5 | `ts_created` varchar(14) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', 6 | `ts_last_change` varchar(14) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', 7 | `status` varchar(8) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', 8 | `message` mediumtext COLLATE utf8mb4_unicode_ci NOT NULL, 9 | `last_item` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', 10 | `site` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT 'wikidata', 11 | PRIMARY KEY (`id`), 12 | KEY `status` (`status`), 13 | KEY `user` (`user`(191)), 14 | KEY `ts_last_change` (`ts_last_change`), 15 | KEY `user_2` (`user`(160),`ts_last_change`) 16 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; 17 | 18 | CREATE TABLE `command` ( 19 | `id` int(11) unsigned NOT NULL AUTO_INCREMENT, 20 | `batch_id` int(11) NOT NULL, 21 | `num` int(11) NOT NULL, 22 | `json` mediumtext COLLATE utf8mb4_unicode_ci NOT NULL, 23 | `status` varchar(8) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', 24 | `message` tinytext COLLATE utf8mb4_unicode_ci NOT NULL, 25 | `ts_change` varchar(14) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', 26 | PRIMARY KEY (`id`), 27 | KEY `batch_id` (`batch_id`), 28 | KEY `batch_id_2` (`batch_id`,`status`) 29 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; 30 | 31 | CREATE TABLE `user` ( 32 | `id` int(11) unsigned NOT NULL AUTO_INCREMENT, 33 | `name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', 34 | `api_hash` varchar(64) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '', 35 | PRIMARY KEY (`id`), 36 | KEY `name` (`name`(191)) 37 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; 38 | 39 | CREATE TABLE `batch_oauth` ( 40 | `id` int(11) unsigned NOT NULL AUTO_INCREMENT, 41 | `batch_id` int(11) NOT NULL, 42 | `serialized` mediumtext COLLATE utf8mb4_unicode_ci NOT NULL, 43 | `serialized_json` mediumtext COLLATE utf8mb4_unicode_ci NOT NULL, 44 | PRIMARY KEY (`id`), 45 | KEY `batch_id` (`batch_id`) 46 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; 47 | -------------------------------------------------------------------------------- /bot.php: -------------------------------------------------------------------------------- 1 | #!/usr/bin/php 2 | getDB() ; 10 | $sql = "SELECT id,status FROM batch WHERE status IN ('INIT','RUN')" ; 11 | if(!$result = $db->query($sql)) die('There was an error running the query [' . $db->error . ']'); 12 | while($o = $result->fetch_object()){ 13 | $qs2 = new QuickStatements ; 14 | if ( $o->status == 'INIT' ) { 15 | if ( !$qs2->startBatch ( $o->id ) ) { 16 | print $qs2->last_error_message."\n" ; 17 | continue ; 18 | } 19 | } 20 | if ( !$qs2->runNextCommandInBatch ( $o->id ) ) print $qs2->last_error_message."\n" ; 21 | else $ret++ ; 22 | } 23 | return $ret ; 24 | } 25 | 26 | if ( isset($argv[1]) and $argv[1] == 'single_batch' ) { 27 | $min_sec_inactive = 60 * 60 ; # 1h 28 | $qs = new QuickStatements ; 29 | $db = $qs->getDB() ; 30 | 31 | $sql = "SELECT * FROM batch WHERE status IN ('INIT','RUN') ORDER BY ts_last_change" ; 32 | if ( isset($argv[2]) ) { 33 | $sql = "SELECT * FROM batch WHERE id=".($argv[2]*1) ; 34 | } 35 | if(!$result = $db->query($sql)) die('There was an error running the query [' . $db->error . ']'); 36 | while ( $o = $result->fetch_object() ) { 37 | $ts_last_change = $o->ts_last_change ; 38 | if ( !preg_match ( '/^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})$/' , $ts_last_change , $m ) ) continue ; #die ( "Bad time format in ts_last_change batch #{$o->id}\n" ) ; 39 | $ts_last_change = $m[1].'-'.$m[2].'-'.$m[3].' '.$m[4].':'.$m[5].':'.$m[6] ; 40 | $diff_sec = time() - strtotime ( $ts_last_change ) ; 41 | #print "{$ts_last_change}\n{$diff_sec}\n" ; 42 | if ( $diff_sec < $min_sec_inactive and $o->status != 'INIT' ) continue ; #exit ( 0 ) ; # Oldest batch is still too young 43 | print "Using {$o->id}\n" ; 44 | 45 | if ( $o->status == 'INIT' ) { 46 | if ( !$qs->startBatch ( $o->id ) ) { 47 | print "{$o->id}: " . $qs->last_error_message."\n" ; 48 | exit(0) ; 49 | } 50 | } 51 | 52 | while ( 1 ) { 53 | $qs2 = new QuickStatements ; 54 | if ( !$qs2->runNextCommandInBatch ( $o->id ) ) break ; 55 | $status = $qs2->getBatchStatus ( [$o->id] ) ; 56 | if ( $status[$o->id]['batch']->status != 'RUN' ) break ; 57 | } 58 | break ; 59 | } 60 | 61 | exit ( 0 ) ; 62 | } 63 | 64 | while ( 1 ) { 65 | $worked = iterate() ; 66 | if ( $worked == 0 ) sleep ( 5 ) ; 67 | else if ( $worked == 1 ) sleep ( 1 ) ; 68 | else sleep ( 1 ) ; 69 | } 70 | 71 | ?> -------------------------------------------------------------------------------- /public_html/vue.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | let router ; 4 | let app ; 5 | let wd = new WikiData() ; 6 | 7 | let config = {} ; 8 | let prop_map = {} ; 9 | let working = false ; 10 | 11 | $(document).ready ( function () { 12 | vue_components.toolname = 'quickstatements' ; 13 | // vue_components.components_base_url = 'https://tools.wmflabs.org/magnustools/resources/vue/' ; // For testing; turn off to use tools-static 14 | Promise.all ( [ 15 | vue_components.loadComponents ( ['wd-date','wd-link','tool-translate','tool-navbar','commons-thumbnail', 16 | 'vue_components/batch_access_mixin.html', 17 | 'vue_components/user.html', 18 | 'vue_components/main-page.html', 19 | 'vue_components/command.html', 20 | 'vue_components/batch-commands.html', 21 | 'vue_components/batch.html', 22 | 'vue_components/batches.html', 23 | 'vue_components/user-page.html', 24 | ] ) , 25 | new Promise(function(resolve, reject) { 26 | $.get ( './config.json' , function (d) { 27 | config = d ; 28 | resolve() ; 29 | } , 'json' ) ; 30 | } ) 31 | ] ) .then ( () => { 32 | const siteConfig = config.sites[config.site] ; 33 | const apiUrl = siteConfig.publicApi || siteConfig.api ; 34 | wd_link_base = siteConfig.pageBase ; 35 | wd_link_wd = wd ; 36 | wd.api = apiUrl + '?callback=?' ; 37 | wd.main_languages.unshift( tt.language ) ; 38 | wd_ns_prefixes = {} ; 39 | for ( var letter in siteConfig.types ) 40 | wd_ns_prefixes[letter] = siteConfig.types[letter].ns_prefix ; 41 | 42 | const routes = [ 43 | { path: '/', component: MainPage , props:true }, 44 | { path: '/batches', component: BatchesPage , props:true }, 45 | { path: '/batches/:user_name', component: BatchesPage , props:true }, 46 | { path: '/batch', component: BatchPage , props:true }, 47 | { path: '/batch/:batch', component: BatchPage , props:true }, 48 | { path: '/user', component: UserPage , props:true }, 49 | { path: '/user/:given_user_name', component: UserPage , props:true }, 50 | { path: '/:url_params', component: MainPage , props:true }, 51 | ] ; 52 | router = new VueRouter({routes}) ; 53 | app = new Vue ( { router } ) .$mount('#app') ; 54 | } ) ; 55 | } ) ; 56 | 57 | -------------------------------------------------------------------------------- /public_html/vue_components/batch_access_mixin.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /public_html/vue_components/user.html: -------------------------------------------------------------------------------- 1 | 12 | 13 | 14 | 80 | -------------------------------------------------------------------------------- /public_html/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 27 | 28 | 29 | 30 | 31 |
32 | 33 | 53 | 54 | 55 | 56 |
57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /public_html/vue_components/batches.html: -------------------------------------------------------------------------------- 1 | 61 | 62 | 63 | 129 | -------------------------------------------------------------------------------- /public_html/vue_components/user-page.html: -------------------------------------------------------------------------------- 1 | 82 | 83 | 84 | 128 | -------------------------------------------------------------------------------- /public_html/vue_components/main-page.html: -------------------------------------------------------------------------------- 1 | 35 | 36 | 109 | -------------------------------------------------------------------------------- /public_html/vue_components/batch-commands.html: -------------------------------------------------------------------------------- 1 | 10 | 11 | 58 | 59 | 170 | -------------------------------------------------------------------------------- /public_html/index_old.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 73 | 74 | 75 | 76 | 77 | 78 | 131 | 132 | 133 |
134 | 135 |
136 |
137 |
138 |

Your token:

139 | 140 |
141 |
142 |
143 | 144 |
145 | 146 |
147 | 148 |
149 |
150 |
151 |

Batch #

152 |
153 |
154 |
155 | 156 |
157 | 158 |
159 | 160 |
161 |
162 |

Batches

163 |
164 |
165 |
166 | 167 |
168 | 169 | 170 | 171 |
172 | 173 |
174 |
175 |
176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 187 |
188 |
189 |
190 |
191 | 192 |
193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 |
202 |
203 |
204 | 205 |
206 | 207 | 208 | 209 | 210 | 232 | 233 | 234 | 256 | 257 | 258 | 259 | 260 | -------------------------------------------------------------------------------- /public_html/api.php: -------------------------------------------------------------------------------- 1 | config) ) return ; 35 | if ( !isset($qs->config->valid_origin) ) return ; 36 | if ( $qs->config->valid_origin == '' ) return ; 37 | $valid_origin = $qs->config->valid_origin ; 38 | if ( !is_array($valid_origin) ) $valid_origin = [ $valid_origin ] ; 39 | $origin = get_origin() ; 40 | if ( in_array($origin,$valid_origin) ) return ; // OK 41 | fin('Invalid origin'); 42 | } 43 | 44 | $qs = new QuickStatements ; 45 | $out = [ 'status' => 'OK' ] ; 46 | $action = get_request ( 'action' , '' ) ; 47 | 48 | if ( isset ( $_REQUEST['oauth_verifier'] ) ) { 49 | $oa = $qs->getOA() ; // Answer to OAuth 50 | header( "Location: " . $qs->getToolBase() ); 51 | exit(0) ; 52 | } 53 | 54 | if ( $action == 'import' ) { 55 | 56 | ini_set('memory_limit','2500M'); 57 | 58 | $format = get_request ( 'format' , 'v1' ) ; 59 | $username = get_request ( 'username' , '' ) ; 60 | $token = get_request ( 'token' , '' ) ; 61 | $temporary = get_request ( 'temporary' , false ) ; 62 | $openpage = get_request ( 'openpage' , 0 ) * 1 ; 63 | $submit = get_request ( 'submit' , false ) ; 64 | $data = get_request ( 'data' , '' ) ; 65 | $compress = get_request ( 'compress' , 1 ) * 1 ; 66 | $site = get_request ( 'site' , '' ) ; 67 | $out = $qs->importData ( $data , $format , false ) ; 68 | if ( $compress ) { 69 | $qs->use_command_compression = true ; 70 | $out['data']['commands'] = $qs->compressCommands ( $out['data']['commands'] ) ; 71 | } 72 | $out['debug']['format'] = $format ; 73 | $out['debug']['temporary'] = $temporary ; 74 | $out['debug']['openpage'] = $openpage ; 75 | if ( $temporary ) { 76 | $dir = './tmp' ; 77 | if ( !file_exists($dir) ) mkdir ( $dir ) ; 78 | $filename = tempnam ( $dir , 'qs_' ) ; 79 | $handle = fopen($filename, "w"); 80 | fwrite($handle, json_encode($out) ); 81 | fclose($handle); 82 | $out['data'] = preg_replace ( '|^.+/|' , '' , $filename ) ; 83 | 84 | if ( $openpage ) { 85 | $url = "./#/batch/?tempfile=" . urlencode ( $out['data'] ) ; 86 | if ( $site != '' ) $url .= "&site=" . urlencode($site) ; 87 | print "" ; 88 | exit(0); 89 | } 90 | 91 | fin() ; 92 | } 93 | 94 | if ( $submit ) { 95 | $batchname = get_request ( 'batchname' , '' ) ; 96 | 97 | if ( $site != '' ) { 98 | $qs->config->site = $site ; 99 | $out['site'] = $site ; 100 | } 101 | $user_id = $qs->getUserIDfromNameAndToken ( $username , $token ) ; 102 | if ( !isset($user_id) ) { 103 | unset ( $out['data'] ) ; 104 | fin ( "User name and token do not match" ) ; 105 | } 106 | if ( !$qs->fillOA ( $user_id ) ) { 107 | unset ( $out['data'] ) ; 108 | fin ( "Problem generating OAuth signature; user '{}' needs to have submitted a batch namually at least once before" ) ; 109 | } 110 | 111 | $batch_id = $qs->addBatch ( $out['data']['commands'] , $user_id , $batchname , $site ) ; 112 | unset ( $out['data'] ) ; 113 | if ( $batch_id === false ) { 114 | $out['status'] = $qs->last_error_message ; 115 | } else { 116 | $out['batch_id'] = $batch_id ; 117 | } 118 | } 119 | 120 | } else if ( $action == 'oauth_redirect' ) { 121 | 122 | $oa = $qs->getOA() ; 123 | $oa->doAuthorizationRedirect($qs->getToolBase() . 'api.php') ; 124 | exit(0) ; 125 | 126 | } else if ( $action == 'get_token' ) { 127 | 128 | $force_generate = get_request ( 'force_generate' , 0 ) * 1 ; 129 | $oa = $qs->getOA() ; 130 | $ili = $oa->isAuthOK() ; # Is Logged In 131 | $out['data'] = (object) [] ; 132 | if ( $ili ) { 133 | $cr = $oa->getConsumerRights() ; 134 | $user_name = $cr->query->userinfo->name ; 135 | $out['data']->token = $qs->generateToken ( $user_name , $force_generate ) ; 136 | } 137 | $out['data']->is_logged_in = $ili ; 138 | 139 | } else if ( $action == 'is_logged_in' ) { 140 | 141 | $oa = $qs->getOA() ; 142 | $ili = $oa->isAuthOK() ; 143 | $out['data'] = (object) [] ; 144 | if ( $ili ) { 145 | $out['data'] = $oa->getConsumerRights() ; 146 | } 147 | $out['data']->is_logged_in = $ili ; 148 | 149 | } else if ( $action == 'get_batch_info' ) { 150 | 151 | $batch = get_request ( 'batch' , '0' ) * 1 ; 152 | 153 | if ( $batch == 0 ) { 154 | $out['status'] = 'Missing batch number' ; 155 | } else { 156 | $out['data'] = $qs->getBatchStatus ( array($batch) ) ; 157 | } 158 | 159 | } else if ( $action == 'get_batches_info' ) { 160 | 161 | $out['debug'] = $_REQUEST ; 162 | 163 | $user = get_request ( 'user' , '' ) ; 164 | $limit = get_request ( 'limit' , '20' ) * 1 ; 165 | $offset = get_request ( 'offset' , '0' ) * 1 ; 166 | 167 | $db = $qs->getDB() ; 168 | $sql = "SELECT DISTINCT batch.id AS id FROM batch" ; 169 | if ( $user != '' ) $sql .= ",{$qs->auth_db}.user" ; 170 | 171 | $conditions = [] ; 172 | if ( $user != '' ) $conditions[] = "user.id=batch.user AND user.name='" . $db->real_escape_string($user) . "'" ; 173 | if ( count($conditions) > 0 ) $sql .= ' WHERE ' . implode ( ' AND ' , $conditions ) ; 174 | 175 | $sql .= " ORDER BY ts_last_change DESC" ; 176 | $sql .= " LIMIT $limit" ; 177 | if ( $offset != 0 ) $sql .= " OFFSET $offset" ; 178 | 179 | if(!$result = $db->query($sql)) { 180 | $out['status'] = $db->error ; 181 | } else { 182 | $batches = [] ; 183 | while ( $o = $result->fetch_object() ) $batches[] = $o->id ; 184 | $out['data'] = $qs->getBatchStatus ( $batches ) ; 185 | } 186 | 187 | } else if ( $action == 'get_commands_from_batch' ) { 188 | 189 | $batch_id = get_request ( 'batch' , 0 ) * 1 ; 190 | $start = get_request ( 'start' , 0 ) * 1 ; 191 | $limit = get_request ( 'limit' , 0 ) * 1 ; 192 | $filter = get_request ( 'filter' , '' ) ; 193 | 194 | $db = $qs->getDB() ; 195 | $sql = "SELECT * FROM command WHERE batch_id={$batch_id} AND num>={$start}" ; // num BETWEEN {$start} AND {$end} 196 | if ( $filter != '' ) { 197 | $filter = explode ( ',' , $filter ) ; 198 | foreach ( $filter AS $k => $v ) { 199 | $v = $db->real_escape_string ( trim ( strtoupper ( $v ) ) ) ; 200 | $filter[$k] = $v ; 201 | } 202 | $sql .= " AND `status` IN ('" . implode("','",$filter) . "')" ; 203 | } 204 | $sql .= " ORDER BY num LIMIT {$limit}" ; 205 | 206 | if(!$result = $db->query($sql)) { 207 | $out['status'] = $db->error ; 208 | } else { 209 | $batches = [] ; 210 | $out['data'] = [] ; 211 | while ( $o = $result->fetch_object() ) { 212 | $o->json = json_decode ( $o->json ) ; 213 | $out['data'][] = $o ; 214 | } 215 | } 216 | 217 | } else if ( $action == 'run_single_command' ) { 218 | 219 | validate_origin(); 220 | $site = get_request ( 'site' , '' ) ; 221 | if ( !$qs->setSite ( $site ) ) { 222 | $out['status'] = "Error while setting site '{$site}': " . $qs->last_error_message ; 223 | } else { 224 | 225 | $oa = $qs->getOA() ; 226 | $oa->delay_after_create_s = 0 ; 227 | $oa->delay_after_edit_s = 0 ; 228 | 229 | $qs->last_item = get_request ( 'last_item' , '' ) ; 230 | $command = json_decode ( get_request ( 'command' , '' ) ) ; 231 | if ( $command == null ) { 232 | $out['status'] = 'Bad command JSON' ; 233 | $out['debug'] = get_request ( 'command' , '' ) ; 234 | } else { 235 | $out['command'] = $qs->runSingleCommand ( $command ) ; 236 | $out['last_item'] = $qs->last_item ; 237 | } 238 | } 239 | 240 | } else if ( $action == 'start_batch' or $action == 'stop_batch' ) { 241 | 242 | $batch_id = get_request ( 'batch' , 0 ) * 1 ; 243 | 244 | $res = false ; 245 | if ( $action == 'start_batch' ) $res = $qs->userChangeBatchStatus ( $batch_id , 'INIT' ) ; 246 | if ( $action == 'stop_batch' ) $res = $qs->userChangeBatchStatus ( $batch_id , 'STOP' ) ; 247 | 248 | if ( !$res ) { 249 | $out['status'] = $qs->last_error_message ; 250 | } 251 | 252 | } else if ( $action == 'run_batch' ) { 253 | 254 | $user_id = $qs->getCurrentUserID() ; 255 | $name = trim ( get_request ( 'name' , '' ) ) ; 256 | $site = strtolower ( trim ( get_request ( 'site' , '' ) ) ) ; 257 | if ( $user_id === false ) { 258 | $out['status'] = $qs->last_error_message ; 259 | } else { 260 | $commands = json_decode ( get_request('commands','[]') ) ; 261 | $batch_id = $qs->addBatch ( $commands , $user_id , $name , $site ) ; 262 | if ( $batch_id === false ) { 263 | $out['status'] = $qs->last_error_message ; 264 | } else { 265 | $out['batch_id'] = $batch_id ; 266 | } 267 | } 268 | // $qs->addBatch ( $commands ) ; 269 | 270 | } else if ( $action == 'get_batch' ) { 271 | 272 | $id = get_request ( 'id' , '' ) ; 273 | $out['id'] = $id ; 274 | $out['data'] = $qs->getBatch ( $id ) ; 275 | 276 | } else if ( $action == 'reset_errors' ) { 277 | 278 | $batch_id = get_request ( 'batch_id' , 0 ) * 1 ; 279 | if ( $batch_id <= 0 ) fin("Bad batch ID #{$batch_id}") ; 280 | 281 | $out['init'] = 0 ; 282 | $db = $qs->getDB() ; 283 | $sql = "SELECT * FROM command WHERE batch_id={$batch_id} AND `status`='ERROR'" ; 284 | if(!$result = $db->query($sql)) fin($db->error) ; 285 | 286 | $ids = [] ; 287 | while ( $o = $result->fetch_object() ) { 288 | if ( stristr($o->message,'no-such-entity') ) continue ; // No such item exists, no point in re-trying 289 | if ( !isset($o->json) ) continue ; // No actual command 290 | $j = @json_decode ( $o->json ) ; 291 | if ( !isset($j) or $j === null ) continue ; // Bad JSON 292 | if ( isset($j->item) and $j->item == 'LAST' ) continue ; // Don't know which item to re-apply for 293 | if ( isset($j->action) and $j->action == 'CREATE' and !isset($j->data) ) continue ; // Empty CREATE command 294 | $ids[] = $o->id ; 295 | } 296 | 297 | if ( count($ids) > 0 ) { 298 | $sql = "UPDATE command SET `status`='INIT' WHERE id IN (" . implode(',',$ids) . ")" ; 299 | $out['sql'] = $sql ; 300 | if(!$result = $db->query($sql)) fin($db->error) ; 301 | 302 | $out['init'] = count($ids) ; 303 | $res = $qs->userChangeBatchStatus ( $batch_id , 'INIT' ) ; 304 | if ( !$res ) { 305 | $out['status'] = $qs->last_error_message ; 306 | } 307 | } 308 | 309 | } 310 | 311 | fin(); 312 | 313 | ?> 314 | -------------------------------------------------------------------------------- /public_html/vue_components/command.html: -------------------------------------------------------------------------------- 1 | 52 | 53 | 92 | 93 | 255 | 256 | 257 | 303 | -------------------------------------------------------------------------------- /public_html/vue_components/batch.html: -------------------------------------------------------------------------------- 1 | 13 | 14 | 140 | 141 | 142 | 464 | -------------------------------------------------------------------------------- /public_html/quickstatements.js: -------------------------------------------------------------------------------- 1 | // For data tables, see https://datatables.net/examples/api/add_row.html 2 | 3 | var QuickStatements = { 4 | 5 | api : './api.php' , 6 | config : {} , 7 | params : {} , 8 | data : {} , 9 | oauth : {} , 10 | sites : {} , // Loaded from sites.json 11 | types : {} , 12 | run_state : { running:false } , 13 | batch_update_interval : 5000 , 14 | 15 | init : function () { 16 | var me = this ; 17 | 18 | var running = 3 ; 19 | function fin () { 20 | running-- ; 21 | if ( running > 0 ) return ; 22 | 23 | me.tt.addILdropdown ( $('#interface_language_wrapper') ) ; 24 | me.updateUserInfo() ; 25 | me.params = me.getUrlVars() ; 26 | 27 | $('#import_v1_dialog').on('shown.bs.modal', function () { $('#v1_commands').val('').focus() }) 28 | $('#v1_import').click ( function(){me.onImportV1(); $('#import_v1_dialog').modal('hide') } ) ; 29 | 30 | $('#import_csv_dialog').on('shown.bs.modal', function () { $('#csv_input').val('').focus() }) 31 | $('#csv_import').click ( function(){me.onImportCSV(); $('#import_csv_dialog').modal('hide') } ) ; 32 | 33 | $('#main_table').DataTable ( { 34 | ordering:false, 35 | info:false 36 | } ); 37 | 38 | $('#link_import_qs1').click ( me.onClickImportV1 ) ; 39 | $('#link_import_csv').click ( me.onClickImportCSV ) ; 40 | $('#run').click ( function () { me.run ( false ) ; return false } ) ; 41 | $('#run_background').click ( function () { me.run ( true ) ; return false } ) ; 42 | $('#stop').click ( function () { me.stop() ; return false } ) ; 43 | 44 | if ( typeof me.params.v1 != 'undefined' ) me.importFromV1 ( me.params.v1 ) ; 45 | if ( typeof me.params.csv != 'undefined' ) me.importFromCSV ( me.params.csv ) ; 46 | 47 | me.updateUnlabeledItems() ; 48 | 49 | if ( me.params.mode == 'batch' ) me.run_state.batch_id = me.params.batch ; 50 | if ( me.params.mode == 'batches' ) { 51 | me.run_state.filters = {} ; 52 | if ( typeof me.params.user != 'undefined' ) me.run_state.filters.user = me.params.user ; 53 | } 54 | me.switchMode ( me.params.mode ) ; 55 | } 56 | 57 | me.tt = new ToolTranslation ( { tool:'quickstatements' , language:me.lang() , fallback:'en' , callback : function () { fin() } } ) ; 58 | 59 | $.get ( 'config.json' , function ( d ) { 60 | me.config = d ; 61 | me.sites = d.sites ; 62 | me.setSite ( me.config.site ) ; 63 | fin() ; 64 | } ) ; 65 | 66 | me.oauth = { is_logged_in:false } ; 67 | $.post ( me.api , { action:'is_logged_in' }, 'json' ) 68 | .then( function( d ) { me.oauth = d.data; } ) 69 | .always( fin ); 70 | } , 71 | 72 | setSite : function ( site ) { 73 | var me = this ; 74 | me.site = site ; 75 | me.types = me.sites[me.site].types ; 76 | } , 77 | 78 | getSiteAPI : function () { 79 | var me = this ; 80 | return me.sites[me.site].publicApi || me.sites[me.site].api ; 81 | } , 82 | 83 | getSitePageURL : function ( page ) { 84 | var me = this ; 85 | return me.sites[me.site].pageBase + encodeURIComponent ( page.replace(/ /g,'_') ) ; 86 | } , 87 | 88 | lang : function () { 89 | return 'en' ; // FIXME current language 90 | } , 91 | 92 | stop : function () { 93 | var me = this ; 94 | me.run_state.running = false ; 95 | if ( typeof me.run_state.batch_watcher != 'undefined' ) clearInterval ( me.run_state.batch_watcher ) ; 96 | $('#stop_buttons button').prop ( 'disabled' , true ) ; 97 | $('#run_buttons button').prop ( 'disabled' , false ) ; 98 | $('#stop_buttons').hide() ; 99 | } , 100 | 101 | ts2string : function ( ts ) { 102 | return ts.substr(0,4) + '-' + ts.substr(4,2) + '-' + ts.substr(6,2) + ' ' + ts.substr(8,2) + ':' + ts.substr(10,2) + ':' + ts.substr(12,2) ; 103 | } , 104 | 105 | canUserStopBatch : function ( batch_user_name ) { 106 | var me = this ; 107 | if ( typeof me.oauth.query == 'undefined' ) return false ; 108 | if ( typeof me.oauth.query.userinfo == 'undefined' ) return false ; 109 | if ( me.oauth.query.userinfo.name == batch_user_name ) return true ; 110 | if ( typeof me.oauth.query.userinfo.groups == 'undefined' ) return false ; 111 | 112 | var ret = false ; 113 | $.each ( me.oauth.query.userinfo.groups , function ( k , v ) { 114 | if ( v == 'sysop' ) ret = true ; 115 | } ) ; 116 | return ret ; 117 | } , 118 | 119 | safeHTML : function ( h ) { 120 | return h.replace(//,'>').replace(/&/,'&') 121 | } , 122 | 123 | getBatchButtons : function ( d2 ) { 124 | var me = this ; 125 | var h = '' ; 126 | var batch_id = d2.batch.id ; 127 | if ( me.canUserStopBatch(d2.batch.user_name) ) { 128 | h += "
" ; 129 | 130 | h += "" ; 556 | h += "" ; 557 | h += "
" ; 558 | h += "
" ; 559 | h += "" ; 560 | form.html(h).show() ; 561 | me.tt.updateInterface(form) ; 562 | 563 | me.addTypeahead ( $(container.find('div.pq_typeahead')) ) ; 564 | $(container.find('form')).submit ( function () { me.onSubmitPQ ( container , true ) ; return false } ) ; 565 | $(container.find('button.cancel')).click ( function () { me.onSubmitPQ ( container , false ) ; return false } ) ; 566 | } , 567 | 568 | updateRef : function ( json_string , value ) { 569 | var me = this ; 570 | if ( typeof json_string == 'undefined' ) return ; 571 | if ( json_string == '' ) return ; 572 | var j = JSON.parse ( json_string ) ; 573 | if ( typeof j == 'undefined' ) return ; 574 | if ( typeof j.cmdnum != 'undefined' ) { 575 | var i = me.data.commands[j.cmdnum] ; 576 | while ( ( m = j.attr.match ( /^(.+?)\.(.+)$/ ) ) != null ) { 577 | i = i[m[1]] ; 578 | j.attr = m[2] ; 579 | } 580 | i[j.attr] = value ; 581 | } else { 582 | console.log ( "COUND NOT STORE REF" , j , value ) ; 583 | } 584 | } , 585 | 586 | onSubmitPQ : function ( container , do_store ) { 587 | var me = this ; 588 | var ta = $(container.find('div.pq_typeahead')) ; 589 | var input = $(container.find('input.typeahead_input')) ; 590 | var pq = input.attr('pq') ; 591 | var title = input.val() ; 592 | var pqv = container.find('div.pq_value') ; 593 | if ( do_store && pq != '' ) { 594 | me.updateRef ( container.attr('ref') , pq ) ; // store in data structure 595 | pqv.html ( me.renderPQvalue ( pq ) ) ; 596 | container.attr ( { pq:pq } ) ; 597 | } 598 | ta.html('').hide() ; 599 | pqv.show() ; 600 | container.find('div.pq_button').show() ; 601 | } , 602 | 603 | addTypeahead : function ( o ) { 604 | var me = this ; 605 | me.lastTypeahead = '' ; 606 | var input = $(o.find('input.typeahead_input')) ; 607 | var select = $(o.find('ul.pq_dropdown')) ; 608 | input.keyup ( function () { 609 | me.typeAhead ( o , input , select ) ; 610 | } ) ; 611 | input.focus() ; 612 | me.typeAhead ( o , input , select ) ; 613 | } , 614 | 615 | typeAhead : function ( o , input , select ) { 616 | var me = this ; 617 | var type = o.attr('type') ; 618 | var l = me.lang() ; 619 | var s = input.val() ; 620 | if ( s == me.lastTypeahead ) return ; 621 | me.lastTypeahead = s ; 622 | select.html ( '' ) ; 623 | $.getJSON ( me.getSiteAPI()+"?callback=?" , { 624 | action:'wbsearchentities', 625 | search:s, 626 | language:l, 627 | type:type, 628 | format:'json' 629 | } , function ( d ) { 630 | var h = '' ; 631 | $.each ( d.search , function ( k , v ) { 632 | h += "
  • " ; 633 | h += "
    " + me.htmlSafe(v.label) + " (" + v.title + ")
    " ; 634 | h += "
    " + me.htmlSafe(v.description||'') + "
    " ; 635 | h += "
  • " ; 636 | } ) ; 637 | select.html ( h ) ; 638 | select.find('li').click ( function () { 639 | var a = $(this) ; 640 | var pq = a.attr('pq').replace(/^.+:/,'') ; 641 | var container = $(input.parents('div.pq_container').get(0)) ; 642 | input.attr ( { pq:pq } ) ; 643 | me.onSubmitPQ ( container , true ) ; 644 | } ) ; 645 | } ) ; 646 | } , 647 | 648 | importFromV1 : function ( v1 ) { 649 | var me = this ; 650 | if ( v1.length < 1000 ) location.hash = 'v1='+v1 ; 651 | $.post ( me.api , { 652 | action:'import', 653 | data:v1, 654 | format:'v1', 655 | persistent:0, 656 | } , function ( d ) { 657 | // TODO status/error check 658 | me.data = d.data ; 659 | me.setupTableFromCommands() ; 660 | } , 'json' ) ; 661 | } , 662 | 663 | importFromCSV : function ( csv ) { 664 | var me = this ; 665 | if ( csv.length < 1000 ) location.hash = 'csv='+csv ; 666 | $.post ( me.api , { 667 | action:'import', 668 | data:csv, 669 | format:'csv', 670 | persistent:0, 671 | } , function ( d ) { 672 | // TODO status/error check 673 | me.data = d.data ; 674 | me.setupTableFromCommands() ; 675 | } , 'json' ) ; 676 | } , 677 | 678 | renderPQvalue : function ( i ) { 679 | var me = this ; 680 | var html = '???' ; 681 | if ( i == 'LAST' ) { 682 | html = 'LAST ITEM' ; 683 | } else if ( i.match ( /^(?:[PQL]\d+|L\d+-[FS]\d+)$/i ) ) { // ENTITY 684 | var letter = i.substr(0,1).toUpperCase() ; 685 | var title = me.types[letter].ns_prefix + i.replace('-', '#') ; 686 | html = "" + i + " [" + i + "]" 687 | 688 | } else { // DUNNO 689 | html = me.htmlSafe ( i ) ; 690 | } 691 | return html ; 692 | } , 693 | 694 | renderPQ : function ( i , ref ) { 695 | var me = this ; 696 | html = "
    " ; 700 | html += "
    " ; 701 | html += "
    FORM
    " ; 702 | html += "
    " ; 703 | return html ; 704 | } , 705 | 706 | htmlSafe : function ( s ) { 707 | var me = this ; 708 | if ( typeof s == 'undefined' ) return '' ; 709 | var html_safe = (''+s).replace(/&/g,'&').replace(//g,'>').replace(/"/g,'"').replace(/'/g,''').replace(/\//g,'/') ; 710 | return html_safe ; 711 | } , 712 | 713 | renderString : function ( s , ref ) { 714 | var me = this ; 715 | var h = '' ; 716 | h += "
    " + JSON.stringify(v) + "" ; 776 | if ( v.type == 'novalue' || v.type == 'somevalue' ) return "" + v.type + "" ; 777 | if ( v.type == 'wikibase-entityid' ) { 778 | ref.attr += '.value.id' ; 779 | return me.renderPQ ( v.value.id , ref ) ; 780 | } 781 | if ( v.type == 'string' ) return me.renderString ( v.value , ref ) ; 782 | if ( v.type == 'time' ) return me.renderTime ( v.value , ref ) ; 783 | if ( v.type == 'globecoordinate' ) return me.renderCoordinate ( v.value , ref ) ; 784 | if ( v.type == 'quantity' ) return me.renderQuantity ( v.value , ref ) ; 785 | if ( v.type == 'monolingualtext' ) return me.renderMonolingualtext ( v.value , ref ) ; 786 | return "" + JSON.stringify(v) + "" ; 787 | } , 788 | 789 | renderAction : function ( cmd ) { 790 | var ret = cmd.action.toUpperCase() ; 791 | if ( typeof cmd.what != 'undefined' && cmd.what != 'statement' ) ret += " " + cmd.what.toUpperCase() ; 792 | return ret ; 793 | } , 794 | 795 | renderQualifier : function ( qualifier , ref ) { 796 | var me = this ; 797 | var h = '' ; 798 | ref.attr = 'qualifier.prop' ; 799 | h += "
    " + me.renderPQ ( qualifier.prop , ref ) + "
    " ; 800 | ref.attr = 'qualifier.value' ; 801 | h += "
    " + me.renderValue ( qualifier.value , ref ) + "
    " ; 802 | return h ; 803 | } , 804 | 805 | renderSources : function ( sources , ref ) { 806 | var me = this ; 807 | var h = '' ; 808 | $.each ( sources , function ( k , v ) { 809 | if ( k > 0 ) h += "
    " ; 810 | ref.attr = 'sources.'+k+'.prop' ; 811 | h += "
    " + me.renderPQ ( v.prop , ref ) + "
    " ; 812 | ref.attr = 'sources.'+k+'.value' ; 813 | h += "
    " + me.renderValue ( v.value , ref ) + "
    " ; 814 | } ) ; 815 | return h ; 816 | } , 817 | 818 | wrapStatusAlert : function ( s , key , msg ) { 819 | var me = this ; 820 | var ret = '