├── README.md └── kibana.php /README.md: -------------------------------------------------------------------------------- 1 | # Website Utils 2 | 3 | This repository contains utilities we use on phpBB.com and related websites (area51). 4 | -------------------------------------------------------------------------------- /kibana.php: -------------------------------------------------------------------------------- 1 | 0, 'KB'=>1, 'MB'=>2, 'GB'=>3, 'TB'=>4, 'PB'=>5, 'EB'=>6, 'ZB'=>7, 'YB'=>8); 162 | $sUnit = strtoupper(trim(substr($p_sFormatted, -2))); 163 | if (intval($sUnit) !== 0) { 164 | $sUnit = 'B'; 165 | } 166 | if (!in_array($sUnit, array_keys($aUnits))) { 167 | return false; 168 | } 169 | $iUnits = trim(substr($p_sFormatted, 0, strlen($p_sFormatted) - 2)); 170 | if (!intval($iUnits) == $iUnits) { 171 | return false; 172 | } 173 | return $iUnits * pow(1024, $aUnits[$sUnit]); 174 | } 175 | 176 | function request($method, $endpoint, $data = null) { 177 | $ci = curl_init(); 178 | curl_setopt($ci, CURLOPT_URL, 'http://TOCHANGE_ELASTICSERACH_IP:9200/'.$endpoint); 179 | curl_setopt($ci, CURLOPT_PORT, 9200); 180 | curl_setopt($ci, CURLOPT_TIMEOUT, 200); 181 | curl_setopt($ci, CURLOPT_RETURNTRANSFER, 1); 182 | curl_setopt($ci, CURLOPT_FORBID_REUSE, 0); 183 | 184 | if (!empty($data)) { 185 | curl_setopt($ci, CURLOPT_POSTFIELDS, $data); 186 | } 187 | 188 | curl_setopt($ci, CURLOPT_CUSTOMREQUEST, $method); 189 | 190 | $result = curl_exec($ci); 191 | $httpcode = curl_getinfo($ci, CURLINFO_HTTP_CODE); 192 | if ($httpcode >= 400) { 193 | echo $result; 194 | } 195 | } 196 | 197 | function push($data) { 198 | request('PUT', '_bulk', $data); 199 | } 200 | 201 | $start = microtime(true); 202 | 203 | request('DELETE', 'phpbb_questionnaire'); 204 | printf("[%s] Index deleted\n", gmdate('H:i:s', microtime(true) - $start)); 205 | request('PUT', 'phpbb_questionnaire', $schema); 206 | printf("[%s] Index created\n", gmdate('H:i:s', microtime(true) - $start)); 207 | 208 | $query = 'SELECT `install_id`, 209 | `entry_id`, 210 | `forwarded`, 211 | `PHP_version`, 212 | `PHP_sapi`, 213 | `PHP_int_size`, 214 | `PHP_safe_mode`, 215 | `PHP_open_basedir`, 216 | `PHP_memory_limit`, 217 | `PHP_allow_url_fopen`, 218 | `PHP_allow_url_include`, 219 | `PHP_file_uploads`, 220 | `PHP_upload_max_filesize`, 221 | `PHP_post_max_size`, 222 | `PHP_disable_functions`, 223 | `PHP_disable_classes`, 224 | `PHP_enable_dl`, 225 | `PHP_magic_quotes_gpc`, 226 | `PHP_register_globals`, 227 | `PHP_filter.default` as PHP_filter, 228 | `PHP_zend.ze1_compatibility_mode` as PHP_zend_ze1_compatibility_mode, 229 | `PHP_unicode.semantics` as PHP_unicode_semantics, 230 | `PHP_zend_thread_safty`, 231 | `System_os`, 232 | `System_httpd`, 233 | `System_ip`, 234 | `phpBB_config_version`, 235 | `phpBB_dbms`, 236 | `phpBB_config_num_posts`, 237 | `phpBB_config_num_topics`, 238 | `phpBB_config_num_users`, 239 | `timestamp` FROM statistics_main'; 240 | 241 | $detailsQuery = 'SELECT d.*, m.timestamp FROM statistics_details d INNER JOIN statistics_main m ON m.entry_id = d.entry_id'; 242 | 243 | $mysqli = new mysqli('SERVER', 'USER', 'PASS', 'DB'); 244 | $st = $mysqli->prepare($query); 245 | 246 | $res = $mysqli->query($query, MYSQLI_USE_RESULT); 247 | 248 | printf("[%s] Pushing main\n", gmdate('H:i:s', microtime(true) - $start)); 249 | 250 | $i = 0; 251 | $j = 0; 252 | $data = ''; 253 | while ($row = $res->fetch_assoc()) { 254 | $i++; 255 | $j++; 256 | 257 | $row['PHP_disable_functions'] = explode(',', $row['PHP_disable_functions']); 258 | $row['PHP_disable_classes'] = explode(',', $row['PHP_disable_classes']); 259 | $row['PHP_upload_max_filesize'] = toByteSize($row['PHP_upload_max_filesize']); 260 | $row['PHP_memory_limit'] = toByteSize($row['PHP_memory_limit']); 261 | $row['forwarded'] = (bool) $row['forwarded']; 262 | $row['PHP_safe_mode'] = (bool) $row['PHP_safe_mode']; 263 | $row['PHP_open_basedir'] = (bool) $row['PHP_open_basedir']; 264 | $row['PHP_allow_url_fopen'] = (bool) $row['PHP_allow_url_fopen']; 265 | $row['PHP_allow_url_include'] = (bool) $row['PHP_allow_url_include']; 266 | $row['PHP_file_uploads'] = (bool) $row['PHP_file_uploads']; 267 | $row['PHP_enable_dl'] = (bool) $row['PHP_enable_dl']; 268 | $row['PHP_magic_quotes_gpc'] = (bool) $row['PHP_magic_quotes_gpc']; 269 | $row['PHP_register_globals'] = (bool) $row['PHP_register_globals']; 270 | $row['PHP_zend_ze1_compatibility_mode'] = (bool) $row['PHP_zend_ze1_compatibility_mode']; 271 | $row['PHP_unicode_semantics'] = (bool) $row['PHP_unicode_semantics']; 272 | $row['PHP_zend_thread_safty'] = (bool) $row['PHP_zend_thread_safty']; 273 | $row['PHP_int_size'] = (int) $row['PHP_int_size']; 274 | $row['phpBB_config_num_posts'] = (int) $row['phpBB_config_num_posts']; 275 | $row['phpBB_config_num_topics'] = (int) $row['phpBB_config_num_topics']; 276 | $row['phpBB_config_num_users'] = (int) $row['phpBB_config_num_users']; 277 | $row['timestamp'] = strtotime($row['timestamp']); 278 | $row['PHP_version_full'] = $row['PHP_version']; 279 | $phpVersion = explode('.', $row['PHP_version']); 280 | $row['PHP_version'] = $phpVersion[0].'.'.$phpVersion[1]; 281 | $phpBBVersion = explode('.', $row['phpBB_config_version']); 282 | $row['phpBB_config_version_major'] = $phpBBVersion[0].'.'.$phpBBVersion[1]; 283 | 284 | $data .= json_encode(['create' => ['_index' => 'phpbb_questionnaire', '_type' => 'statistics_main', '_id' => $row['entry_id']]])."\n"; 285 | $data .= json_encode($row)."\n"; 286 | 287 | if ($i >= 50000) { 288 | printf("[%s] %d entries pushed to easticsearch\n", $j, gmdate('H:i:s', microtime(true) - $start)); 289 | push($data); 290 | $i = 0; 291 | $data = ''; 292 | } 293 | } 294 | push($data); 295 | printf("[%s] %d entries pushed to easticsearch\n", $j, gmdate('H:i:s', microtime(true) - $start)); 296 | 297 | printf("[%s] Pushing details\n", gmdate('H:i:s', microtime(true) - $start)); 298 | 299 | $res = $mysqli->query($detailsQuery, MYSQLI_USE_RESULT); 300 | 301 | $i = 0; 302 | $j = 0; 303 | $data = ''; 304 | while ($row = $res->fetch_assoc()) { 305 | $i++; 306 | $j++; 307 | 308 | $row['value_int'] = (int) $row['value_int']; 309 | $row['timestamp'] = strtotime($row['timestamp']); 310 | 311 | $data .= json_encode(['create' => ['_index' => 'phpbb_questionnaire', '_type' => 'statistics_details', '_id' => $row['entry_id'].$row['variable']]])."\n"; 312 | $data .= json_encode($row)."\n"; 313 | 314 | if ($i >= 50000) { 315 | printf("[%s] %d entries pushed to easticsearch\n", $j, gmdate('H:i:s', microtime(true) - $start)); 316 | push($data); 317 | $i = 0; 318 | $data = ''; 319 | } 320 | } 321 | push($data); 322 | printf("[%s] %d entries pushed to easticsearch\n", $j, gmdate('H:i:s', microtime(true) - $start)); 323 | --------------------------------------------------------------------------------