', var_export($mixed, true), ''; 61 | } else { 62 | echo $param; 63 | } 64 | } 65 | } 66 | 67 | public function setDebug(bool $debug): void { 68 | $this->debug = $debug; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /backend/common/Import.php: -------------------------------------------------------------------------------- 1 | d = Debug::getInstance(); 19 | $this->cli = (\PHP_SAPI === 'cli'); 20 | $this->d->setDebug($this->verbose); 21 | } 22 | 23 | /** 24 | * @throws \Exception 25 | */ 26 | public function start(\DateTime $dateStart): void { 27 | $sources = Config::$cfg['general']['sources']; 28 | $processedSources = 0; 29 | 30 | // if in force mode, reset existing data 31 | if ($this->force === true) { 32 | if ($this->cli === true) { 33 | echo 'Resetting existing data...' . \PHP_EOL; 34 | } 35 | Config::$db->reset([]); 36 | } 37 | 38 | // start progress bar (CLI only) 39 | $daysTotal = ((int) $dateStart->diff(new \DateTime())->format('%a') + 1) * \count($sources); 40 | if ($this->cli === true && $this->quiet === false) { 41 | echo \PHP_EOL . \mbolli\nfsen_ng\vendor\ProgressBar::start($daysTotal, 'Processing ' . \count($sources) . ' sources...'); 42 | } 43 | 44 | // process each source, e.g. gateway, mailserver, etc. 45 | foreach ($sources as $nr => $source) { 46 | $sourcePath = Config::$cfg['nfdump']['profiles-data'] . \DIRECTORY_SEPARATOR . Config::$cfg['nfdump']['profile']; 47 | if (!file_exists($sourcePath)) { 48 | throw new \Exception('Could not read nfdump profile directory ' . $sourcePath); 49 | } 50 | if ($this->cli === true && $this->quiet === false) { 51 | echo \PHP_EOL . 'Processing source ' . $source . ' (' . ($nr + 1) . '/' . \count($sources) . ')...' . \PHP_EOL; 52 | } 53 | 54 | $date = clone $dateStart; 55 | 56 | // check if we want to continue a stopped import 57 | // assumes the last update of a source is similar to the last update of its ports... 58 | $lastUpdateDb = Config::$db->last_update($source); 59 | 60 | $lastUpdate = null; 61 | if ($lastUpdateDb !== false && $lastUpdateDb !== 0) { 62 | $lastUpdate = (new \DateTime())->setTimestamp($lastUpdateDb); 63 | } 64 | 65 | if ($this->force === false && isset($lastUpdate)) { 66 | $daysSaved = (int) $date->diff($lastUpdate)->format('%a'); 67 | $daysTotal -= $daysSaved; 68 | if ($this->quiet === false) { 69 | $this->d->log('Last update: ' . $lastUpdate->format('Y-m-d H:i'), \LOG_INFO); 70 | } 71 | if ($this->cli === true && $this->quiet === false) { 72 | \mbolli\nfsen_ng\vendor\ProgressBar::setTotal($daysTotal); 73 | } 74 | 75 | // set progress to the date when the import was stopped 76 | $date->setTimestamp($lastUpdateDb); 77 | $date->setTimezone(new \DateTimeZone(date_default_timezone_get())); 78 | } 79 | 80 | // iterate from $datestart until today 81 | while ((int) $date->format('Ymd') <= (int) (new \DateTime())->format('Ymd')) { 82 | $scan = [$sourcePath, $source, $date->format('Y'), $date->format('m'), $date->format('d')]; 83 | $scanPath = implode(\DIRECTORY_SEPARATOR, $scan); 84 | 85 | // set date to tomorrow for next iteration 86 | $date->modify('+1 day'); 87 | 88 | // if no data exists for current date (e.g. .../2017/03/03) 89 | if (!file_exists($scanPath)) { 90 | $this->d->dpr($scanPath . ' does not exist!'); 91 | if ($this->cli === true && $this->quiet === false) { 92 | echo \mbolli\nfsen_ng\vendor\ProgressBar::next(1); 93 | } 94 | continue; 95 | } 96 | 97 | // scan path 98 | $this->d->log('Scanning path ' . $scanPath, \LOG_INFO); 99 | $scanFiles = scandir($scanPath); 100 | 101 | if ($this->cli === true && $this->quiet === false) { 102 | echo \mbolli\nfsen_ng\vendor\ProgressBar::next(1, 'Scanning ' . $scanPath . '...'); 103 | } 104 | 105 | foreach ($scanFiles as $file) { 106 | if (\in_array($file, ['.', '..'], true)) { 107 | continue; 108 | } 109 | 110 | try { 111 | // parse date of file name to compare against last_update 112 | preg_match('/nfcapd\.([0-9]{12})$/', (string) $file, $fileDate); 113 | if (\count($fileDate) !== 2) { 114 | throw new \LengthException('Bad file name format of nfcapd file: ' . $file); 115 | } 116 | $fileDatetime = new \DateTime($fileDate[1]); 117 | } catch (\LengthException $e) { 118 | $this->d->log('Caught exception: ' . $e->getMessage(), \LOG_DEBUG); 119 | continue; 120 | } 121 | 122 | // compare file name date with last update 123 | if ($fileDatetime <= $lastUpdate) { 124 | continue; 125 | } 126 | 127 | // let nfdump parse each nfcapd file 128 | $statsPath = implode(\DIRECTORY_SEPARATOR, \array_slice($scan, 2, 5)) . \DIRECTORY_SEPARATOR . $file; 129 | 130 | try { 131 | // fill source.rrd 132 | $this->writeSourceData($source, $statsPath); 133 | 134 | // write general port data (queries data for all sources, should only be executed when data for all sources exists...) 135 | if ($this->processPorts === true && $nr === \count($sources) - 1) { 136 | $this->writePortsData($statsPath); 137 | } 138 | 139 | // if enabled, process ports per source as well (source_80.rrd) 140 | if ($this->processPortsBySource === true) { 141 | $this->writePortsData($statsPath, $source); 142 | } 143 | } catch (\Exception $e) { 144 | $this->d->log('Caught exception: ' . $e->getMessage(), \LOG_WARNING); 145 | } 146 | } 147 | } 148 | ++$processedSources; 149 | } 150 | if ($processedSources === 0) { 151 | $this->d->log('Import did not process any sources.', \LOG_WARNING); 152 | } 153 | if ($this->cli === true && $this->quiet === false) { 154 | echo \mbolli\nfsen_ng\vendor\ProgressBar::finish(); 155 | } 156 | } 157 | 158 | /** 159 | * @throws \Exception 160 | */ 161 | private function writeSourceData(string $source, string $statsPath): bool { 162 | // set options and get netflow summary statistics (-I) 163 | $nfdump = Nfdump::getInstance(); 164 | $nfdump->reset(); 165 | $nfdump->setOption('-I', null); 166 | $nfdump->setOption('-M', $source); 167 | $nfdump->setOption('-r', $statsPath); 168 | 169 | if ($this->dbUpdatable($statsPath, $source) === false) { 170 | return false; 171 | } 172 | 173 | try { 174 | $input = $nfdump->execute(); 175 | } catch (\Exception $e) { 176 | $this->d->log('Exception: ' . $e->getMessage(), \LOG_WARNING); 177 | 178 | return false; 179 | } 180 | 181 | $date = new \DateTime(mb_substr($statsPath, -12)); 182 | $data = [ 183 | 'fields' => [], 184 | 'source' => $source, 185 | 'port' => 0, 186 | 'date_iso' => $date->format('Ymd\THis'), 187 | 'date_timestamp' => $date->getTimestamp(), 188 | ]; 189 | // $input data is an array of lines looking like this: 190 | // flows_tcp: 323829 191 | foreach ($input as $i => $line) { 192 | if (!\is_string($line)) { 193 | $this->d->log('Got no output of previous command', \LOG_DEBUG); 194 | } 195 | if ($i === 0) { 196 | continue; 197 | } // skip nfdump command 198 | if (!preg_match('/:/', (string) $line)) { 199 | continue; 200 | } // skip invalid lines like error messages 201 | [$type, $value] = explode(': ', (string) $line); 202 | 203 | // we only need flows/packets/bytes values, the source and the timestamp 204 | if (preg_match('/^(flows|packets|bytes)/i', $type)) { 205 | $data['fields'][mb_strtolower($type)] = (int) $value; 206 | } 207 | } 208 | 209 | // write to database 210 | if (Config::$db->write($data) === false) { 211 | throw new \Exception('Error writing to ' . $statsPath); 212 | } 213 | 214 | return true; 215 | } 216 | 217 | /** 218 | * @throws \Exception 219 | */ 220 | private function writePortsData(string $statsPath, string $source = ''): bool { 221 | $ports = Config::$cfg['general']['ports']; 222 | 223 | foreach ($ports as $port) { 224 | $this->writePortData($port, $statsPath, $source); 225 | } 226 | 227 | return true; 228 | } 229 | 230 | /** 231 | * @throws \Exception 232 | */ 233 | private function writePortData(int $port, string $statsPath, string $source = ''): bool { 234 | $sources = Config::$cfg['general']['sources']; 235 | 236 | // set options and get netflow statistics 237 | $nfdump = Nfdump::getInstance(); 238 | $nfdump->reset(); 239 | 240 | if (empty($source)) { 241 | // if no source is specified, get data for all sources 242 | $nfdump->setOption('-M', implode(':', $sources)); 243 | if ($this->dbUpdatable($statsPath, '', $port) === false) { 244 | return false; 245 | } 246 | } else { 247 | $nfdump->setOption('-M', $source); 248 | if ($this->dbUpdatable($statsPath, $source, $port) === false) { 249 | return false; 250 | } 251 | } 252 | 253 | $nfdump->setFilter('dst port ' . $port); 254 | $nfdump->setOption('-s', 'dstport:p'); 255 | $nfdump->setOption('-r', $statsPath); 256 | 257 | try { 258 | $input = $nfdump->execute(); 259 | } catch (\Exception $e) { 260 | $this->d->log('Exception: ' . $e->getMessage(), \LOG_WARNING); 261 | 262 | return false; 263 | } 264 | 265 | // parse and turn into usable data 266 | 267 | $date = new \DateTime(mb_substr($statsPath, -12)); 268 | $data = [ 269 | 'fields' => [ 270 | 'flows' => 0, 271 | 'packets' => 0, 272 | 'bytes' => 0, 273 | ], 274 | 'source' => $source, 275 | 'port' => $port, 276 | 'date_iso' => $date->format('Ymd\THis'), 277 | 'date_timestamp' => $date->getTimestamp(), 278 | ]; 279 | 280 | // process protocols 281 | // headers: ts,te,td,pr,val,fl,flP,ipkt,ipktP,ibyt,ibytP,ipps,ipbs,ibpp 282 | foreach ($input as $i => $line) { 283 | if (!\is_array($line) && $line instanceof \Countable === false) { 284 | continue; 285 | } // skip anything uncountable 286 | if (\count($line) !== 14) { 287 | continue; 288 | } // skip anything invalid 289 | if ($line[0] === 'ts') { 290 | continue; 291 | } // skip header 292 | 293 | $proto = mb_strtolower((string) $line[3]); 294 | 295 | // add protocol-specific 296 | $data['fields']['flows_' . $proto] = (int) $line[5]; 297 | $data['fields']['packets_' . $proto] = (int) $line[7]; 298 | $data['fields']['bytes_' . $proto] = (int) $line[9]; 299 | 300 | // add to overall stats 301 | $data['fields']['flows'] += (int) $line[5]; 302 | $data['fields']['packets'] += (int) $line[7]; 303 | $data['fields']['bytes'] += (int) $line[9]; 304 | } 305 | 306 | // write to database 307 | if (Config::$db->write($data) === false) { 308 | throw new \Exception('Error writing to ' . $statsPath); 309 | } 310 | 311 | return true; 312 | } 313 | 314 | /** 315 | * Import a single nfcapd file. 316 | */ 317 | public function importFile(string $file, string $source, bool $last): void { 318 | try { 319 | $this->d->log('Importing file ' . $file . ' (' . $source . '), last=' . (int) $last, \LOG_INFO); 320 | 321 | // fill source.rrd 322 | $this->writeSourceData($source, $file); 323 | 324 | // write general port data (not depending on source, so only executed per port) 325 | if ($last === true) { 326 | $this->writePortsData($file); 327 | } 328 | 329 | // if enabled, process ports per source as well (source_80.rrd) 330 | if ($this->processPorts === true) { 331 | $this->writePortsData($file, $source); 332 | } 333 | } catch (\Exception $e) { 334 | $this->d->log('Caught exception: ' . $e->getMessage(), \LOG_WARNING); 335 | } 336 | } 337 | 338 | /** 339 | * Check if db is free to update (some databases only allow inserting data at the end). 340 | * 341 | * @throws \Exception 342 | */ 343 | public function dbUpdatable(string $file, string $source = '', int $port = 0): bool { 344 | if ($this->checkLastUpdate === false) { 345 | return true; 346 | } 347 | 348 | // parse capture file's datetime. can't use filemtime as we need the datetime in the file name. 349 | $date = []; 350 | if (!preg_match('/nfcapd\.([0-9]{12})$/', $file, $date)) { 351 | return false; 352 | } // nothing to import 353 | 354 | $fileDatetime = new \DateTime($date[1]); 355 | 356 | // get last updated time from database 357 | $lastUpdateDb = Config::$db->last_update($source, $port); 358 | $lastUpdate = null; 359 | if ($lastUpdateDb !== 0) { 360 | $lastUpdate = new \DateTime(); 361 | $lastUpdate->setTimestamp($lastUpdateDb); 362 | } 363 | 364 | // prevent attempt to import the same file again 365 | return $fileDatetime > $lastUpdate; 366 | } 367 | 368 | public function setVerbose(bool $verbose): void { 369 | if ($verbose === true) { 370 | $this->d->setDebug(true); 371 | } 372 | $this->verbose = $verbose; 373 | } 374 | 375 | public function setProcessPorts(bool $processPorts): void { 376 | $this->processPorts = $processPorts; 377 | } 378 | 379 | public function setForce(bool $force): void { 380 | $this->force = $force; 381 | } 382 | 383 | public function setQuiet(bool $quiet): void { 384 | $this->quiet = $quiet; 385 | } 386 | 387 | public function setProcessPortsBySource($processPortsBySource): void { 388 | $this->processPortsBySource = $processPortsBySource; 389 | } 390 | 391 | public function setCheckLastUpdate(bool $checkLastUpdate): void { 392 | $this->checkLastUpdate = $checkLastUpdate; 393 | } 394 | } 395 | -------------------------------------------------------------------------------- /backend/datasources/Akumuli.php: -------------------------------------------------------------------------------- 1 | d = Debug::getInstance(); 14 | $this->connect(); 15 | } 16 | 17 | /** 18 | * connects to TCP socket. 19 | */ 20 | public function connect(): void { 21 | try { 22 | $this->client = stream_socket_client('tcp://' . Config::$cfg['db']['akumuli']['host'] . ':' . Config::$cfg['db']['akumuli']['port'], $errno, $errmsg); 23 | 24 | if ($this->client === false) { 25 | throw new \Exception('Failed to connect to Akumuli: ' . $errmsg); 26 | } 27 | } catch (\Exception $e) { 28 | $this->d->dpr($e); 29 | } 30 | } 31 | 32 | /** 33 | * Convert data to redis-compatible string and write to Akumuli. 34 | */ 35 | public function write(array $data): bool { 36 | $fields = array_keys($data['fields']); 37 | $values = array_values($data['fields']); 38 | 39 | // writes assume redis protocol. first byte identification: 40 | // "+" simple strings "-" errors ":" integers "$" bulk strings "*" array 41 | $query = '+' . implode('|', $fields) . ' source=' . $data['source'] . "\r\n" 42 | . '+' . $data['date_iso'] . "\r\n" // timestamp 43 | . '*' . \count($fields) . "\r\n"; // length of following array 44 | 45 | // add the $values corresponding to $fields 46 | foreach ($values as $v) { 47 | $query .= ':' . $v . "\r\n"; 48 | } 49 | 50 | $this->d->dpr([$query]); 51 | 52 | // write redis-compatible string to socket 53 | fwrite($this->client, $query); 54 | 55 | return stream_get_contents($this->client); 56 | 57 | // to read: 58 | // curl localhost:8181/api/query -d "{'select':'flows'}" 59 | } 60 | 61 | public function __destruct() { 62 | if (\is_resource($this->client)) { 63 | fclose($this->client); 64 | } 65 | } 66 | 67 | /** 68 | * Gets data for plotting the graph in the frontend. 69 | * Each row in $return['data'] will be one line in the graph. 70 | * The lines can be 71 | * * protocols - $sources must not contain more than one source (legend e.g. gateway_flows_udp, gateway_flows_tcp) 72 | * * sources - $protocols must not contain more than one protocol (legend e.g. gateway_traffic_icmp, othersource_traffic_icmp). 73 | * 74 | * @param int $start timestamp 75 | * @param int $end timestamp 76 | * @param array $sources subset of sources specified in settings 77 | * @param array $protocols UDP/TCP/ICMP/other 78 | * @param string $type flows/packets/traffic 79 | * 80 | * @return array in the following format: 81 | * 82 | * $return = array( 83 | * 'start' => 1490484600, // timestamp of first value 84 | * 'end' => 1490652000, // timestamp of last value 85 | * 'step' => 300, // resolution of the returned data in seconds. lowest value would probably be 300 = 5 minutes 86 | * 'data' => array( 87 | * 0 => array( 88 | * 'legend' => 'source_type_protocol', 89 | * 'data' => array( 90 | * 1490484600 => 33.998333333333, 91 | * 1490485200 => 37.005, ... 92 | * ) 93 | * ), 94 | * 1 => array( e.g. gateway_flows_udp ...) 95 | * ) 96 | * ); 97 | */ 98 | public function get_graph_data(int $start, int $end, array $sources, array $protocols, array $ports, string $type = 'flows', string $display = 'sources'): array|string { 99 | // TODO: Implement get_graph_data() method. 100 | return []; 101 | } 102 | 103 | /** 104 | * Gets the timestamps of the first and last entry in the datasource (for this specific source). 105 | * 106 | * @return array (timestampfirst, timestamplast) 107 | */ 108 | public function date_boundaries(string $source): array { 109 | // TODO: Implement date_boundaries() method. 110 | return []; 111 | } 112 | 113 | /** 114 | * Gets the timestamp of the last update of the datasource (for this specific source). 115 | */ 116 | public function last_update(string $source, int $port = 0): int { 117 | // TODO: Implement last_update() method. 118 | return 0; 119 | } 120 | 121 | /** 122 | * Gets the path where the datasource's data is stored. 123 | */ 124 | public function get_data_path(): string { 125 | // TODO: Implement get_data_path() method. 126 | return ''; 127 | } 128 | 129 | /** 130 | * Removes all existing data for every source in $sources. 131 | * If $sources is empty, remove all existing data. 132 | */ 133 | public function reset(array $sources): bool { 134 | // TODO: Implement reset() method. 135 | return true; 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /backend/datasources/Datasource.php: -------------------------------------------------------------------------------- 1 | array( 10 | * 'source' => 'name_of_souce', 11 | * 'date_timestamp' => 000000000, 12 | * 'date_iso' => 'Ymd\THis', 13 | * 'fields' => array( 14 | * 'flows', 15 | * 'flows_tcp', 16 | * 'flows_udp', 17 | * 'flows_icmp', 18 | * 'flows_other', 19 | * 'packets', 20 | * 'packets_tcp', 21 | * 'packets_udp', 22 | * 'packets_icmp', 23 | * 'packets_other', 24 | * 'bytes', 25 | * 'bytes_tcp', 26 | * 'bytes_udp', 27 | * 'bytes_icmp', 28 | * 'bytes_other') 29 | * );. 30 | * 31 | * @return bool TRUE on success or FALSE on failure 32 | * 33 | * @throws \Exception on error 34 | */ 35 | public function write(array $data): bool; 36 | 37 | /** 38 | * Gets data for plotting the graph in the frontend. 39 | * Each row in $return['data'] will be a time point in the graph. 40 | * The lines can be 41 | * * protocols - $sources must not contain more than one source (legend e.g. gateway_flows_udp, gateway_flows_tcp) 42 | * * sources - $protocols must not contain more than one protocol (legend e.g. gateway_traffic_icmp, othersource_traffic_icmp) 43 | * * ports. 44 | * 45 | * @param int $start timestamp 46 | * @param int $end timestamp 47 | * @param array $sources subset of sources specified in settings 48 | * @param array $protocols UDP/TCP/ICMP/other 49 | * @param string $type flows/packets/traffic 50 | * @param string $display protocols/sources/ports 51 | * 52 | * @return array in the following format: 53 | * 54 | * $return = array( 55 | * 'start' => 1490484600, // timestamp of first value 56 | * 'end' => 1490652000, // timestamp of last value 57 | * 'step' => 300, // resolution of the returned data in seconds. lowest value would probably be 300 = 5 minutes 58 | * 'legend' => array('swi6_flows_tcp', 'gate_flows_tcp'), // legend describes the graph series 59 | * 'data' => array( 60 | * 1490484600 => array(33.998333333333, 22.4), // the values/measurements for this specific timestamp are in an array 61 | * 1490485200 => array(37.005, 132.8282), 62 | * ... 63 | * ) 64 | * ); 65 | */ 66 | public function get_graph_data( 67 | int $start, 68 | int $end, 69 | array $sources, 70 | array $protocols, 71 | array $ports, 72 | string $type = 'flows', 73 | string $display = 'sources', 74 | ): array|string; 75 | 76 | /** 77 | * Removes all existing data for every source in $sources. 78 | * If $sources is empty, remove all existing data. 79 | */ 80 | public function reset(array $sources): bool; 81 | 82 | /** 83 | * Gets the timestamps of the first and last entry in the datasource (for this specific source). 84 | * 85 | * @return array (timestampfirst, timestamplast) 86 | */ 87 | public function date_boundaries(string $source): array; 88 | 89 | /** 90 | * Gets the timestamp of the last update of the datasource (for this specific source). 91 | */ 92 | public function last_update(string $source, int $port = 0): int; 93 | 94 | /** 95 | * Gets the path where the datasource's data is stored. 96 | */ 97 | public function get_data_path(): string; 98 | } 99 | -------------------------------------------------------------------------------- /backend/datasources/Rrd.php: -------------------------------------------------------------------------------- 1 | d = Debug::getInstance(); 39 | 40 | if (!\function_exists('rrd_version')) { 41 | throw new \Exception('Please install the PECL rrd library.'); 42 | } 43 | } 44 | 45 | /** 46 | * Gets the timestamps of the first and last entry of this specific source. 47 | */ 48 | public function date_boundaries(string $source): array { 49 | $rrdFile = $this->get_data_path($source); 50 | 51 | return [rrd_first($rrdFile), rrd_last($rrdFile)]; 52 | } 53 | 54 | /** 55 | * Gets the timestamp of the last update of this specific source. 56 | * 57 | * @return int timestamp or false 58 | */ 59 | public function last_update(string $source = '', int $port = 0): int { 60 | $rrdFile = $this->get_data_path($source, $port); 61 | $last_update = rrd_last($rrdFile); 62 | 63 | // $this->d->log('Last update of ' . $rrdFile . ': ' . date('d.m.Y H:i', $last_update), LOG_DEBUG); 64 | return (int) $last_update; 65 | } 66 | 67 | /** 68 | * Create a new RRD file for a source. 69 | * 70 | * @param string $source e.g. gateway or server_xyz 71 | * @param bool $reset overwrites existing RRD file if true 72 | */ 73 | public function create(string $source, int $port = 0, bool $reset = false): bool { 74 | $rrdFile = $this->get_data_path($source, $port); 75 | 76 | // check if folder exists 77 | if (!file_exists(\dirname($rrdFile))) { 78 | mkdir(\dirname($rrdFile), 0o755, true); 79 | } 80 | 81 | // check if folder has correct access rights 82 | if (!is_writable(\dirname($rrdFile))) { 83 | $this->d->log('Error creating ' . $rrdFile . ': Not writable', \LOG_CRIT); 84 | 85 | return false; 86 | } 87 | // check if file already exists 88 | if (file_exists($rrdFile)) { 89 | if ($reset === true) { 90 | unlink($rrdFile); 91 | } else { 92 | $this->d->log('Error creating ' . $rrdFile . ': File already exists', \LOG_ERR); 93 | 94 | return false; 95 | } 96 | } 97 | 98 | $start = strtotime('3 years ago'); 99 | $starttime = (int) $start - ($start % 300); 100 | 101 | $creator = new \RRDCreator($rrdFile, (string) $starttime, 60 * 5); 102 | foreach ($this->fields as $field) { 103 | $creator->addDataSource($field . ':ABSOLUTE:600:U:U'); 104 | } 105 | foreach ($this->layout as $rra) { 106 | $creator->addArchive('AVERAGE:' . $rra); 107 | $creator->addArchive('MAX:' . $rra); 108 | } 109 | 110 | $saved = $creator->save(); 111 | if ($saved === false) { 112 | $this->d->log('Error saving RRD data structure to ' . $rrdFile, \LOG_ERR); 113 | } 114 | 115 | return $saved; 116 | } 117 | 118 | /** 119 | * Write to an RRD file with supplied data. 120 | * 121 | * @throws \Exception 122 | */ 123 | public function write(array $data): bool { 124 | $rrdFile = $this->get_data_path($data['source'], $data['port']); 125 | if (!file_exists($rrdFile)) { 126 | $this->create($data['source'], $data['port'], false); 127 | } 128 | 129 | $nearest = (int) $data['date_timestamp'] - ($data['date_timestamp'] % 300); 130 | $this->d->log('Writing to file ' . $rrdFile, \LOG_DEBUG); 131 | 132 | // write data 133 | $updater = new \RRDUpdater($rrdFile); 134 | 135 | return $updater->update($data['fields'], (string) $nearest); 136 | } 137 | 138 | /** 139 | * @param string $type flows/packets/traffic 140 | * @param string $display protocols/sources/ports 141 | */ 142 | public function get_graph_data( 143 | int $start, 144 | int $end, 145 | array $sources, 146 | array $protocols, 147 | array $ports, 148 | #[ExpectedValues(['flows', 'packets', 'bytes', 'bits'])] 149 | string $type = 'flows', 150 | #[ExpectedValues(['protocols', 'sources', 'ports'])] 151 | string $display = 'sources', 152 | ): array|string { 153 | $options = [ 154 | '--start', 155 | $start - ($start % 300), 156 | '--end', 157 | $end - ($end % 300), 158 | '--maxrows', 159 | 300, 160 | // number of values. works like the width value (in pixels) in rrd_graph 161 | // '--step', 1200, // by default, rrdtool tries to get data for each row. if you want rrdtool to get data at a one-hour resolution, set step to 3600. 162 | '--json', 163 | ]; 164 | 165 | $useBits = false; 166 | if ($type === 'bits') { 167 | $type = 'bytes'; 168 | $useBits = true; 169 | } 170 | 171 | if (empty($protocols)) { 172 | $protocols = ['tcp', 'udp', 'icmp', 'other']; 173 | } 174 | if (empty($sources)) { 175 | $sources = Config::$cfg['general']['sources']; 176 | } 177 | if (empty($ports)) { 178 | $ports = Config::$cfg['general']['ports']; 179 | } 180 | 181 | switch ($display) { 182 | case 'protocols': 183 | foreach ($protocols as $protocol) { 184 | $rrdFile = $this->get_data_path($sources[0]); 185 | $proto = ($protocol === 'any') ? '' : '_' . $protocol; 186 | $legend = array_filter([$protocol, $type, $sources[0]]); 187 | $options[] = 'DEF:data' . $sources[0] . $protocol . '=' . $rrdFile . ':' . $type . $proto . ':AVERAGE'; 188 | $options[] = 'XPORT:data' . $sources[0] . $protocol . ':' . implode('_', $legend); 189 | } 190 | break; 191 | case 'sources': 192 | foreach ($sources as $source) { 193 | $rrdFile = $this->get_data_path($source); 194 | $proto = ($protocols[0] === 'any') ? '' : '_' . $protocols[0]; 195 | $legend = array_filter([$source, $type, $protocols[0]]); 196 | $options[] = 'DEF:data' . $source . '=' . $rrdFile . ':' . $type . $proto . ':AVERAGE'; 197 | $options[] = 'XPORT:data' . $source . ':' . implode('_', $legend); 198 | } 199 | break; 200 | case 'ports': 201 | foreach ($ports as $port) { 202 | $source = ($sources[0] === 'any') ? '' : $sources[0]; 203 | $proto = ($protocols[0] === 'any') ? '' : '_' . $protocols[0]; 204 | $legend = array_filter([$port, $type, $source, $protocols[0]]); 205 | $rrdFile = $this->get_data_path($source, $port); 206 | $options[] = 'DEF:data' . $source . $port . '=' . $rrdFile . ':' . $type . $proto . ':AVERAGE'; 207 | $options[] = 'XPORT:data' . $source . $port . ':' . implode('_', $legend); 208 | } 209 | } 210 | 211 | ob_start(); 212 | $data = rrd_xport($options); 213 | $error = ob_get_clean(); // rrd_xport weirdly prints stuff on error 214 | 215 | if (!\is_array($data)) { 216 | return $error . '. ' . rrd_error(); 217 | } 218 | 219 | // remove invalid numbers and create processable array 220 | $output = [ 221 | 'data' => [], 222 | 'start' => $data['start'], 223 | 'end' => $data['end'], 224 | 'step' => $data['step'], 225 | 'legend' => [], 226 | ]; 227 | foreach ($data['data'] as $source) { 228 | $output['legend'][] = $source['legend']; 229 | foreach ($source['data'] as $date => $measure) { 230 | // ignore non-valid measures 231 | if (is_nan($measure)) { 232 | $measure = null; 233 | } 234 | 235 | if ($type === 'bytes' && $useBits) { 236 | $measure *= 8; 237 | } 238 | 239 | // add measure to output array 240 | if (\array_key_exists($date, $output['data'])) { 241 | $output['data'][$date][] = $measure; 242 | } else { 243 | $output['data'][$date] = [$measure]; 244 | } 245 | } 246 | } 247 | 248 | return $output; 249 | } 250 | 251 | /** 252 | * Creates a new database for every source/port combination. 253 | */ 254 | public function reset(array $sources): bool { 255 | $return = false; 256 | if (empty($sources)) { 257 | $sources = Config::$cfg['general']['sources']; 258 | } 259 | $ports = Config::$cfg['general']['ports']; 260 | $ports[] = 0; 261 | foreach ($ports as $port) { 262 | if ($port !== 0) { 263 | $return = $this->create('', $port, true); 264 | } 265 | if ($return === false) { 266 | return false; 267 | } 268 | 269 | foreach ($sources as $source) { 270 | $return = $this->create($source, $port, true); 271 | if ($return === false) { 272 | return false; 273 | } 274 | } 275 | } 276 | 277 | return true; 278 | } 279 | 280 | /** 281 | * Concatenates the path to the source's rrd file. 282 | */ 283 | public function get_data_path(string $source = '', int $port = 0): string { 284 | if ((int) $port === 0) { 285 | $port = ''; 286 | } else { 287 | $port = (empty($source)) ? $port : '_' . $port; 288 | } 289 | $path = Config::$path . \DIRECTORY_SEPARATOR . 'datasources' . \DIRECTORY_SEPARATOR . 'data' . \DIRECTORY_SEPARATOR . $source . $port . '.rrd'; 290 | 291 | if (!file_exists($path)) { 292 | $this->d->log('Was not able to find ' . $path, \LOG_INFO); 293 | } 294 | 295 | return $path; 296 | } 297 | } 298 | -------------------------------------------------------------------------------- /backend/index.php: -------------------------------------------------------------------------------- 1 | log('Fatal: ' . $e->getMessage(), \LOG_ALERT); 22 | exit; 23 | } 24 | 25 | $folder = __DIR__; 26 | $lock_file = fopen($folder . '/nfsen-ng.pid', 'c'); 27 | $got_lock = flock($lock_file, \LOCK_EX | \LOCK_NB, $wouldblock); 28 | if ($lock_file === false || (!$got_lock && !$wouldblock)) { 29 | exit(128); 30 | } 31 | if (!$got_lock && $wouldblock) { 32 | exit(129); 33 | } 34 | 35 | // Lock acquired; let's write our PID to the lock file for the convenience 36 | // of humans who may wish to terminate the script. 37 | ftruncate($lock_file, 0); 38 | fwrite($lock_file, getmypid() . \PHP_EOL); 39 | 40 | // first import missed data if available 41 | $start = new DateTime(); 42 | $start->setDate(date('Y') - 3, (int) date('m'), (int) date('d')); 43 | $i = new Import(); 44 | $i->setQuiet(false); 45 | $i->setVerbose(true); 46 | $i->setProcessPorts(true); 47 | $i->setProcessPortsBySource(true); 48 | $i->setCheckLastUpdate(true); 49 | $i->start($start); 50 | 51 | $d->log('Starting periodic execution', \LOG_INFO); 52 | 53 | /* @phpstan-ignore-next-line */ 54 | while (1) { 55 | // next import in 30 seconds 56 | sleep(30); 57 | 58 | // import from last db update 59 | $i->start($start); 60 | } 61 | 62 | // all done; blank the PID file and explicitly release the lock 63 | /* @phpstan-ignore-next-line */ 64 | ftruncate($lock_file, 0); 65 | flock($lock_file, \LOCK_UN); 66 | -------------------------------------------------------------------------------- /backend/processor/Nfdump.php: -------------------------------------------------------------------------------- 1 | [], 11 | 'option' => [], 12 | 'format' => null, 13 | 'filter' => [], 14 | ]; 15 | private array $clean; 16 | private readonly Debug $d; 17 | public static ?self $_instance = null; 18 | 19 | public function __construct() { 20 | $this->d = Debug::getInstance(); 21 | $this->clean = $this->cfg; 22 | $this->reset(); 23 | } 24 | 25 | public static function getInstance(): self { 26 | if (!(self::$_instance instanceof self)) { 27 | self::$_instance = new self(); 28 | } 29 | 30 | return self::$_instance; 31 | } 32 | 33 | /** 34 | * Sets an option's value. 35 | */ 36 | public function setOption(string $option, $value): void { 37 | switch ($option) { 38 | case '-M': // set sources 39 | // only sources specified in settings allowed 40 | $queried_sources = explode(':', (string) $value); 41 | foreach ($queried_sources as $s) { 42 | if (!\in_array($s, Config::$cfg['general']['sources'], true)) { 43 | continue; 44 | } 45 | $this->cfg['env']['sources'][] = $s; 46 | } 47 | 48 | // cancel if no sources remain 49 | if (empty($this->cfg['env']['sources'])) { 50 | break; 51 | } 52 | 53 | // set sources path 54 | $this->cfg['option'][$option] = implode(\DIRECTORY_SEPARATOR, [ 55 | $this->cfg['env']['profiles-data'], 56 | $this->cfg['env']['profile'], 57 | implode(':', $this->cfg['env']['sources']), 58 | ]); 59 | 60 | break; 61 | case '-R': // set path 62 | $this->cfg['option'][$option] = $this->convert_date_to_path($value[0], $value[1]); 63 | break; 64 | case '-o': // set output format 65 | $this->cfg['format'] = $value; 66 | break; 67 | default: 68 | $this->cfg['option'][$option] = $value; 69 | $this->cfg['option']['-o'] = 'csv'; // always get parsable data todo user-selectable? calculations bps/bpp/pps not in csv 70 | break; 71 | } 72 | } 73 | 74 | /** 75 | * Sets a filter's value. 76 | */ 77 | public function setFilter(string $filter): void { 78 | $this->cfg['filter'] = $filter; 79 | } 80 | 81 | /** 82 | * Executes the nfdump command, tries to throw an exception based on the return code. 83 | * 84 | * @throws \Exception 85 | */ 86 | public function execute(): array { 87 | $output = []; 88 | $processes = []; 89 | $return = ''; 90 | $timer = microtime(true); 91 | $filter = (empty($this->cfg['filter'])) ? '' : ' ' . escapeshellarg((string) $this->cfg['filter']); 92 | $command = $this->cfg['env']['bin'] . ' ' . $this->flatten($this->cfg['option']) . $filter . ' 2>&1'; 93 | $this->d->log('Trying to execute ' . $command, \LOG_DEBUG); 94 | 95 | // check for already running nfdump processes 96 | exec('ps -eo user,pid,args | grep -v grep | grep `whoami` | grep "' . $this->cfg['env']['bin'] . '"', $processes); 97 | if (\count($processes) / 2 > (int) Config::$cfg['nfdump']['max-processes']) { 98 | throw new \Exception('There already are ' . \count($processes) / 2 . ' processes of NfDump running!'); 99 | } 100 | 101 | // execute nfdump 102 | exec($command, $output, $return); 103 | 104 | // prevent logging the command usage description 105 | if (isset($output[0]) && preg_match('/^usage/i', $output[0])) { 106 | $output = []; 107 | } 108 | 109 | switch ($return) { 110 | case 127: 111 | throw new \Exception('NfDump: Failed to start process. Is nfdump installed?
Graph Scale
448 |Series display
459 |Series
478 | 479 |Values
483 | 484 |