├── CMakeLists.txt ├── LICENSE └── osmupdate.c /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.6) 2 | 3 | add_compile_options("-O3") 4 | 5 | add_executable(osmupdate osmupdate.c) 6 | add_executable(osmfilter osmfilter.c) 7 | add_executable(osmconvert osmconvert.c) 8 | 9 | target_link_libraries( 10 | osmconvert 11 | z 12 | ) 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | OSM C Tools 2 | Copyright (C) 2011..2014 Markus Weber, Nuernberg 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU Affero General Public License 6 | version 3 as published by the Free Software Foundation. 7 | 8 | This program is distributed in the hope that it will be useful, 9 | but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | GNU Affero General Public License for more details. 12 | 13 | You should have received a copy of the GNU Affero General Public License 14 | along with this program. If not, see http://www.gnu.org/licenses/. 15 | 16 | Other licenses are available on request; please ask the author. 17 | -------------------------------------------------------------------------------- /osmupdate.c: -------------------------------------------------------------------------------- 1 | // osmupdate 2017-02-26 16:40 2 | #define VERSION "0.4.4" 3 | // 4 | // compile this file: 5 | // gcc osmupdate.c -o osmupdate 6 | // 7 | // (c) 2011..2017 Markus Weber, Nuernberg 8 | // 9 | // This program is free software; you can redistribute it and/or 10 | // modify it under the terms of the GNU Affero General Public License 11 | // version 3 as published by the Free Software Foundation. 12 | // This program is distributed in the hope that it will be useful, 13 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | // GNU Affero General Public License for more details. 16 | // You should have received a copy of this license along 17 | // with this program; if not, see http://www.gnu.org/licenses/. 18 | // 19 | // Other licenses are available on request; please ask the author. 20 | 21 | #define MAXLOGLEVEL 2 22 | const char* helptext= 23 | "\nosmupdate " VERSION "\n" 24 | "\n" 25 | "This program cares about updating an .osm, .o5m or .pbf file. It\n" 26 | "will download and apply OSM Change files (.osc) from the servers of\n" 27 | "\"planet.openstreetmap.org\".\n" 28 | "It also can assemble a new .osc or .o5c file which can be used to\n" 29 | "update your OSM data file at a later time.\n" 30 | "\n" 31 | "Prequesites\n" 32 | "\n" 33 | "To run this program, please download and install two other programs\n" 34 | "first: \"osmconvert\" and \"wget\".\n" 35 | "\n" 36 | "Usage\n" 37 | "\n" 38 | "Two command line arguments are mandatory: the name of the old and the\n" 39 | "name of the new OSM data file. If the old data file does not have a\n" 40 | "file timestamp, you may want to specify this timestamp manually on\n" 41 | "the command line. If you do not, the program will try to determine\n" 42 | "the timestamp by examining the whole old data file.\n" 43 | "Instead of the second parameter, you alternatively may specify the\n" 44 | "name of a change file (.osc or .o5c). In this case, you also may\n" 45 | "replace the name of the old OSM data file by a timestamp.\n" 46 | "Command line arguments which are not recognized by osmupdate will be\n" 47 | "passed to osmconvert. Use this opportunity to supply a bounding box\n" 48 | "or a bounding polygon if you are going to update a regional change\n" 49 | "file. You also may exclude unneeded meta data from your file by\n" 50 | "specifying this osmconvert option: --drop-author\n" 51 | "\n" 52 | "Usage Examples\n" 53 | "\n" 54 | " ./osmupdate old_file.o5m new_file.o5m\n" 55 | " ./osmupdate old_file.pbf new_file.pbf\n" 56 | " ./osmupdate old_file.osm new_file.osm\n" 57 | " The old OSM data will be updated and written as new_file.o5m\n" 58 | " or new_file.o5m. For safety reasons osmupdate will not delete\n" 59 | " the old file. If you do not need it as backup file, please\n" 60 | " delete it by yourself.\n" 61 | "\n" 62 | " ./osmupdate old_file.osm 2011-07-15T23:30:00Z new_file.osm\n" 63 | " ./osmupdate old_file.osm NOW-86400 new_file.osm\n" 64 | " If your old OSM data file does not contain a file timestamp,\n" 65 | " or you do not want to rely on this timestamp, it can be\n" 66 | " specified manually. Relative times are in seconds to NOW.\n" 67 | "\n" 68 | " ./osmupdate old_file.o5m change_file.o5c\n" 69 | " ./osmupdate old_file.osm change_file.osc\n" 70 | " ./osmupdate 2011-07-15T23:30:00Z change_file.o5c\n" 71 | " ./osmupdate 2011-07-15T23:30:00Z change_file.osc.gz\n" 72 | " ./osmupdate NOW-3600 change_file.osc.gz\n" 73 | " Here, the old OSM data file is not updated directly. An OSM\n" 74 | " changefile is written instead. This changefile can be used to\n" 75 | " update the OSM data file afterwards.\n" 76 | " You will have recognized the extension .gz in the last\n" 77 | " example. In this case, the OSM Change file will be written\n" 78 | " with gzip compression. To accomplish this, you need to have\n" 79 | " the program gzip installed on your system.\n" 80 | "\n" 81 | " ./osmupdate london_old.o5m london_new.o5m -B=london.poly\n" 82 | " The OSM data file london_old.o5m will be updated. Hence the\n" 83 | " downloaded OSM changefiles contain not only London, but the\n" 84 | " whole planet, a lot of unneeded data will be added to this\n" 85 | " regional file. The -B= argument will clip these superfluous\n" 86 | " data.\n" 87 | "\n" 88 | "The program osmupdate recognizes a few command line options:\n" 89 | "\n" 90 | "--max-days=UPDATE_RANGE\n" 91 | " By default, the maximum time range for to assemble a\n" 92 | " cumulated changefile is 250 days. You can change this by\n" 93 | " giving a different maximum number of days, for example 300.\n" 94 | " If you do, please ensure that there are daily change files\n" 95 | " available for such a wide range of time.\n" 96 | "\n" 97 | "--minute\n" 98 | "--hour\n" 99 | "--day\n" 100 | "--sporadic\n" 101 | " By default, osmupdate uses a combination of minutely, hourly\n" 102 | " and daily changefiles. If you want to limit these changefile\n" 103 | " categories, use one or two of these options and choose that\n" 104 | " category/ies you want to be used.\n" 105 | " The option --sporadic allows processing changefile sources\n" 106 | " which do not have the usual \"minute\", \"hour\" and \"day\"\n" 107 | " subdirectories.\n" 108 | "\n" 109 | "--max-merge=COUNT\n" 110 | " The subprogram osmconvert is able to merge more than two\n" 111 | " changefiles in one run. This ability increases merging speed.\n" 112 | " Unfortunately, every changefile consumes about 200 MB of main\n" 113 | " memory while being processed. For this reason, the number of\n" 114 | " parallely processable changefiles is limited.\n" 115 | " Use this commandline argument to determine the maximum number\n" 116 | " of parallely processed changefiles. The default value is 7.\n" 117 | "\n" 118 | "-t=TEMPPATH\n" 119 | "--tempfiles=TEMPPATH\n" 120 | " On order to cache changefiles, osmupdate needs a separate\n" 121 | " directory. This parameter defines the name of this directory,\n" 122 | " including the prefix of the tempfiles' names.\n" 123 | " The default value is \"osmupdate_temp/temp\".\n" 124 | "\n" 125 | "--keep-tempfiles\n" 126 | " Use this option if you want to keep local copies of every\n" 127 | " downloaded file. This is strongly recommended if you are\n" 128 | " going to assemble different changefiles which overlap in\n" 129 | " time ranges. Your data traffic will be minimized.\n" 130 | " Do not invoke this option if you are going to use different\n" 131 | " change file sources (option --base-url). This would cause\n" 132 | " severe data corruption.\n" 133 | "\n" 134 | "--trust-tempfiles\n" 135 | " Use this option if you want to use the saved local copies\n" 136 | " of already downloaded changefiles without checking their\n" 137 | " lengths against to their server-hosted originals.\n" 138 | " Downloads will be limited to files not saved yet.\n" 139 | " Do not invoke this option if you suspect incomplete\n" 140 | " downloads.\n" 141 | "\n" 142 | "--compression-level=LEVEL\n" 143 | " Define level for gzip compression. Values between 1 (low\n" 144 | " compression, but fast) and 9 (high compression, but slow).\n" 145 | "\n" 146 | "--base-url=BASE_URL\n" 147 | " To accelerate downloads or to get regional file updates you\n" 148 | " may specify an alternative download location. Please enter\n" 149 | " its URL, or simply the word \"mirror\" if you want to use\n" 150 | " gwdg's planet server.\n" 151 | "\n" 152 | "--base-url-suffix=BASE_URL_SUFFIX\n" 153 | " To use old planet URLs, you may need to add the suffix\n" 154 | " \"-replicate\" because it was custom to have this word in the\n" 155 | " URL, right after the period identifier \"day\" etc.\n" 156 | "\n" 157 | "-v\n" 158 | "--verbose\n" 159 | " With activated \'verbose\' mode, some statistical data and\n" 160 | " diagnosis data will be displayed.\n" 161 | " If -v resp. --verbose is the first parameter in the line,\n" 162 | " osmupdate will display all input parameters.\n" 163 | "\n" 164 | "This program is for experimental use. Expect malfunctions and data\n" 165 | "loss. Do not use the program in productive or commercial systems.\n" 166 | "\n" 167 | "There is NO WARRANTY, to the extent permitted by law.\n" 168 | "Please send any bug reports to markus.weber@gmx.com\n\n"; 169 | 170 | #define _FILE_OFFSET_BITS 64 171 | #include 172 | #include 173 | #include 174 | #include 175 | #include 176 | #include 177 | #include 178 | #include 179 | #include 180 | #include 181 | #include 182 | 183 | typedef enum {false= 0,true= 1} bool; 184 | typedef uint8_t byte; 185 | typedef unsigned int uint; 186 | #define isdig(x) isdigit((unsigned char)(x)) 187 | static int loglevel= 0; // logging to stderr; 188 | // 0: no logging; 1: small logging; 2: normal logging; 189 | // 3: extended logging; 190 | #define DP(f) fprintf(stderr,"Debug: " #f "\n"); 191 | #define DPv(f,...) fprintf(stderr,"Debug: " #f "\n",__VA_ARGS__); 192 | #define DPM(f,p,m) { byte* pp; int i,mm; static int msgn= 3; \ 193 | if(--msgn>=0) { fprintf(stderr,"Debug memory: " #f); \ 194 | pp= (byte*)(p); mm= (m); if(pp==NULL) fprintf(stderr,"\n (null)"); \ 195 | else for(i= 0; i0 && *src!=0) 232 | *d++= *src++; 233 | *d= 0; 234 | return dest; 235 | } // end strmcpy() 236 | #define strMcpy(d,s) strmcpy((d),(s),sizeof(d)) 237 | 238 | static inline char *stecpy(char** destp, char* destend, 239 | const char* src) { 240 | // same as stppcpy(), but you may define a pointer which the 241 | // destination will not be allowed to cross, whether or not the 242 | // string will be completed at that moment; 243 | // in either case, the destination string will be terminated with 0; 244 | char* dest; 245 | 246 | dest= *destp; 247 | if(dest>=destend) 248 | return dest; 249 | destend--; 250 | while(*src!=0 && dest=destend) 284 | return dest; 285 | destend-= 2; 286 | while(*src!=0 && dest0 && *src!=0) 311 | *d++= *src++; 312 | *d= 0; 313 | return d; 314 | } // end stpmcpy() 315 | #define stpMcpy(d,s) stpmcpy(d,s,sizeof(d)) 316 | 317 | static inline int strzcmp(const char* s1,const char* s2) { 318 | // similar to strcmp(), this procedure compares two character strings; 319 | // here, the number of characters which are to be compared is limited 320 | // to the length of the second string; 321 | // i.e., this procedure can be used to identify a short string s2 322 | // within a long string s1; 323 | // s1[]: first string; 324 | // s2[]: string to compare with the first string; 325 | // return: 326 | // 0: both strings are identical; the first string may be longer than 327 | // the second; 328 | // -1: the first string is alphabetical smaller than the second; 329 | // 1: the first string is alphabetical greater than the second; 330 | while(*s1==*s2 && *s1!=0) { s1++; s2++; } 331 | if(*s2==0) 332 | return 0; 333 | return *(unsigned char*)s1 < *(unsigned char*)s2? -1: 1; 334 | } // end strzcmp() 335 | 336 | static inline int strycmp(const char* s1,const char* s2) { 337 | // similar to strcmp(), this procedure compares two character strings; 338 | // here, both strings are end-aligned; 339 | // not more characters will be compared than are existing in string s2; 340 | // i.e., this procedure can be used to identify a file name extension; 341 | const char* s1e; 342 | int l; 343 | 344 | l= strchr(s2,0)-s2; 345 | s1e= strchr(s1,0); 346 | if(s1e-s1=10) 365 | break; 366 | i= i*10+b; 367 | } 368 | return i; 369 | } // end strtouint32() 370 | 371 | static inline int64_t strtosint64(const char* s) { 372 | // read a number and convert it to a signed 64-bit integer; 373 | // return: number; 374 | int sign; 375 | int64_t i; 376 | byte b; 377 | 378 | if(*s=='-') { s++; sign= -1; } else sign= 1; 379 | i= 0; 380 | for(;;) { 381 | b= (byte)(*s++ -'0'); 382 | if(b>=10) 383 | break; 384 | i= i*10+b; 385 | } 386 | return i*sign; 387 | } // end strtosint64() 388 | 389 | static inline int64_t strtimetosint64(const char* s) { 390 | // read a timestamp in OSM format, e.g.: "2010-09-30T19:23:30Z", 391 | // and convert it to a signed 64-bit integer; 392 | // also allowed: relative time to NOW, e.g.: "NOW-86400", 393 | // which means '24 hours ago'; 394 | // return: time as a number (seconds since 1970); 395 | // ==0: syntax error; 396 | if(s[0]=='N') { // presumably a relative time to 'now' 397 | if(s[1]!='O' || s[2]!='W' || (s[3]!='+' && s[3]!='-') || 398 | !isdig(s[4])) // wrong syntax 399 | return 0; 400 | s+= 3; // jump over "NOW" 401 | if(*s=='+') s++; // jump over '+', if any 402 | return time(NULL)+strtosint64(s); 403 | } // presumably a relative time to 'now' 404 | if((s[0]!='1' && s[0]!='2') || 405 | !isdig(s[1]) || !isdig(s[2]) || !isdig(s[3]) || 406 | s[4]!='-' || !isdig(s[5]) || !isdig(s[6]) || 407 | s[7]!='-' || !isdig(s[8]) || !isdig(s[9]) || 408 | s[10]!='T' || !isdig(s[11]) || !isdig(s[12]) || 409 | s[13]!=':' || !isdig(s[14]) || !isdig(s[15]) || 410 | s[16]!=':' || !isdig(s[17]) || !isdig(s[18]) || 411 | s[19]!='Z') // wrong syntax 412 | return 0; 413 | /* regular timestamp */ { 414 | struct tm tm; 415 | 416 | tm.tm_isdst= 0; 417 | tm.tm_year= 418 | (s[0]-'0')*1000+(s[1]-'0')*100+(s[2]-'0')*10+(s[3]-'0')-1900; 419 | tm.tm_mon= (s[5]-'0')*10+s[6]-'0'-1; 420 | tm.tm_mday= (s[8]-'0')*10+s[9]-'0'; 421 | tm.tm_hour= (s[11]-'0')*10+s[12]-'0'; 422 | tm.tm_min= (s[14]-'0')*10+s[15]-'0'; 423 | tm.tm_sec= (s[17]-'0')*10+s[18]-'0'; 424 | #if __WIN32__ 425 | return mktime(&tm)-timezone; 426 | #else 427 | return timegm(&tm); 428 | #endif 429 | } // regular timestamp 430 | } // end strtimetosint64() 431 | 432 | static inline void int64tostrtime(uint64_t v,char* sp) { 433 | // write a timestamp in OSM format, e.g.: "2010-09-30T19:23:30Z", 434 | // into a string; 435 | // v: value of the timestamp; 436 | // sp[21]: destination string; 437 | time_t vtime; 438 | struct tm tm; 439 | int i; 440 | 441 | vtime= v; 442 | #if __WIN32__ 443 | memcpy(&tm,gmtime(&vtime),sizeof(tm)); 444 | #else 445 | gmtime_r(&vtime,&tm); 446 | #endif 447 | i= tm.tm_year+1900; 448 | sp+= 3; *sp--= i%10+'0'; 449 | i/=10; *sp--= i%10+'0'; 450 | i/=10; *sp--= i%10+'0'; 451 | i/=10; *sp= i%10+'0'; 452 | sp+= 4; *sp++= '-'; 453 | i= tm.tm_mon+1; 454 | *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= '-'; 455 | i= tm.tm_mday; 456 | *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= 'T'; 457 | i= tm.tm_hour; 458 | *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= ':'; 459 | i= tm.tm_min; 460 | *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= ':'; 461 | i= tm.tm_sec%60; 462 | *sp++= i/10+'0'; *sp++= i%10+'0'; *sp++= 'Z'; *sp= 0; 463 | } // end int64tostrtime() 464 | 465 | static inline bool file_exists(const char* file_name) { 466 | // query if a file exists; 467 | // file_name[]: name of the file in question; 468 | // return: the file exists; 469 | return access(file_name,R_OK)==0; 470 | } // file_exists() 471 | 472 | static inline int64_t file_length(const char* file_name) { 473 | // retrieve length of a file; 474 | // file_name[]: file name; 475 | // return: number of bytes of this file; 476 | // if the file could not be accessed, the return value is -1; 477 | struct stat s; 478 | int r; 479 | 480 | r= stat(file_name,&s); 481 | if(r==0) 482 | return s.st_size; 483 | #if !__WIN32__ 484 | if(errno==EOVERFLOW) // size larger than 2^31 485 | return 0x7fffffff; 486 | #endif 487 | return -1; 488 | } // end file_length() 489 | 490 | 491 | 492 | //------------------------------------------------------------ 493 | // Module Global global variables for this program 494 | //------------------------------------------------------------ 495 | 496 | // to distinguish global variable from local or module global 497 | // variables, they are preceded by 'global_'; 498 | 499 | static const char global_osmconvert_program_here_in_dir[]= 500 | "./osmconvert"; 501 | static const char* global_osmconvert_program= 502 | global_osmconvert_program_here_in_dir+2; 503 | // path to osmconvert program 504 | static char global_tempfile_name[450]= ""; 505 | // prefix of names for temporary files 506 | static bool global_keep_tempfiles= false; 507 | // temporary files shall not be deleted at program end 508 | static bool global_trust_tempfiles= false; 509 | // Cached files are considered to be intact 510 | static char global_osmconvert_arguments[2000]= ""; 511 | // general command line arguments for osmconvert; 512 | #define max_number_of_changefiles_in_cache 100 513 | static int global_max_merge= 7; 514 | // maximum number of parallely processed changefiles 515 | static const char* global_gzip_parameters= ""; 516 | // parameters for gzip compression 517 | static char global_base_url[400]= 518 | "https://planet.openstreetmap.org/replication"; 519 | static char global_base_url_suffix[100]=""; 520 | // for old replication URL, to get "day-replication" instead of "day" 521 | 522 | #define PERR(f) \ 523 | fprintf(stderr,"osmupdate Error: " f "\n"); 524 | // print error message 525 | #define PERRv(f,...) \ 526 | fprintf(stderr,"osmupdate Error: " f "\n",__VA_ARGS__); 527 | // print error message with value(s) 528 | #define WARN(f) { static int msgn= 3; if(--msgn>=0) \ 529 | fprintf(stderr,"osmupdate Warning: " f "\n"); } 530 | // print a warning message, do it maximal 3 times 531 | #define WARNv(f,...) { static int msgn= 3; if(--msgn>=0) \ 532 | fprintf(stderr,"osmupdate Warning: " f "\n",__VA_ARGS__); } 533 | // print a warning message with value(s), do it maximal 3 times 534 | #define PINFO(f) \ 535 | fprintf(stderr,"osmupdate: " f "\n"); // print info message 536 | #define PINFOv(f,...) \ 537 | fprintf(stderr,"osmupdate: " f "\n",__VA_ARGS__); 538 | #define ONAME(i) \ 539 | (i==0? "node": i==1? "way": i==2? "relation": "unknown object") 540 | 541 | //------------------------------------------------------------ 542 | // end Module Global global variables for this program 543 | //------------------------------------------------------------ 544 | 545 | 546 | 547 | static void shell_command(const char* command,char* result) { 548 | // execute a shell command; 549 | // command[]: shell command; 550 | // result[1000]: result of the command; 551 | FILE* fp; 552 | char* result_p; 553 | int maxlen; 554 | int r; 555 | 556 | if(loglevel>=2) 557 | PINFOv("Executing shell command:\n%s",command) 558 | fp= popen(command,"r"); 559 | if(fp==NULL) { 560 | PERR("Could not execute shell command.") 561 | result[0]= 0; 562 | exit(1); 563 | } 564 | result_p= result; 565 | maxlen= 1000-1; 566 | while(maxlen>0) { 567 | r= read(fileno(fp),result_p,maxlen); 568 | if(r==0) // end of data 569 | break; 570 | if(r<0) 571 | exit(errno); // (thanks to Ben Konrath) 572 | result_p+= r; 573 | maxlen-= r; 574 | } 575 | *result_p= 0; 576 | if(pclose(fp)==-1) 577 | exit(errno); // (thanks to Ben Konrath) 578 | if(loglevel>=2) 579 | PINFOv("Got shell command result:\n%s",result) 580 | } // end shell_command() 581 | 582 | typedef enum {cft_UNKNOWN,cft_MINUTELY,cft_HOURLY,cft_DAILY,cft_SPORADIC} 583 | changefile_type_t; 584 | #define CFTNAME(i) \ 585 | (i==cft_MINUTELY? "minutely": i==cft_HOURLY? "hourly": \ 586 | i==cft_DAILY? "daily": i==cft_SPORADIC? "sporadic": "unknown") 587 | 588 | static int64_t get_file_timestamp(const char* file_name) { 589 | // get the timestamp of a specific file; 590 | // if the file timestamp is not available, this procedure tries 591 | // to retrieve the timestamp from the file's statistics; 592 | // return: timestamp of the file (seconds from Jan 01 1970); 593 | // ==0: no file timestamp available; 594 | char command[500],*command_p,result[1000]; 595 | char* command_e= command+sizeof(command); 596 | int64_t file_timestamp; 597 | 598 | command_p= command; 599 | stecpy(&command_p,command_e,global_osmconvert_program); 600 | stecpy(&command_p,command_e," --out-timestamp \""); 601 | steesccpy(&command_p,command_e,file_name); 602 | stecpy(&command_p,command_e,"\" 2>&1"); 603 | shell_command(command,result); 604 | if(result[0]!='(' && (result[0]<'1' || result[0]>'2')) { 605 | // command not found 606 | PERR("Please install program osmconvert first.") 607 | exit(1); 608 | } // command not found 609 | file_timestamp= strtimetosint64(result); 610 | if(file_timestamp==0) { // the file has no file timestamp 611 | // try to get the timestamp from the file's statistics 612 | char* p; 613 | 614 | if(loglevel>0) { // verbose mode 615 | PINFOv("file %s has no file timestamp.",file_name) 616 | PINFO("Running statistics to get the timestamp.") 617 | } 618 | command_p= command; 619 | stecpy(&command_p,command_e,global_osmconvert_program); 620 | stecpy(&command_p,command_e," --out-statistics \""); 621 | steesccpy(&command_p,command_e,file_name); 622 | stecpy(&command_p,command_e,"\" 2>&1"); 623 | shell_command(command,result); 624 | p= strstr(result,"timestamp max: "); 625 | if(p!=NULL) { 626 | file_timestamp= strtimetosint64(p+15); 627 | PINFO("Aging the timestamp by 4 hours for safety reasons.") 628 | file_timestamp-= 4*3600; 629 | } 630 | } // the file has no file timestamp 631 | if(loglevel>0) { // verbose mode 632 | char ts[30]; 633 | 634 | if(file_timestamp==0) 635 | strcpy(ts,"(no timestamp)"); 636 | else 637 | int64tostrtime(file_timestamp,ts); 638 | PINFOv("timestamp of %s: %s",file_name,ts) 639 | } // verbose mode 640 | return file_timestamp; 641 | } // get_file_timestamp() 642 | 643 | static int64_t get_newest_changefile_timestamp( 644 | changefile_type_t changefile_type,int32_t* file_sequence_number) { 645 | // get sequence number and timestamp of the newest changefile 646 | // of a specific changefile type; 647 | // changefile_type: minutely, hourly, daily, sporadic changefile; 648 | // return: timestamp of the file (seconds from Jan 01 1970); 649 | // ==0: no file timestamp available; 650 | // *file_sequence_number: sequence number of the newest changefile; 651 | static bool firstrun= true; 652 | char command[1000],*command_p; 653 | char* command_e= command+sizeof(command); 654 | char result[1000]; 655 | int64_t changefile_timestamp; 656 | 657 | #if __WIN32__ 658 | static char newest_timestamp_file_name[400]; 659 | if(firstrun) { 660 | // create the file name for the newest timestamp; 661 | // this is only needed for Windows as Windows' wget 662 | // cannot write the downloaded file to standard output; 663 | // usually: "osmupdate_temp/temp.7" 664 | strcpy(stpmcpy(newest_timestamp_file_name,global_tempfile_name, 665 | sizeof(newest_timestamp_file_name)-5),".7"); 666 | } 667 | #endif 668 | command_p= command; 669 | stecpy(&command_p,command_e, 670 | "wget -q "); 671 | stecpy(&command_p,command_e,global_base_url); 672 | switch(changefile_type) { // changefile type 673 | case cft_MINUTELY: 674 | stecpy(&command_p,command_e,"/minute"); 675 | break; 676 | case cft_HOURLY: 677 | stecpy(&command_p,command_e,"/hour"); 678 | break; 679 | case cft_DAILY: 680 | stecpy(&command_p,command_e,"/day"); 681 | break; 682 | case cft_SPORADIC: 683 | break; 684 | default: // invalid change file type 685 | return 0; 686 | } // changefile type 687 | stecpy(&command_p,command_e,global_base_url_suffix); 688 | stecpy(&command_p,command_e,"/state.txt"); 689 | #if __WIN32__ 690 | stecpy(&command_p,command_e," -O \""); 691 | steesccpy(&command_p,command_e,newest_timestamp_file_name); 692 | stecpy(&command_p,command_e,"\" 2>&1"); 693 | #else 694 | stecpy(&command_p,command_e," -O - 2>&1"); 695 | #endif 696 | shell_command(command,result); 697 | if(firstrun) { // first run 698 | firstrun= false; 699 | if(result[0]!='#' && (result[0]<'1' || result[0]>'2') && ( 700 | strstr(result,"not found")!=NULL || 701 | strstr(result,"cannot find")!=NULL)) { // command not found 702 | PERR("Please install program wget first.") 703 | exit(1); 704 | } // command not found 705 | } // first run 706 | #if __WIN32__ 707 | result[0]= 0; 708 | /* copy tempfile contents to result[] */ { 709 | int fd,r; 710 | 711 | fd= open(newest_timestamp_file_name,O_RDONLY); 712 | if(fd>0) { 713 | r= read(fd,result,sizeof(result)-1); 714 | if(r>=0) 715 | result[r]= 0; 716 | close(fd); 717 | } 718 | } 719 | if(loglevel<2) 720 | unlink(newest_timestamp_file_name); 721 | #endif 722 | if(result[0]=='#') { // full status information 723 | // get sequence number 724 | char* sequence_number_p; 725 | sequence_number_p= strstr(result,"sequenceNumber="); 726 | if(sequence_number_p!=NULL) // found sequence number 727 | *file_sequence_number= strtouint32(sequence_number_p+15); 728 | // get timestamp 729 | char* timestamp_p; 730 | timestamp_p= strstr(result,"timestamp="); 731 | // search timestamp line 732 | if(timestamp_p!=NULL && timestamp_p-result0) { // verbose mode 743 | char ts[30]; 744 | 745 | if(changefile_timestamp==0) 746 | strcpy(ts,"(no timestamp)"); 747 | else 748 | int64tostrtime(changefile_timestamp,ts); 749 | PINFOv("newest %s timestamp: %s",CFTNAME(changefile_type),ts) 750 | } // verbose mode 751 | return changefile_timestamp; 752 | } // get_newest_changefile_timestamp 753 | 754 | static int64_t get_changefile_timestamp( 755 | changefile_type_t changefile_type,int32_t file_sequence_number) { 756 | // download and inspect the timestamp of a specific changefile which 757 | // is available in the Internet; 758 | // a timestamp file will not be downloaded if it 759 | // already exists locally as temporary file; 760 | // changefile_type: minutely, hourly, daily, sporadic changefile; 761 | // file_sequence_number: sequence number of the file; 762 | // uses: 763 | // global_tempfile_name 764 | // return: timestamp of the changefile (seconds from Jan 01 1970); 765 | // ==0: no file timestamp available; 766 | char command[2000]; char* command_p; 767 | char* command_e= command+sizeof(command); 768 | char result[1000]; 769 | char timestamp_cachefile_name[400]; 770 | int fd,r; // file descriptor; number of bytes which have been read 771 | char timestamp_contents[1000]; // contents of the timestamp 772 | int64_t changefile_timestamp; 773 | char* sp; 774 | 775 | // create the file name for the cached timestamp; example: 776 | // "osmupdate_temp/temp.m000012345.txt" 777 | sp= stpmcpy(timestamp_cachefile_name,global_tempfile_name, 778 | sizeof(timestamp_cachefile_name[0])-20); 779 | *sp++= '.'; 780 | *sp++= CFTNAME(changefile_type)[0]; 781 | // 'm', 'h', 'd', 's': minutely, hourly, daily, sporadic timestamps 782 | sprintf(sp,"%09"PRIi32".txt",file_sequence_number); 783 | // add sequence number and file name extension 784 | 785 | // download the timestamp into a cache file 786 | if(file_length(timestamp_cachefile_name)<10) { 787 | // timestamp has not been downloaded yet 788 | command_p= command; 789 | stecpy(&command_p,command_e,"wget -nv -c "); 790 | stecpy(&command_p,command_e,global_base_url); 791 | if(changefile_type==cft_MINUTELY) 792 | stecpy(&command_p,command_e,"/minute"); 793 | else if(changefile_type==cft_HOURLY) 794 | stecpy(&command_p,command_e,"/hour"); 795 | else if(changefile_type==cft_DAILY) 796 | stecpy(&command_p,command_e,"/day"); 797 | else if(changefile_type==cft_SPORADIC) 798 | ; 799 | else // invalid change file type 800 | return 0; 801 | stecpy(&command_p,command_e,global_base_url_suffix); 802 | stecpy(&command_p,command_e,"/"); 803 | /* assemble Sequence path */ { 804 | int l; 805 | l= sprintf(command_p,"%03i/%03i/%03i", 806 | file_sequence_number/1000000,file_sequence_number/1000%1000, 807 | file_sequence_number%1000); 808 | command_p+= l; 809 | } 810 | stecpy(&command_p,command_e,".state.txt -O \""); 811 | steesccpy(&command_p,command_e,timestamp_cachefile_name); 812 | stecpy(&command_p,command_e,"\" 2>&1"); 813 | shell_command(command,result); 814 | } // timestamp has not been downloaded yet 815 | 816 | // read the timestamp cache file 817 | fd= open(timestamp_cachefile_name,O_RDONLY|O_BINARY); 818 | if(fd<=0) // could not open the file 819 | timestamp_contents[0]= 0; // hence we did not read anything 820 | else { // could open the file 821 | r= read(fd,timestamp_contents,sizeof(timestamp_contents)-1); 822 | if(r<0) r= 0; 823 | timestamp_contents[r]= 0; // set string terminator 824 | close(fd); 825 | } // could open the file 826 | 827 | // parse the timestamp information 828 | if(timestamp_contents[0]=='#') { // full status information 829 | // get timestamp 830 | char* timestamp_p; 831 | timestamp_p= strstr(timestamp_contents,"timestamp="); 832 | // search timestamp line 833 | if(timestamp_p!=NULL && 834 | timestamp_p-timestamp_contents0) { // verbose mode 846 | char ts[30]; 847 | 848 | if(changefile_timestamp==0) 849 | strcpy(ts,"(no timestamp)"); 850 | else 851 | int64tostrtime(changefile_timestamp,ts); 852 | PINFOv("%s changefile %i: %s", 853 | CFTNAME(changefile_type),file_sequence_number,ts) 854 | } // verbose mode 855 | if(changefile_timestamp==0) { // no timestamp 856 | if(file_sequence_number==0) // first file in repository 857 | changefile_timestamp= 1; // set virtual very old timestamp 858 | else { 859 | PERRv("no timestamp for %s changefile %i.", 860 | CFTNAME(changefile_type),file_sequence_number) 861 | exit(1); 862 | } 863 | } // no timestamp 864 | return changefile_timestamp; 865 | } // get_changefile_timestamp 866 | 867 | static void process_changefile( 868 | changefile_type_t changefile_type,int32_t file_sequence_number, 869 | int64_t new_timestamp) { 870 | // download and process a change file; 871 | // change files will not be processed one by one, but cumulated 872 | // until some files have been downloaded and then processed in a group; 873 | // a file will not be downloaded if it already exists locally as 874 | // temporary file; 875 | // changefile_type: minutely, hourly, daily, sporadic changefile; 876 | // file_sequence_number: sequence number of the file; 877 | // ==0: process the remaining files which 878 | // are waiting in the cache; cleanup; 879 | // new_timestamp: timestamp of the new file which is to be created; 880 | // ==0: the procedure will assume the newest of all 881 | // timestamps which has been passed since the 882 | // program has been started; 883 | // uses: 884 | // global_max_merge 885 | // global_tempfile_name 886 | static bool firstrun= true; 887 | static int number_of_changefiles_in_cache= 0; 888 | static int64_t newest_new_timestamp= 0; 889 | static char master_cachefile_name[400]; 890 | static char master_cachefile_name_temp[400]; 891 | static char cachefile_name[max_number_of_changefiles_in_cache][400]; 892 | char command[4000+200*max_number_of_changefiles_in_cache]; 893 | char* command_e= command+sizeof(command); 894 | char* command_p; 895 | char result[1000]; 896 | 897 | if(firstrun) { 898 | firstrun= false; 899 | // create the file name for the cached master changefile; 900 | // usually: "osmupdate_temp/temp.8" 901 | strcpy(stpmcpy(master_cachefile_name,global_tempfile_name, 902 | sizeof(master_cachefile_name)-5),".8"); 903 | strcpy(stpmcpy(master_cachefile_name_temp,global_tempfile_name, 904 | sizeof(master_cachefile_name_temp)-5),".9"); 905 | unlink(master_cachefile_name); 906 | unlink(master_cachefile_name_temp); 907 | } 908 | if(new_timestamp>newest_new_timestamp) 909 | newest_new_timestamp= new_timestamp; 910 | if(file_sequence_number!=0) { // changefile download requested 911 | char* this_cachefile_name= 912 | cachefile_name[number_of_changefiles_in_cache]; 913 | int64_t old_file_length; 914 | char* sp; 915 | 916 | // create the file name for the cached changefile; example: 917 | // "osmupdate_temp/temp.m000012345.osc.gz" 918 | sp= stpmcpy(this_cachefile_name,global_tempfile_name, 919 | sizeof(cachefile_name[0])-20); 920 | *sp++= '.'; 921 | *sp++= CFTNAME(changefile_type)[0]; 922 | // 'm', 'h', 'd', 's': minutely, hourly, daily, 923 | // sporadic changefiles 924 | sprintf(sp,"%09"PRIi32".osc.gz",file_sequence_number); 925 | // add sequence number and file name extension 926 | 927 | // assemble the URL and download the changefile 928 | old_file_length= file_length(this_cachefile_name); 929 | if(global_trust_tempfiles && old_file_length>=10) { 930 | // trusted file already in cache 931 | if(loglevel>0) // verbose mode 932 | PINFOv("%s changefile %i: trusting local copy", 933 | CFTNAME(changefile_type),file_sequence_number) 934 | } // trusted file already in cache 935 | else { // file not in cache or not trusted 936 | if(loglevel>0) { // verbose mode 937 | if(old_file_length<10) // file not downloaded yet 938 | PINFOv("%s changefile %i: downloading", 939 | CFTNAME(changefile_type),file_sequence_number) 940 | else // file had been downloaded at least partially 941 | PINFOv("%s changefile %i: checking", 942 | CFTNAME(changefile_type),file_sequence_number) 943 | } // verbose mode 944 | command_p= command; 945 | stecpy(&command_p,command_e,"wget -nv -c "); 946 | stecpy(&command_p,command_e,global_base_url); 947 | switch(changefile_type) { // changefile type 948 | case cft_MINUTELY: 949 | stecpy(&command_p,command_e,"/minute"); 950 | break; 951 | case cft_HOURLY: 952 | stecpy(&command_p,command_e,"/hour"); 953 | break; 954 | case cft_DAILY: 955 | stecpy(&command_p,command_e,"/day"); 956 | break; 957 | case cft_SPORADIC: 958 | break; 959 | default: // invalid change file type 960 | return; 961 | } // changefile type 962 | stecpy(&command_p,command_e,global_base_url_suffix); 963 | stecpy(&command_p,command_e,"/"); 964 | 965 | /* process sequence number */ { 966 | int l; 967 | l= sprintf(command_p,"%03i/%03i/%03i.osc.gz", 968 | file_sequence_number/1000000,file_sequence_number/1000%1000, 969 | file_sequence_number%1000); 970 | command_p+= l; 971 | } // process sequence number 972 | 973 | stecpy(&command_p,command_e," -O \""); 974 | steesccpy(&command_p,command_e,this_cachefile_name); 975 | stecpy(&command_p,command_e,"\" 2>&1 && echo \"Wget Command Ok\""); 976 | shell_command(command,result); 977 | if(strstr(result,"Wget Command Ok")==NULL) { // download error 978 | PERRv("Could not download %s changefile %i", 979 | CFTNAME(changefile_type),file_sequence_number) 980 | PINFOv("wget Error message:\n%s",result) 981 | exit(1); 982 | } 983 | if(loglevel>0 && old_file_length>=10) { 984 | // verbose mode AND file was already in cache 985 | if(file_length(this_cachefile_name)!=old_file_length) 986 | PINFOv("%s changefile %i: download completed", 987 | CFTNAME(changefile_type),file_sequence_number) 988 | else 989 | PINFOv("%s changefile %i: already in cache", 990 | CFTNAME(changefile_type),file_sequence_number) 991 | } // verbose mode 992 | } // file not in cache or not trusted 993 | number_of_changefiles_in_cache++; 994 | } // changefile download requested 995 | 996 | if(number_of_changefiles_in_cache>=global_max_merge 997 | || (file_sequence_number==0 && number_of_changefiles_in_cache>0)) { 998 | // at least one change files must be merged 999 | // merge all changefiles which are waiting in cache 1000 | if(loglevel>0) 1001 | PINFO("Merging changefiles.") 1002 | command_p= command; 1003 | stecpy(&command_p,command_e,global_osmconvert_program); 1004 | stecpy(&command_p,command_e," --merge-versions "); 1005 | stecpy(&command_p,command_e,global_osmconvert_arguments); 1006 | while(number_of_changefiles_in_cache>0) { 1007 | // for all changefiles in cache 1008 | number_of_changefiles_in_cache--; 1009 | stecpy(&command_p,command_e," \""); 1010 | steesccpy(&command_p,command_e, 1011 | cachefile_name[number_of_changefiles_in_cache]); 1012 | stecpy(&command_p,command_e,"\""); 1013 | } // for all changefiles in cache 1014 | if(file_exists(master_cachefile_name)) { 1015 | stecpy(&command_p,command_e," \""); 1016 | steesccpy(&command_p,command_e,master_cachefile_name); 1017 | stecpy(&command_p,command_e,"\""); 1018 | } 1019 | if(newest_new_timestamp!=0) { 1020 | stecpy(&command_p,command_e," --timestamp="); 1021 | if(command_e-command_p>=30) 1022 | int64tostrtime(newest_new_timestamp,command_p); 1023 | command_p= strchr(command_p,0); 1024 | } 1025 | stecpy(&command_p,command_e," --out-o5c >\""); 1026 | steesccpy(&command_p,command_e,master_cachefile_name_temp); 1027 | stecpy(&command_p,command_e,"\""); 1028 | shell_command(command,result); 1029 | if(file_length(master_cachefile_name_temp)<10 || 1030 | strstr(result,"Error")!=NULL || 1031 | strstr(result,"error")!=NULL || 1032 | strstr(result,"Warning")!=NULL || 1033 | strstr(result,"warning")!=NULL) { // merging failed 1034 | PERRv("Merging of changefiles failed:\n%s",command) 1035 | if(result[0]!=0) 1036 | PERRv("%s",result) 1037 | exit(1); 1038 | } // merging failed 1039 | unlink(master_cachefile_name); 1040 | rename(master_cachefile_name_temp,master_cachefile_name); 1041 | } // at least one change file must be merged 1042 | } // process_changefile() 1043 | 1044 | #if !__WIN32__ 1045 | void sigcatcher(int sig) { 1046 | fprintf(stderr,"osmupdate: Output has been terminated.\n"); 1047 | exit(1); 1048 | } // end sigchatcher() 1049 | #endif 1050 | 1051 | int main(int argc,const char** argv) { 1052 | // main procedure; 1053 | // for the meaning of the calling line parameters please look at the 1054 | // contents of helptext[]; 1055 | 1056 | // variables for command line arguments 1057 | int main_return_value; 1058 | const char* a; // command line argument 1059 | char* osmconvert_arguments_p; 1060 | // pointer in global_osmconvert_arguments[] 1061 | char final_osmconvert_arguments[2000]; 1062 | // command line arguments for the final run of osmconvert; 1063 | char* final_osmconvert_arguments_p; 1064 | // pointer in final_osmconvert_arguments[] 1065 | const char* old_file; // name of the old OSM file 1066 | int64_t old_timestamp; // timestamp of the old OSM file 1067 | const char* new_file; // name of the new OSM file or OSM Change file 1068 | bool new_file_is_o5; // the new file is type .o5m or .o5c 1069 | bool new_file_is_pbf; // the new file is type .pbf 1070 | bool new_file_is_changefile; // the new file is a changefile 1071 | bool new_file_is_gz; // the new file is a gzip compressed 1072 | int64_t max_update_range; // maximum range for cumulating changefiles 1073 | // in order to update an OSM file; unit: seconds; 1074 | char tempfile_directory[400]; // directory for temporary files 1075 | bool process_minutely,process_hourly,process_daily,process_sporadic; 1076 | // if one of these variables is true, then only the chosen categories 1077 | // shall be processed; 1078 | bool no_minutely,no_hourly,no_daily; 1079 | // the category shall not be processed; 1080 | 1081 | // regular variables 1082 | int64_t minutely_timestamp,hourly_timestamp,daily_timestamp, 1083 | sporadic_timestamp; 1084 | // timestamps for changefiles which are available in the Internet; 1085 | // unit: seconds after Jan 01 1970; 1086 | int32_t minutely_sequence_number,hourly_sequence_number, 1087 | daily_sequence_number,sporadic_sequence_number; 1088 | int64_t timestamp; 1089 | int64_t next_timestamp; 1090 | 1091 | // care about clean-up procedures 1092 | #if !__WIN32__ 1093 | /* care about signal handler */ { 1094 | static struct sigaction siga; 1095 | 1096 | siga.sa_handler= sigcatcher; 1097 | sigemptyset(&siga.sa_mask); 1098 | siga.sa_flags= 0; 1099 | sigaction(SIGPIPE,&siga,NULL); 1100 | } 1101 | #endif 1102 | 1103 | // initializations 1104 | main_return_value= 0; // (default) 1105 | #if __WIN32__ 1106 | setmode(fileno(stdout),O_BINARY); 1107 | setmode(fileno(stdin),O_BINARY); 1108 | #endif 1109 | osmconvert_arguments_p= global_osmconvert_arguments; 1110 | final_osmconvert_arguments[0]= 0; 1111 | final_osmconvert_arguments_p= final_osmconvert_arguments; 1112 | old_file= NULL; 1113 | old_timestamp= 0; 1114 | new_file= NULL; 1115 | new_file_is_o5= false; 1116 | new_file_is_pbf= false; 1117 | new_file_is_changefile= false; 1118 | new_file_is_gz= false; 1119 | max_update_range= 250*86400; 1120 | process_minutely= process_hourly= process_daily= process_sporadic= 1121 | false; 1122 | no_minutely= no_hourly= no_daily= false; 1123 | if(file_exists(global_osmconvert_program)) 1124 | // must be Linux (no ".exe" at the end) AND 1125 | // osmconvert program seems to be in this directory 1126 | global_osmconvert_program= global_osmconvert_program_here_in_dir; 1127 | 1128 | // read command line parameters 1129 | if(argc<=1) { // no command line parameters given 1130 | fprintf(stderr,"osmupdate " VERSION "\n" 1131 | "Updates .osm, .o5m, .pbf files, downloads .osc, .o5c files.\n" 1132 | "To get detailed help, please enter: ./osmupdate -h\n"); 1133 | return 0; // end the program, because without having parameters 1134 | // we do not know what to do; 1135 | } 1136 | while(--argc>0) { // for every parameter in command line 1137 | argv++; // switch to next parameter; as the first one is just 1138 | // the program name, we must do this prior reading the 1139 | // first 'real' parameter; 1140 | a= argv[0]; 1141 | if(loglevel>0) // verbose mode 1142 | fprintf(stderr,"osmupdate Parameter: %.2000s\n",a); 1143 | if(strcmp(a,"-h")==0 || strcmp(a,"-help")==0 || 1144 | strcmp(a,"--help")==0) { // user wants help text 1145 | fprintf(stdout,"%s",helptext); // print help text 1146 | // (took "%s", to prevent oversensitive compiler reactions) 1147 | return 0; 1148 | } 1149 | if((strcmp(a,"-v")==0 || strcmp(a,"--verbose")==0 || 1150 | strzcmp(a,"-v=")==0 || strzcmp(a,"--verbose=")==0) && 1151 | loglevel==0) { // test mode - if not given already 1152 | char* sp; 1153 | 1154 | sp= strchr(a,'='); 1155 | if(sp!=NULL) loglevel= sp[1]-'0'; else loglevel= 1; 1156 | if(loglevel<1) loglevel= 1; 1157 | if(loglevel>MAXLOGLEVEL) loglevel= MAXLOGLEVEL; 1158 | if(a[1]=='-') { // must be "--verbose" and not "-v" 1159 | if(loglevel==1) 1160 | fprintf(stderr,"osmupdate: Verbose mode.\n"); 1161 | else 1162 | fprintf(stderr,"osmupdate: Verbose mode %i.\n",loglevel); 1163 | } 1164 | continue; // take next parameter 1165 | } 1166 | if(strzcmp(a,"--max-days=")==0) { // maximum update range 1167 | max_update_range= (int64_t)strtouint32(a+11)*86400; 1168 | continue; // take next parameter 1169 | } 1170 | if((strzcmp(a,"-t=")==0 || strzcmp(a,"--tempfiles=")==0) && 1171 | global_tempfile_name[0]==0) { 1172 | // user-defined prefix for names of temporary files 1173 | strmcpy(global_tempfile_name,strchr(a,'=')+1, 1174 | sizeof(global_tempfile_name)-50); 1175 | continue; // take next parameter 1176 | } 1177 | if(strzcmp(a,"--keep-tempfiles")==0) { 1178 | // temporary files shall not be deleted at program end 1179 | global_keep_tempfiles= true; 1180 | continue; // take next parameter 1181 | } 1182 | if(strzcmp(a,"--trust-tempfiles")==0) { 1183 | // cached files are considered to be intact 1184 | global_trust_tempfiles= true; 1185 | continue; // take next parameter 1186 | } 1187 | if(strzcmp(a,"--compression-level=")==0) { 1188 | // gzip compression level 1189 | static char gzip_par[3]= ""; 1190 | 1191 | if(a[20]<'1' || a[20]>'9' || a[21]!=0) { 1192 | PINFO("Range error. Changed to --compression-level=3") 1193 | gzip_par[0]= '-'; gzip_par[1]= '3'; gzip_par[2]= 0; 1194 | } 1195 | else { 1196 | gzip_par[0]= '-'; gzip_par[1]= a[20]; gzip_par[2]= 0; 1197 | global_gzip_parameters= gzip_par; 1198 | } 1199 | continue; // take next parameter 1200 | } 1201 | if(strzcmp(a,"--max-merge=")==0) { 1202 | // maximum number of parallely processed changefiles 1203 | global_max_merge= strtouint32(a+12); 1204 | if(global_max_merge<2) { 1205 | global_max_merge= 2; 1206 | PINFO("Range error. Increased to --max-merge=2") 1207 | } 1208 | if(global_max_merge>max_number_of_changefiles_in_cache) { 1209 | global_max_merge= max_number_of_changefiles_in_cache; 1210 | PINFOv("Range error. Decreased to --max-merge=%i", 1211 | max_number_of_changefiles_in_cache) 1212 | } 1213 | continue; // take next parameter 1214 | } 1215 | if(strzcmp(a,"--minute")==0) { // process minutely data 1216 | // accept "--minute" as well as old syntax "--minutely" 1217 | process_minutely= true; 1218 | continue; // take next parameter 1219 | } 1220 | if(strzcmp(a,"--hour")==0) { // process hourly data 1221 | // accept "--hour" as well as old syntax "--hourly" 1222 | process_hourly= true; 1223 | continue; // take next parameter 1224 | } 1225 | if(strcmp(a,"--day")==0 || strzcmp(a,"--daily")==0) { 1226 | // process daily data; 1227 | // accept "--day" as well as old syntax "--daily" 1228 | process_daily= true; 1229 | continue; // take next parameter 1230 | } 1231 | if(strcmp(a,"--sporadic")==0) { 1232 | // process sporadic data; 1233 | process_sporadic= true; 1234 | continue; // take next parameter 1235 | } 1236 | if((strzcmp(a,"--base-url=")==0 && a[11]!=0) || 1237 | (strzcmp(a,"--planet-url=")==0 && a[13]!=0)) { 1238 | // change base url 1239 | // the second option keyword is deprecated but still supported 1240 | const char* ap; 1241 | char* sp; 1242 | 1243 | ap= a+11; 1244 | if(a[2]=='p') ap+= 2; 1245 | if(strcmp(ap,"mirror")==0) 1246 | strcpy(global_base_url,"ftp://ftp5.gwdg.de/pub/misc/" 1247 | "openstreetmap/planet.openstreetmap.org/replication"); 1248 | else if(strstr(ap,"://")!=NULL) 1249 | strmcpy(global_base_url,ap,sizeof(global_base_url)-1); 1250 | else { 1251 | strcpy(global_base_url,"http://"); 1252 | strmcpy(global_base_url+7,ap,sizeof(global_base_url)-8); 1253 | } 1254 | sp= strchr(global_base_url,0); 1255 | if(sp>global_base_url && sp[-1]=='/') 1256 | sp[-1]= 0; 1257 | continue; // take next parameter 1258 | } 1259 | if(strzcmp(a,"--base-url-suffix=")==0 && a[20]!=0) { 1260 | // change base url suffix 1261 | strMcpy(global_base_url_suffix,a+18); 1262 | continue; // take next parameter 1263 | } 1264 | if(strzcmp(a,"--planet-url-suffix=")==0 && a[20]!=0) { 1265 | // change base url suffix (this option keyword is deprecated) 1266 | strMcpy(global_base_url_suffix,a+20); 1267 | continue; // take next parameter 1268 | } 1269 | if(a[0]=='-') { 1270 | // command line argument not recognized by osmupdate 1271 | // store it so we can pass it to osmconvert later 1272 | int len; 1273 | 1274 | len= strlen(a)+3; 1275 | if(osmconvert_arguments_p-global_osmconvert_arguments+len 1276 | >=sizeof(global_osmconvert_arguments) || 1277 | final_osmconvert_arguments_p-final_osmconvert_arguments+len 1278 | >=sizeof(final_osmconvert_arguments)) { 1279 | PERR("too many command line arguments for osmconvert.") 1280 | return 1; 1281 | } 1282 | if(strcmp(a,"--complete-ways")==0 || 1283 | strcmp(a,"--complex-ways")==0 || 1284 | strcmp(a,"--drop-brokenrefs")==0 || 1285 | strcmp(a,"--drop-broken-refs")==0) { 1286 | WARNv("option %.80s does not work with updates.",a) 1287 | continue; // take next parameter 1288 | } 1289 | if(strzcmp(a,"-b=")!=0 && strzcmp(a,"-B=")!=0) { 1290 | // not a bounding box and not a bounding polygon 1291 | *osmconvert_arguments_p++= ' '; 1292 | *osmconvert_arguments_p++= '\"'; 1293 | osmconvert_arguments_p= stpesccpy(osmconvert_arguments_p,a); 1294 | *osmconvert_arguments_p++= '\"'; 1295 | *osmconvert_arguments_p= 0; 1296 | } 1297 | *final_osmconvert_arguments_p++= ' '; 1298 | *final_osmconvert_arguments_p++= '\"'; 1299 | final_osmconvert_arguments_p= 1300 | stpesccpy(final_osmconvert_arguments_p,a); 1301 | *final_osmconvert_arguments_p++= '\"'; 1302 | *final_osmconvert_arguments_p= 0; 1303 | continue; // take next parameter 1304 | } 1305 | if(old_timestamp==0) { 1306 | old_timestamp= strtimetosint64(a); 1307 | if(old_timestamp!=0) // this is a valid timestamp 1308 | continue; // take next parameter 1309 | } 1310 | // here: parameter must be a file name 1311 | if(old_file==NULL && old_timestamp==0) { // name of the old file 1312 | old_file= a; 1313 | continue; // take next parameter 1314 | } 1315 | if(new_file==NULL) { // name of the new file 1316 | new_file= a; 1317 | new_file_is_o5= 1318 | strycmp(new_file,".o5m")==0 || strycmp(new_file,".o5c")==0 || 1319 | strycmp(new_file,".o5m.gz")==0 || 1320 | strycmp(new_file,".o5c.gz")==0; 1321 | new_file_is_pbf= 1322 | strycmp(new_file,".pbf")==0; 1323 | new_file_is_changefile= 1324 | strycmp(new_file,".osc")==0 || strycmp(new_file,".o5c")==0 || 1325 | strycmp(new_file,".osc.gz")==0 || 1326 | strycmp(new_file,".o5c.gz")==0; 1327 | new_file_is_gz= strycmp(new_file,".gz")==0; 1328 | continue; // take next parameter 1329 | } 1330 | } // end for every parameter in command line 1331 | 1332 | /* create tempfile directory for cached timestamps and changefiles */ { 1333 | char *sp; 1334 | 1335 | if(strlen(global_tempfile_name)<2) // not set yet 1336 | strcpy(global_tempfile_name,"osmupdate_temp"DIRSEPS"temp"); 1337 | // take default 1338 | sp= strchr(global_tempfile_name,0); 1339 | if(sp[-1]==DIRSEP) // it's a bare directory 1340 | strcpy(sp,"temp"); // add a file name prefix 1341 | strMcpy(tempfile_directory,global_tempfile_name); 1342 | sp= strrchr(tempfile_directory,DIRSEP); 1343 | // get last directory separator 1344 | if(sp!=NULL) *sp= 0; // if found any, cut the string here 1345 | #if __WIN32__ 1346 | mkdir(tempfile_directory); 1347 | #else 1348 | mkdir(tempfile_directory,0700); 1349 | #endif 1350 | } 1351 | 1352 | // get file timestamp of OSM input file 1353 | if(old_timestamp==0) { // no timestamp given by the user 1354 | if(old_file==NULL) { // no file name given for the old OSM file 1355 | PERR("Specify at least the old OSM file's name or its timestamp.") 1356 | return 1; 1357 | } 1358 | if(!file_exists(old_file)) { // old OSM file does not exist 1359 | PERRv("Old OSM file does not exist: %.80s",old_file); 1360 | return 1; 1361 | } 1362 | old_timestamp= get_file_timestamp(old_file); 1363 | if(old_timestamp==0) { 1364 | PERRv("Old OSM file does not contain a timestamp: %.80s",old_file); 1365 | PERR("Please specify the timestamp manually, e.g.: " 1366 | "2011-07-15T23:30:00Z"); 1367 | return 1; 1368 | } 1369 | } // end no timestamp given by the user 1370 | 1371 | // parameter consistency check 1372 | if(new_file==NULL) { 1373 | PERR("No output file was specified."); 1374 | return 1; 1375 | } 1376 | if(old_file!=NULL && strcmp(old_file,new_file)==0) { 1377 | PERR("Input file and output file are identical."); 1378 | return 1; 1379 | } 1380 | if(old_file==NULL && !new_file_is_changefile) { 1381 | PERR("If no old OSM file is specified, osmupdate can only " 1382 | "generate a changefile."); 1383 | return 1; 1384 | } 1385 | 1386 | // initialize sequence numbers and timestamps 1387 | minutely_sequence_number= hourly_sequence_number= 1388 | daily_sequence_number= sporadic_sequence_number= 0; 1389 | minutely_timestamp= hourly_timestamp= 1390 | daily_timestamp= sporadic_timestamp= 0; 1391 | 1392 | // care about user defined processing categories 1393 | if(process_minutely || process_hourly || 1394 | process_daily || process_sporadic) { 1395 | // user wants specific type(s) of changefiles to be processed 1396 | if(!process_minutely) no_minutely= true; 1397 | if(!process_hourly) no_hourly= true; 1398 | if(!process_daily) no_daily= true; 1399 | } 1400 | else { 1401 | // try to get sporadic timestamp 1402 | sporadic_timestamp= get_newest_changefile_timestamp( 1403 | cft_SPORADIC,&sporadic_sequence_number); 1404 | if(sporadic_timestamp!=0) { 1405 | // there is a timestamp at the highest directory level, 1406 | // this must be a so-called sporadic timestamp 1407 | if(loglevel>0) { 1408 | PINFO("Found status information in base URL root.") 1409 | PINFO("Ignoring subdirectories \"minute\", \"hour\", \"day\".") 1410 | } 1411 | process_sporadic= true; // let's take it 1412 | no_minutely= no_hourly= no_daily= true; 1413 | } 1414 | } 1415 | 1416 | // get last timestamp for each, minutely, hourly, daily, 1417 | // and sporadic diff files 1418 | if(!no_minutely) { 1419 | minutely_timestamp= get_newest_changefile_timestamp( 1420 | cft_MINUTELY,&minutely_sequence_number); 1421 | if(minutely_timestamp==0) { 1422 | PERR("Could not get the newest minutely timestamp from the Internet.") 1423 | return 1; 1424 | } 1425 | } 1426 | if(!no_hourly) { 1427 | hourly_timestamp= get_newest_changefile_timestamp( 1428 | cft_HOURLY,&hourly_sequence_number); 1429 | if(hourly_timestamp==0) { 1430 | PERR("Could not get the newest hourly timestamp from the Internet.") 1431 | return 1; 1432 | } 1433 | } 1434 | if(!no_daily) { 1435 | daily_timestamp= get_newest_changefile_timestamp( 1436 | cft_DAILY,&daily_sequence_number); 1437 | if(daily_timestamp==0) { 1438 | PERR("Could not get the newest daily timestamp from the Internet.") 1439 | return 1; 1440 | } 1441 | } 1442 | if(process_sporadic && sporadic_timestamp==0) { 1443 | sporadic_timestamp= get_newest_changefile_timestamp( 1444 | cft_SPORADIC,&sporadic_sequence_number); 1445 | if(sporadic_timestamp==0) { 1446 | PERR("Could not get the newest sporadic timestamp " 1447 | "from the Internet.") 1448 | return 1; 1449 | } 1450 | } 1451 | 1452 | // check maximum update range 1453 | if(minutely_timestamp-old_timestamp>max_update_range) { 1454 | // update range too large 1455 | int days; 1456 | days= (int)((minutely_timestamp-old_timestamp+86399)/86400); 1457 | PERRv("Update range too large: %i days.",days) 1458 | PINFOv("To allow such a wide range, add: --max-days=%i",days) 1459 | return 1; 1460 | } // update range too large 1461 | 1462 | // clear last hourly timestamp if 1463 | // OSM old file's timestamp > latest hourly timestamp - 30 minutes 1464 | if(old_timestamp>hourly_timestamp-30*60 && !no_minutely) 1465 | hourly_timestamp= 0; // (let's take minutely updates) 1466 | 1467 | // clear last daily timestamp if 1468 | // OSM file timestamp > latest daily timestamp - 16 hours 1469 | if(old_timestamp>daily_timestamp-16*3600 && 1470 | !(no_hourly && no_minutely)) 1471 | daily_timestamp= 0; // (let's take hourly and minutely updates) 1472 | 1473 | // initialize start timestamp 1474 | timestamp= 0; 1475 | if(timestamphourly_timestamp && 1486 | next_timestamp>old_timestamp) { 1487 | timestamp= next_timestamp; 1488 | process_changefile(cft_MINUTELY,minutely_sequence_number,timestamp); 1489 | minutely_sequence_number--; 1490 | next_timestamp= get_changefile_timestamp( 1491 | cft_MINUTELY,minutely_sequence_number); 1492 | } 1493 | } 1494 | 1495 | // get and process hourly diff files from last hourly timestamp 1496 | // backward; stop just before last daily timestamp or OSM 1497 | // file timestamp has been reached; 1498 | if(hourly_timestamp!=0) { 1499 | next_timestamp= timestamp; 1500 | while(next_timestamp>daily_timestamp && 1501 | next_timestamp>old_timestamp) { 1502 | timestamp= next_timestamp; 1503 | process_changefile(cft_HOURLY,hourly_sequence_number,timestamp); 1504 | hourly_sequence_number--; 1505 | next_timestamp= get_changefile_timestamp( 1506 | cft_HOURLY,hourly_sequence_number); 1507 | } 1508 | } 1509 | 1510 | // get and process daily diff files from last daily timestamp 1511 | // backward; stop just before OSM file timestamp has been reached; 1512 | if(daily_timestamp!=0) { 1513 | next_timestamp= timestamp; 1514 | while(next_timestamp>old_timestamp) { 1515 | timestamp= next_timestamp; 1516 | process_changefile(cft_DAILY,daily_sequence_number,timestamp); 1517 | daily_sequence_number--; 1518 | next_timestamp= get_changefile_timestamp( 1519 | cft_DAILY,daily_sequence_number); 1520 | } 1521 | } 1522 | 1523 | // get and process sporadic diff files from last sporadic timestamp 1524 | // backward; stop just before OSM file timestamp has been reached; 1525 | if(sporadic_timestamp!=0) { 1526 | next_timestamp= timestamp; 1527 | while(next_timestamp>old_timestamp) { 1528 | timestamp= next_timestamp; 1529 | process_changefile( 1530 | cft_SPORADIC,sporadic_sequence_number,timestamp); 1531 | sporadic_sequence_number--; 1532 | next_timestamp= get_changefile_timestamp( 1533 | cft_SPORADIC,sporadic_sequence_number); 1534 | } 1535 | } 1536 | 1537 | // process remaining files which may still wait in the cache; 1538 | process_changefile(0,0,0); 1539 | 1540 | /* create requested output file */ { 1541 | char master_cachefile_name[400]; 1542 | char command[2000],*command_p; 1543 | char* command_e= command+sizeof(command); 1544 | char result[1000]; 1545 | 1546 | if(loglevel>0) 1547 | PINFO("Creating output file.") 1548 | strcpy(stpmcpy(master_cachefile_name,global_tempfile_name, 1549 | sizeof(master_cachefile_name)-5),".8"); 1550 | if(!file_exists(master_cachefile_name)) { 1551 | if(old_file==NULL) 1552 | PINFO("There is no changefile since this timestamp.") 1553 | else 1554 | PINFO("Your OSM file is already up-to-date.") 1555 | return 21; 1556 | } 1557 | command_p= command; 1558 | if(new_file_is_changefile) { // changefile 1559 | if(new_file_is_gz) { // compressed 1560 | if(new_file_is_o5) { // .o5c.gz 1561 | stecpy(&command_p,command_e,"gzip "); 1562 | stecpy(&command_p,command_e,global_gzip_parameters); 1563 | stecpy(&command_p,command_e," <\""); 1564 | steesccpy(&command_p,command_e,master_cachefile_name); 1565 | stecpy(&command_p,command_e,"\" >\""); 1566 | steesccpy(&command_p,command_e,new_file); 1567 | stecpy(&command_p,command_e,"\""); 1568 | } 1569 | else { // .osc.gz 1570 | stecpy(&command_p,command_e,global_osmconvert_program); 1571 | stecpy(&command_p,command_e," "); 1572 | stecpy(&command_p,command_e,global_osmconvert_arguments); 1573 | stecpy(&command_p,command_e," \""); 1574 | steesccpy(&command_p,command_e,master_cachefile_name); 1575 | stecpy(&command_p,command_e,"\" --out-osc |gzip "); 1576 | stecpy(&command_p,command_e,global_gzip_parameters); 1577 | stecpy(&command_p,command_e," >\""); 1578 | steesccpy(&command_p,command_e,new_file); 1579 | stecpy(&command_p,command_e,"\""); 1580 | } 1581 | shell_command(command,result); 1582 | } // compressed 1583 | else { // uncompressed 1584 | if(new_file_is_o5) // .o5c 1585 | rename(master_cachefile_name,new_file); 1586 | else { // .osc 1587 | stecpy(&command_p,command_e,global_osmconvert_program); 1588 | stecpy(&command_p,command_e," "); 1589 | stecpy(&command_p,command_e,global_osmconvert_arguments); 1590 | stecpy(&command_p,command_e," \""); 1591 | steesccpy(&command_p,command_e,master_cachefile_name); 1592 | stecpy(&command_p,command_e,"\" --out-osc >\""); 1593 | steesccpy(&command_p,command_e,new_file); 1594 | stecpy(&command_p,command_e,"\""); 1595 | shell_command(command,result); 1596 | } 1597 | } // uncompressed 1598 | } // changefile 1599 | else { // OSM file 1600 | #if 0 1601 | if(loglevel>=2) { 1602 | PINFOv("oc %s",global_osmconvert_program) 1603 | PINFOv("fa %s",final_osmconvert_arguments) 1604 | PINFOv("of %s",old_file) 1605 | PINFOv("mc %s",master_cachefile_name) 1606 | } 1607 | #endif 1608 | stecpy(&command_p,command_e,global_osmconvert_program); 1609 | stecpy(&command_p,command_e," "); 1610 | stecpy(&command_p,command_e,final_osmconvert_arguments); 1611 | stecpy(&command_p,command_e," \""); 1612 | steesccpy(&command_p,command_e,old_file); 1613 | stecpy(&command_p,command_e,"\" \""); 1614 | steesccpy(&command_p,command_e,master_cachefile_name); 1615 | if(new_file_is_gz) { // compressed 1616 | if(new_file_is_o5) { // .o5m.gz 1617 | stecpy(&command_p,command_e,"\" --out-o5m |gzip "); 1618 | stecpy(&command_p,command_e,global_gzip_parameters); 1619 | stecpy(&command_p,command_e," >\""); 1620 | steesccpy(&command_p,command_e,new_file); 1621 | stecpy(&command_p,command_e,"\""); 1622 | } 1623 | else { // .osm.gz 1624 | stecpy(&command_p,command_e,"\" --out-osm |gzip "); 1625 | stecpy(&command_p,command_e,global_gzip_parameters); 1626 | stecpy(&command_p,command_e," >\""); 1627 | steesccpy(&command_p,command_e,new_file); 1628 | stecpy(&command_p,command_e,"\""); 1629 | } 1630 | } // compressed 1631 | else { // uncompressed 1632 | if(new_file_is_pbf) { // .pbf 1633 | stecpy(&command_p,command_e,"\" --out-pbf >\""); 1634 | steesccpy(&command_p,command_e,new_file); 1635 | stecpy(&command_p,command_e,"\""); 1636 | } 1637 | else if(new_file_is_o5) { // .o5m 1638 | stecpy(&command_p,command_e,"\" --out-o5m >\""); 1639 | steesccpy(&command_p,command_e,new_file); 1640 | stecpy(&command_p,command_e,"\""); 1641 | } 1642 | else { // .osm 1643 | stecpy(&command_p,command_e,"\" --out-osm >\""); 1644 | steesccpy(&command_p,command_e,new_file); 1645 | stecpy(&command_p,command_e,"\""); 1646 | } 1647 | } // uncompressed 1648 | shell_command(command,result); 1649 | } // OSM file 1650 | if(loglevel<2) 1651 | unlink(master_cachefile_name); 1652 | } // create requested output file 1653 | 1654 | // delete tempfiles 1655 | if(global_keep_tempfiles) { // tempfiles shall be kept 1656 | if(loglevel>0) 1657 | PINFO("Keeping temporary files.") 1658 | } // tempfiles shall be kept 1659 | else { // tempfiles shall be deleted 1660 | char command[500],*command_p,result[1000]; 1661 | char* command_e= command+sizeof(command); 1662 | 1663 | if(loglevel>0) 1664 | PINFO("Deleting temporary files.") 1665 | command_p= command; 1666 | stecpy(&command_p,command_e,DELFILE" \""); 1667 | steesccpy(&command_p,command_e,global_tempfile_name); 1668 | stecpy(&command_p,command_e,"\".*"); 1669 | shell_command(command,result); 1670 | rmdir(tempfile_directory); 1671 | } // tempfiles shall be deleted 1672 | 1673 | if(main_return_value==0 && loglevel>0) 1674 | PINFO("Completed successfully.") 1675 | 1676 | return main_return_value; 1677 | } // end main() 1678 | 1679 | --------------------------------------------------------------------------------