├── 10to11 ├── cleanup.sh ├── migrate.sh └── sql │ ├── common │ ├── preUpgradeRun.sql │ └── reduceSysLog.sql │ └── projectspecific │ ├── preWizard.sql │ └── sys_template.sql ├── README.md ├── composer.json ├── v12 ├── cleanup.sh ├── migrate.sh └── sql │ ├── common │ ├── preUpgradeRun.sql │ └── reduceSysLog.sql │ └── projectspecific │ ├── content.sql │ ├── preWizard.sql │ ├── rights.sql │ └── template.sql └── v13 ├── cleanup.sh ├── migrate.sh └── sql ├── common ├── preUpgradeRun.sql └── reduceSysLog.sql └── projectspecific ├── content.sql ├── preWizard.sql ├── rights.sql └── template.sql /10to11/cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # stop script run when error occurs 3 | set -euo pipefail 4 | 5 | # use beginning with 8LTS, see typo3_src/typo3/sysext/lowlevel/README.rst 6 | 7 | SCRIPT_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" 8 | # depending on where you have your local extensions, you need to adapt the path here 9 | # this is working out of the box, if you follow 10 | # https://docs.typo3.org/m/typo3/guide-installation/master/en-us/MigrateToComposer/BestPractices.html 11 | PROJECT_ROOT=`readlink -f "${SCRIPT_DIR}/../../../"` 12 | TYPO3_CORE_BIN='./vendor/bin/typo3' 13 | 14 | echo -e '* start cleanup in ${PROJECT_ROOT}' 15 | 16 | date 17 | echo -e '** cleanup:deletedrecords' 18 | echo -e 'for big so far badly maintained systems right upfront: start with deleted to get rid of those deleted records. Nearly all other tasks start with full pagetree (including deleted records)'; 19 | ${TYPO3_CORE_BIN} -q cleanup:deletedrecords 20 | date 21 | 22 | # now as suggested in typo3_src/typo3/sysext/lowlevel/README.rst 23 | echo -e '** cleanup:orphanrecords' 24 | ${TYPO3_CORE_BIN} -q cleanup:orphanrecords 25 | date 26 | 27 | echo -e '** cleanup:multiplereferencedfiles --update-refindex' 28 | ${TYPO3_CORE_BIN} -q cleanup:multiplereferencedfiles --update-refindex 29 | date 30 | 31 | echo -e '** cleanup:deletedrecords' 32 | ${TYPO3_CORE_BIN} -q cleanup:deletedrecords 33 | date 34 | 35 | 36 | echo -e '** cleanup:missingrelations --update-refindex' 37 | ${TYPO3_CORE_BIN} -q cleanup:missingrelations --update-refindex 38 | date 39 | 40 | echo -e '** cleanup:flexforms' 41 | ${TYPO3_CORE_BIN} -q cleanup:flexforms 42 | date 43 | 44 | # just point out missing files, but do NOT remove reference to it 45 | echo -e '** cleanup:missingfiles --dry-run --update-refindex' 46 | ${TYPO3_CORE_BIN} -q cleanup:missingfiles --dry-run --update-refindex 47 | date 48 | 49 | echo -e '** cleanup:lostfiles --update-refindex' 50 | ${TYPO3_CORE_BIN} -q cleanup:lostfiles --update-refindex 51 | date 52 | 53 | # and again in case the above commands freeed some more stuff 54 | echo -e '** cleanup:orphanrecords' 55 | ${TYPO3_CORE_BIN} -q cleanup:orphanrecords 56 | date 57 | 58 | echo -e '** cleanup:deletedrecords' 59 | ${TYPO3_CORE_BIN} -q cleanup:deletedrecords 60 | date 61 | 62 | echo -e '** done!!!' 63 | -------------------------------------------------------------------------------- /10to11/migrate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # stop script run when error occurs 3 | set -e 4 | 5 | SCRIPT_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" 6 | # depending on where you have your local extensions, you need to adapt the path here 7 | # this is working out of the box, if you follow 8 | # https://docs.typo3.org/m/typo3/guide-installation/master/en-us/MigrateToComposer/BestPractices.html 9 | # this might be overwritten in call from upgrade scripts on online runs 10 | projectRoot=`readlink -f "${SCRIPT_DIR}/../../../"` 11 | phpBin='' 12 | composerBin='composer' 13 | 14 | 15 | # first get named parameters (only 1 character possible), see https://unix.stackexchange.com/questions/129391/passing-named-arguments-to-shell-scripts 16 | while getopts ":d:r:s:p:c:" opt; do 17 | case $opt in 18 | d) dumpFile="$OPTARG" 19 | ;; 20 | r) projectRoot="$OPTARG" 21 | ;; 22 | p) phpBin="$OPTARG" 23 | ;; 24 | c) composerBin="$OPTARG" 25 | ;; 26 | \?) echo "option -$OPTARG not given" >&2 27 | ;; 28 | esac 29 | done 30 | 31 | # now use default values or given parameter 32 | PROJECT_ROOT="${projectRoot}" 33 | TYPO3_CONSOLE_BIN="${phpBin} ./vendor/bin/typo3" 34 | COMPOSER_BIN="${phpBin} ${composerBin}" 35 | 36 | # change to PROJECT_DIR to have the same commands here as you would use manually on command line 37 | echo -e "* switch to TYPO3 root" 38 | cd "${PROJECT_ROOT}" 39 | 40 | # if valid path to db dump is given, initialize DB with this dump 41 | if [ -f "${PROJECT_ROOT}/$dumpFile" ]; then 42 | echo -e "* resetting database, drop all tables" 43 | echo "show tables" | ${TYPO3_CONSOLE_BIN} database:import | grep -v Tables_in | grep -v "+" | awk '{print "SET FOREIGN_KEY_CHECKS = 0;drop table " $1 ";"}' | ${TYPO3_CONSOLE_BIN} database:import 44 | echo -e "* resetting database, import old DB, this can take a while (and modify for local dev machines, if needed)" 45 | ${TYPO3_CONSOLE_BIN} database:import < "${PROJECT_ROOT}/$dumpFile" 46 | else 47 | echo -e "* use existing database for this migration, since no dump file is given as parameter" 48 | fi; 49 | 50 | # we might need to fix stuff in DB for commands to run properly 51 | echo -e "* fix blocking DB issues" 52 | ${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/common/preUpgradeRun.sql" 53 | 54 | echo -e "* drop obsolete cache tables starting with cf_" 55 | echo "show tables like 'cf_%'" | ${TYPO3_CONSOLE_BIN} database:import | grep -v Tables_in | grep -v "+" | awk '{print "drop table " $1 ";"}' | ${TYPO3_CONSOLE_BIN} database:import 56 | 57 | # make sure we do the initial upgrade stuff again, no matter where current DB comes from 58 | echo -e "* re-activate initial upgrade step" 59 | ${TYPO3_CONSOLE_BIN} configuration:remove EXTCONF/helhum-typo3-console/initialUpgradeDone --force 60 | 61 | echo -e "* reduces sys_log" 62 | ${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/common/reduceSysLog.sql" 63 | 64 | echo -e "* setup all existing extensions" 65 | ${TYPO3_CONSOLE_BIN} extension:setup 66 | 67 | echo -e "* flush/warmup cache" 68 | ${TYPO3_CONSOLE_BIN} -q cache:flush 69 | ${TYPO3_CONSOLE_BIN} -q cache:warmup 70 | 71 | # you might need to fix some glitches before running the wizards 72 | echo -e "* fix some glitches before running the wizards" 73 | ${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/projectspecific/preWizard.sql" 74 | 75 | echo -e "* run upgrade wizards - core" 76 | ${TYPO3_CONSOLE_BIN} upgrade:run 77 | 78 | # in case you want to avoid needing to interact as much as possible when running the script, try this instead of the previous line 79 | #${TYPO3_CONSOLE_BIN} upgrade:run --no-interaction 80 | #echo -e "* re-run upgrade wizards which need confirmations - if you want them to run" 81 | #${TYPO3_CONSOLE_BIN} upgrade:run svgFilesSanitization 82 | 83 | echo -e "* run upgrade wizards - extensions" 84 | echo -e "** TODO: fill your extensions here" 85 | 86 | # simpler things can be done via sql - the names of the files don't matter, just make sure the path is correct 87 | echo -e "* run sql scripts for simpler migration stuff" 88 | echo -e "** run: sys_template.sql" 89 | ${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/projectspecific/sys_template.sql" 90 | #echo -e "** run: youNameIt.sql" 91 | #${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/projectspecific/youNameIt.sql" 92 | 93 | echo -e "* alter all remaining myisam -> innodb" 94 | echo "show table status where Engine='MyISAM';" | ${TYPO3_CONSOLE_BIN} database:import | awk '{print "ALTER TABLE "$1" ENGINE = InnoDB;"}' | ${TYPO3_CONSOLE_BIN} database:import 95 | 96 | echo -e "* check if a wizard is still missing - add it to the script a few lines above" 97 | ${TYPO3_CONSOLE_BIN} upgrade:list 98 | 99 | echo -e "* update language packs" 100 | ${TYPO3_CONSOLE_BIN} language:update 101 | 102 | #echo -e "* temporarily add admin user in case online users are unknown or not available locally" 103 | #${TYPO3_CONSOLE_BIN} backend:createadmin admin your..fancy..pwd 104 | 105 | echo -e "* exit for now, we suggest to run full script as soon as all extensions are ready and you start with the final iterations" 106 | exit; 107 | 108 | # activate only if you are pretty sure all is fine (after several successful migration runs) 109 | echo -e "* database:updateschema destructive" 110 | # first time rename to zzz_deleted 111 | ${TYPO3_CONSOLE_BIN} database:updateschema "destructive" 112 | # second time really drop 113 | ${TYPO3_CONSOLE_BIN} database:updateschema -v "destructive" 114 | 115 | echo -e "* and finally cleanup DB, might better be placed in bin/online/fullMigration.sh if you have that" 116 | bash ${SCRIPT_DIR}/cleanup.sh 117 | -------------------------------------------------------------------------------- /10to11/sql/common/preUpgradeRun.sql: -------------------------------------------------------------------------------- 1 | -- if we have utf8-mb4, we get sql error because index gets too long. The queries would be executed in the wrong order by DB compare 2 | -- alter table `sys_refindex` drop index lookup_string; 3 | -- ALTER TABLE sys_refindex CHANGE ref_string `ref_string` VARCHAR(1024) DEFAULT '' NOT NULL; 4 | -- CREATE INDEX `lookup_string` ON sys_refindex (ref_string(255)); 5 | 6 | -- there should not be pages in default language having l10n_parent set 7 | -- might break i.e. sitemap rendering 8 | update pages set l10n_parent = 0 where sys_language_uid = 0; 9 | 10 | -- in case you have updated news from an older version and get an error like "Data truncated for column 'related_links' at row xxx" 11 | -- UPDATE tx_news_domain_model_news SET related_links=0 WHERE related_links IS NULL; 12 | -------------------------------------------------------------------------------- /10to11/sql/common/reduceSysLog.sql: -------------------------------------------------------------------------------- 1 | delete FROM `sys_log` where details like '%has cleared the cache%'; 2 | delete FROM `sys_log` where details like '[scheduler%'; 3 | delete FROM `sys_log` where details like 'User %s logged in from%'; 4 | delete FROM `sys_log` where tstamp < UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 360 DAY)); 5 | delete FROM `sys_log` where details like '%was deleted unrecoverable%'; 6 | -- errors from old version should not be relevant in new version 7 | delete FROM `sys_log` where error>0; -------------------------------------------------------------------------------- /10to11/sql/projectspecific/preWizard.sql: -------------------------------------------------------------------------------- 1 | -- fill in whatever you need for your project -------------------------------------------------------------------------------- /10to11/sql/projectspecific/sys_template.sql: -------------------------------------------------------------------------------- 1 | -- fill in whatever you need for your project 2 | -- update sys_template set include_static_file='EXT:fluid_styled_content/Configuration/TypoScript/,EXT:fluid_styled_content/Configuration/TypoScript/Styling/,EXT:bootstrap_package/Configuration/TypoScript,EXT:bootstrap_package/Configuration/TypoScript/ContentElement' where pid=39; -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | This extension is a boilerplate to organize all necessary upgrade steps to bring TYPO3 Installations to higher levels. Thanks to Helmut Hummel for his great EXT:typo3_console, which is heavily used here. 3 | 4 | Beware: this is ONLY A BOILERPLATE which needs individual adaptions for each project. 5 | 6 | Basically - if you did all the adaptions - you can then just run a bash script to do the complete upgrade. We have boilerplates for most recent TYPO3-Versions. 7 | Since TYPO3 v11 LTS we also use versioning and keep only stuff for the latest LTS versions. Check out release v10.0.0 to get all boilerplates for older TYPO3-Versions. 8 | We suggest to use ddev for local development. 9 | 10 | # How to run the migration script 11 | These are examples we use most of the time: 12 | ``` 13 | # with ddev 14 | ddev exec bash packages/upgrader/v12/migrate.sh -d 'dump11.sql' 15 | # regular console (on customer servers) 16 | bash packages/upgrader/v12/migrate.sh -d 'dump11.sql' 17 | bash packages/upgrader/v12/migrate.sh -d 'dump-for-migration.sql' -p /usr/bin/php81 -c '/usr/bin/composer' 18 | ``` 19 | Script Parameters: 20 | 21 | **-d** (relative) path to your DB dump 22 | **-r** path to your project root, if you want to bring that in as parameter from another script 23 | **-p** path to php bin which should be used to call eg typo3 console 24 | **-c** path to composer if composer is not available just calling `composer` 25 | 26 | You can use the script to run the migration several times on your dev machine first until all looks good, then on the staging machine, and finally for the live upgrade, where you can be pretty sure that all will work well (and fast). 27 | 28 | # Questions and answers 29 | We welcome pull requests also for this little documentation, if you have questions or experiences you want to share. 30 | 31 | ## Target versions you can migrate to 32 | ### TYPO3 v10 LTS 33 | You can come from 7LTS, 8LTS or 9LTS. 34 | We are using EXT:core_upgrader there which was developed by @IchHabRecht for us and you. 35 | Check out our release v10 for that ooold kind of stuff. 36 | ### TYPO3 v11 LTS 37 | We removed all outdated stuff to keep the boilerplate lean and simple. Also TYPO3 Upgrade Wizards got better, less to do for us! 38 | We ran a lot of migrations from TYPO3 v10 LTS, but it should work also from TYPO3 v9 LTS. 39 | ### TYPO3 v12 LTS 40 | Since the TYPO3 LTS versions are getting better and better - Kudos to the Core team and all contributors! - from v12 on we switch to name only the target system in our folder structure. It seems to work great for migrations from TYPO3 v10 LTS and TYPO3 v11 LTS. 41 | In the future we try to support the last two LTS versions for each target version, as the core also has upgrade wizards for the last two LTS versions. 42 | 43 | ## How is the best procedure for organizing an upgrade with this extension 44 | Check out https://docs.typo3.org/m/typo3/guide-installation/12.4/en-us/Major/Index.html 45 | We aim to run all steps mentioned there in the Post Upgrade Tasks via our script. It is *only* meant for bringing the old DB smoothly into the new target version of TYPO3. All the coding and extension updates need to be done by you. 46 | 47 | Here is how we suggest to do it: 48 | * Clone your existing project (new DB, new folder), very easy with ddev 49 | * leave DB-dump for later use, you will probably have to run the script several times until all is fine 50 | * uninstall all local extensions (from typo3conf/ext) 51 | * switch to your new TYPO3 version with composer 52 | * upgrade and install all local extensions to versions which are compatible to your new TYPO3 version. You can also add upgraded extensions later on, if you want to try the upgrader as soon as possible. 53 | * install EXT:upgrader (copy it from https://github.com/elementareteilchen/upgrader to your local package folder) 54 | * work through what you need to change in your specific project for migrate.sh to run smoothly. See sql-files to get some ideas. We plan to add more examples from what we had to do in different projects to give more ideas how things can be done. 55 | * run the migration with the original sql dump (so far we never needed to set back the filestructure i.e. in fileadmin, no changes there - you could do that with rsync if needed) 56 | * fix what went wrong and re-run the script, again with the original sql dump 57 | * if all looks good on your machine, run the script on a staging server or the soon-to-be-live environment 58 | * if all looks good on the customer servers too, run the upgrade. We often do the following steps: 59 | * lock the backend for the current TYPO3 60 | * export db dump and rsync fileadmin to the location of the new TYPO3 version 61 | * `bash packages/upgrader/10to11/migrate.sh -dump11.sql | tee migration.log` 62 | * do stuff which still needs to be done manually, ideally nothing 63 | * test your new live instance 64 | * switch domain (or whatever you need to switch) to get the new system live 65 | * enjoy the praise of your customer and editors :) 66 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "elementareteilchen/upgrader", 3 | "type": "typo3-cms-extension", 4 | "license": "GPL-2.0+", 5 | "authors": [ 6 | { 7 | "name": "Franz Kugelmann", 8 | "email": "franz.kugelmann@elementare-teilchen.de" 9 | } 10 | ], 11 | "extra": { 12 | "typo3/cms": { 13 | "extension-key": "upgrader" 14 | } 15 | }, 16 | "autoload": { 17 | "psr-4": { 18 | "ElementareTeilchen\\Upgrader\\": "Classes" 19 | } 20 | }, 21 | "require": { 22 | "helhum/typo3-console": "^8.0.3" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /v12/cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SCRIPT_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" 4 | # depending on where you have your local extensions, you need to adapt the path here 5 | # this is working out of the box, if you follow 6 | # https://docs.typo3.org/m/typo3/guide-installation/master/en-us/MigrateToComposer/BestPractices.html 7 | PROJECT_ROOT=`readlink -f "${SCRIPT_DIR}/../../../"` 8 | TYPO3_CONSOLE_BIN='./vendor/bin/typo3' 9 | 10 | echo -e "* start cleanup in ${PROJECT_ROOT}" 11 | 12 | date +"%d.%m.%Y %T" 13 | echo -e '* typo3 -q cleanup:orphanrecords' 14 | ${TYPO3_CONSOLE_BIN} -q cleanup:orphanrecords 15 | 16 | date +"%d.%m.%Y %T" 17 | echo -e '* cleanup:deletedrecords' 18 | ${TYPO3_CONSOLE_BIN} cleanup:deletedrecords 19 | 20 | date +"%d.%m.%Y %T" 21 | echo -e '* cleanup:missingrelations --update-refindex' 22 | ${TYPO3_CONSOLE_BIN} cleanup:missingrelations --update-refindex 23 | 24 | date +"%d.%m.%Y %T" 25 | echo -e '* typo3 -q cleanup:flexforms' 26 | ${TYPO3_CONSOLE_BIN} -q cleanup:flexforms 27 | 28 | echo -e '** done!!!' 29 | -------------------------------------------------------------------------------- /v12/migrate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # stop script run when error occurs 3 | set -e 4 | 5 | SCRIPT_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" 6 | # depending on where you have your local extensions, you need to adapt the path here 7 | # this is working out of the box, if you follow 8 | # https://docs.typo3.org/m/typo3/guide-installation/master/en-us/MigrateToComposer/BestPractices.html 9 | # this might be overwritten in call from upgrade scripts on online runs 10 | projectRoot=`readlink -f "${SCRIPT_DIR}/../../../"` 11 | phpBin='' 12 | composerBin='composer' 13 | 14 | 15 | # first get named parameters (only 1 character possible), see https://unix.stackexchange.com/questions/129391/passing-named-arguments-to-shell-scripts 16 | while getopts ":d:r:s:p:c:" opt; do 17 | case $opt in 18 | d) dumpFile="$OPTARG" 19 | ;; 20 | r) projectRoot="$OPTARG" 21 | ;; 22 | p) phpBin="$OPTARG" 23 | ;; 24 | c) composerBin="$OPTARG" 25 | ;; 26 | \?) echo "option -$OPTARG not given" >&2 27 | ;; 28 | esac 29 | done 30 | 31 | # now use default values or given parameter 32 | PROJECT_ROOT="${projectRoot}" 33 | TYPO3_CONSOLE_BIN="${phpBin} ./vendor/bin/typo3" 34 | COMPOSER_BIN="${phpBin} ${composerBin}" 35 | 36 | # change to PROJECT_DIR to have the same commands here as you would use manually on command line 37 | echo -e "* switch to TYPO3 root" 38 | cd "${PROJECT_ROOT}" 39 | 40 | # if valid path to db dump is given, initialize DB with this dump 41 | if [ -f "${PROJECT_ROOT}/$dumpFile" ]; then 42 | echo -e "* resetting database, drop all tables" 43 | echo "show tables" | ${TYPO3_CONSOLE_BIN} database:import | grep -v Tables_in | grep -v "+" | awk '{print "SET FOREIGN_KEY_CHECKS = 0;drop table " $1 ";"}' | ${TYPO3_CONSOLE_BIN} database:import 44 | echo -e "* resetting database, import old DB, this can take a while (and modify for local dev machines, if needed)" 45 | ${TYPO3_CONSOLE_BIN} database:import < "${PROJECT_ROOT}/$dumpFile" 46 | else 47 | echo -e "* use existing database for this migration, since no dump file is given as parameter" 48 | fi; 49 | 50 | # we might need to fix stuff in DB for commands to run properly 51 | echo -e "* fix blocking DB issues" 52 | ${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/common/preUpgradeRun.sql" 53 | 54 | echo -e "* drop obsolete cache tables starting with cf_" 55 | echo "show tables like 'cf_%'" | ${TYPO3_CONSOLE_BIN} database:import | grep -v Tables_in | grep -v "+" | awk '{print "drop table " $1 ";"}' | ${TYPO3_CONSOLE_BIN} database:import 56 | 57 | # make sure we do the initial upgrade stuff again, no matter where current DB comes from 58 | echo -e "* re-activate initial upgrade step" 59 | ${TYPO3_CONSOLE_BIN} configuration:remove EXTCONF/helhum-typo3-console/initialUpgradeDone --force 60 | 61 | echo -e "* reduces sys_log" 62 | ${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/common/reduceSysLog.sql" 63 | 64 | echo -e "* setup all existing extensions" 65 | ${TYPO3_CONSOLE_BIN} extension:setup 66 | 67 | echo -e "* flush/warmup cache" 68 | ${TYPO3_CONSOLE_BIN} -q cache:flush 69 | ${TYPO3_CONSOLE_BIN} -q cache:warmup 70 | 71 | # you might need to fix some glitches before running the wizards 72 | echo -e "* fix some glitches before running the wizards" 73 | ${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/projectspecific/preWizard.sql" 74 | 75 | echo -e "* run upgrade wizards - core" 76 | ${TYPO3_CONSOLE_BIN} upgrade:run 77 | 78 | # in case you want to avoid needing to interact as much as possible when running the script, try this instead of the previous line 79 | #${TYPO3_CONSOLE_BIN} upgrade:run --no-interaction 80 | #echo -e "* re-run upgrade wizards which need confirmations - if you want them to run" 81 | #${TYPO3_CONSOLE_BIN} upgrade:run svgFilesSanitization 82 | 83 | echo -e "* run upgrade wizards - extensions" 84 | echo -e "** TODO: fill your extensions here" 85 | 86 | # simpler things can be done via sql - the names of the files don't matter, just make sure the path is correct 87 | echo -e "* run sql scripts for simpler migration stuff" 88 | echo -e "** run: content.sql" 89 | ${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/projectspecific/content.sql" 90 | echo -e "** run: rights.sql" 91 | ${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/projectspecific/rights.sql" 92 | echo -e "** run: template.sql" 93 | ${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/projectspecific/template.sql" 94 | #echo -e "** run: youNameIt.sql" 95 | #${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/projectspecific/youNameIt.sql" 96 | 97 | echo -e "* alter all remaining myisam -> innodb" 98 | echo "show table status where Engine='MyISAM';" | ${TYPO3_CONSOLE_BIN} database:import | awk '{print "ALTER TABLE "$1" ENGINE = InnoDB;"}' | ${TYPO3_CONSOLE_BIN} database:import 99 | 100 | echo -e "* check if a wizard is still missing - add it to the script a few lines above" 101 | ${TYPO3_CONSOLE_BIN} upgrade:list 102 | 103 | echo -e "* update language packs" 104 | ${TYPO3_CONSOLE_BIN} language:update 105 | 106 | #echo -e "* temporarily add admin user in case online users are unknown or not available locally" 107 | #${TYPO3_CONSOLE_BIN} backend:createadmin admin your..fancy..pwd 108 | 109 | echo -e "* exit for now, we suggest to run full script as soon as all extensions are ready and you start with the final iterations" 110 | exit; 111 | 112 | # activate only if you are pretty sure all is fine (after several successful migration runs) 113 | echo -e "* database:updateschema destructive" 114 | # first time rename to zzz_deleted 115 | ${TYPO3_CONSOLE_BIN} database:updateschema "destructive" 116 | # second time really drop 117 | ${TYPO3_CONSOLE_BIN} database:updateschema -v "destructive" 118 | 119 | echo -e "* and finally cleanup DB, might better be placed in bin/online/fullMigration.sh if you have that" 120 | bash ${SCRIPT_DIR}/cleanup.sh 121 | -------------------------------------------------------------------------------- /v12/sql/common/preUpgradeRun.sql: -------------------------------------------------------------------------------- 1 | -- if you have utf8mb4 now, you might get sql errors because index gets too long. The queries would be executed in the wrong order by DB compare 2 | -- alter table `sys_refindex` drop index lookup_string; 3 | -- ALTER TABLE sys_refindex CHANGE ref_string `ref_string` VARCHAR(1024) DEFAULT '' NOT NULL; 4 | -- CREATE INDEX `lookup_string` ON sys_refindex (ref_string(255)); 5 | 6 | -- there should not be pages in default language having l10n_parent set 7 | -- might break i.e. sitemap rendering 8 | update pages set l10n_parent = 0 where sys_language_uid = 0; 9 | 10 | -- remove useless duplicates to allow DB compare to create new indexes 11 | ALTER TABLE sys_category_record_mm ADD COLUMN temp SERIAL; 12 | DELETE t1 FROM sys_category_record_mm t1, sys_category_record_mm t2 13 | WHERE t1.temp > t2.temp 14 | AND t1.uid_local = t2.uid_local 15 | AND t1.uid_foreign = t2.uid_foreign 16 | AND t1.tablenames = t2.tablenames 17 | AND t1.fieldname = t2.fieldname; 18 | ALTER TABLE sys_category_record_mm DROP COLUMN temp; 19 | 20 | -- remove zombie records, which where not found by cleanup commands 21 | select concat ('sys_file_metadata zombies, will be removed: ', count(*)) FROM sys_file_metadata WHERE file not in (select uid from sys_file); 22 | delete FROM sys_file_metadata WHERE file not in (select uid from sys_file); 23 | -------------------------------------------------------------------------------- /v12/sql/common/reduceSysLog.sql: -------------------------------------------------------------------------------- 1 | delete FROM `sys_log` where details like '%has cleared the cache%'; 2 | delete FROM `sys_log` where details like '[scheduler%'; 3 | delete FROM `sys_log` where details like 'User %s logged in from%'; 4 | delete FROM `sys_log` where details like 'User %s logged out from%'; 5 | delete FROM `sys_log` where details like 'Login-attempt from%'; 6 | delete FROM `sys_log` where details like '%switched to user%'; 7 | delete FROM `sys_log` where tstamp < UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 360 DAY)); 8 | delete FROM `sys_log` where details like '%was deleted unrecoverable%'; 9 | -- errors from old version should not be relevant in new version 10 | delete FROM `sys_log` where error>0; 11 | -------------------------------------------------------------------------------- /v12/sql/projectspecific/content.sql: -------------------------------------------------------------------------------- 1 | -- fill in whatever you need for your project 2 | -------------------------------------------------------------------------------- /v12/sql/projectspecific/preWizard.sql: -------------------------------------------------------------------------------- 1 | -- fill in whatever you need for your project done before the wizards run 2 | -------------------------------------------------------------------------------- /v12/sql/projectspecific/rights.sql: -------------------------------------------------------------------------------- 1 | -- fill in whatever you need for your project 2 | 3 | -- since v12 the submodules must be checked as own modules 4 | update be_groups set groupMods = replace(groupMods, 'web_info,', 'web_info,web_info_overview,web_info_translations,'); 5 | -- update be_groups set groupMods = replace(groupMods, 'web_brofix,', 'web_brofix,web_brofix_broken_links,web_brofix_manage_exclusions,'); 6 | -------------------------------------------------------------------------------- /v12/sql/projectspecific/template.sql: -------------------------------------------------------------------------------- 1 | -- fill in whatever you need for your project 2 | -------------------------------------------------------------------------------- /v13/cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SCRIPT_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" 4 | # depending on where you have your local extensions, you need to adapt the path here 5 | # this is working out of the box, if you follow 6 | # https://docs.typo3.org/m/typo3/guide-installation/master/en-us/MigrateToComposer/BestPractices.html 7 | PROJECT_ROOT=`readlink -f "${SCRIPT_DIR}/../../../"` 8 | TYPO3_CONSOLE_BIN='./vendor/bin/typo3' 9 | 10 | echo -e "* start cleanup in ${PROJECT_ROOT}" 11 | 12 | date +"%d.%m.%Y %T" 13 | echo -e '* typo3 -q cleanup:orphanrecords' 14 | ${TYPO3_CONSOLE_BIN} -q cleanup:orphanrecords 15 | 16 | date +"%d.%m.%Y %T" 17 | echo -e '* cleanup:deletedrecords' 18 | ${TYPO3_CONSOLE_BIN} cleanup:deletedrecords 19 | 20 | date +"%d.%m.%Y %T" 21 | echo -e '* cleanup:missingrelations --update-refindex' 22 | ${TYPO3_CONSOLE_BIN} cleanup:missingrelations --update-refindex 23 | 24 | date +"%d.%m.%Y %T" 25 | echo -e '* typo3 -q cleanup:flexforms' 26 | ${TYPO3_CONSOLE_BIN} -q cleanup:flexforms 27 | 28 | echo -e '** done!!!' 29 | -------------------------------------------------------------------------------- /v13/migrate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # stop script run when error occurs 3 | set -e 4 | 5 | SCRIPT_DIR="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" 6 | # depending on where you have your local extensions, you need to adapt the path here 7 | # this is working out of the box, if you follow 8 | # https://docs.typo3.org/m/typo3/guide-installation/master/en-us/MigrateToComposer/BestPractices.html 9 | # this might be overwritten in call from upgrade scripts on online runs 10 | projectRoot=`readlink -f "${SCRIPT_DIR}/../../../"` 11 | phpBin='' 12 | composerBin='composer' 13 | 14 | 15 | # first get named parameters (only 1 character possible), see https://unix.stackexchange.com/questions/129391/passing-named-arguments-to-shell-scripts 16 | while getopts ":d:r:s:p:c:" opt; do 17 | case $opt in 18 | d) dumpFile="$OPTARG" 19 | ;; 20 | r) projectRoot="$OPTARG" 21 | ;; 22 | p) phpBin="$OPTARG" 23 | ;; 24 | c) composerBin="$OPTARG" 25 | ;; 26 | \?) echo "option -$OPTARG not given" >&2 27 | ;; 28 | esac 29 | done 30 | 31 | # now use default values or given parameter 32 | PROJECT_ROOT="${projectRoot}" 33 | TYPO3_CONSOLE_BIN="${phpBin} ./vendor/bin/typo3" 34 | COMPOSER_BIN="${phpBin} ${composerBin}" 35 | 36 | # change to PROJECT_DIR to have the same commands here as you would use manually on command line 37 | echo -e "* switch to TYPO3 root" 38 | cd "${PROJECT_ROOT}" 39 | 40 | # if valid path to db dump is given, initialize DB with this dump 41 | if [ -f "${PROJECT_ROOT}/$dumpFile" ]; then 42 | echo -e "* resetting database, drop all tables" 43 | echo "show tables" | ${TYPO3_CONSOLE_BIN} database:import | grep -v Tables_in | grep -v "+" | awk '{print "SET FOREIGN_KEY_CHECKS = 0;drop table " $1 ";"}' | ${TYPO3_CONSOLE_BIN} database:import 44 | echo -e "* resetting database, import old DB, this can take a while (and modify for local dev machines, if needed)" 45 | ${TYPO3_CONSOLE_BIN} database:import < "${PROJECT_ROOT}/$dumpFile" 46 | else 47 | echo -e "* use existing database for this migration, since no dump file is given as parameter" 48 | fi; 49 | 50 | # we might need to fix stuff in DB for commands to run properly 51 | echo -e "* fix blocking DB issues" 52 | ${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/common/preUpgradeRun.sql" 53 | 54 | echo -e "* drop obsolete cache tables starting with cf_" 55 | echo "show tables like 'cf_%'" | ${TYPO3_CONSOLE_BIN} database:import | grep -v Tables_in | grep -v "+" | awk '{print "drop table " $1 ";"}' | ${TYPO3_CONSOLE_BIN} database:import 56 | 57 | # make sure we do the initial upgrade stuff again, no matter where current DB comes from 58 | echo -e "* re-activate initial upgrade step" 59 | ${TYPO3_CONSOLE_BIN} configuration:remove EXTCONF/helhum-typo3-console/initialUpgradeDone --force 60 | 61 | echo -e "* reduces sys_log" 62 | ${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/common/reduceSysLog.sql" 63 | 64 | echo -e "* setup all existing extensions" 65 | ${TYPO3_CONSOLE_BIN} extension:setup 66 | 67 | echo -e "* flush/warmup cache" 68 | ${TYPO3_CONSOLE_BIN} -q cache:flush 69 | ${TYPO3_CONSOLE_BIN} -q cache:warmup 70 | 71 | # you might need to fix some glitches before running the wizards 72 | echo -e "* fix some glitches before running the wizards" 73 | ${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/projectspecific/preWizard.sql" 74 | 75 | echo -e "* run upgrade wizards - core" 76 | ${TYPO3_CONSOLE_BIN} upgrade:run 77 | 78 | # in case you want to avoid needing to interact as much as possible when running the script, try this instead of the previous line 79 | #${TYPO3_CONSOLE_BIN} upgrade:run --no-interaction 80 | #echo -e "* re-run upgrade wizards which need confirmations - if you want them to run" 81 | #${TYPO3_CONSOLE_BIN} upgrade:run svgFilesSanitization 82 | 83 | echo -e "* run upgrade wizards - extensions" 84 | echo -e "** TODO: fill your extensions here" 85 | 86 | # simpler things can be done via sql - the names of the files don't matter, just make sure the path is correct 87 | echo -e "* run sql scripts for simpler migration stuff" 88 | echo -e "** run: content.sql" 89 | ${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/projectspecific/content.sql" 90 | echo -e "** run: rights.sql" 91 | ${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/projectspecific/rights.sql" 92 | echo -e "** run: template.sql" 93 | ${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/projectspecific/template.sql" 94 | #echo -e "** run: youNameIt.sql" 95 | #${TYPO3_CONSOLE_BIN} database:import < "${SCRIPT_DIR}/sql/projectspecific/youNameIt.sql" 96 | 97 | echo -e "* alter all remaining myisam -> innodb" 98 | echo "show table status where Engine='MyISAM';" | ${TYPO3_CONSOLE_BIN} database:import | awk '{print "ALTER TABLE "$1" ENGINE = InnoDB;"}' | ${TYPO3_CONSOLE_BIN} database:import 99 | 100 | echo -e "* check if a wizard is still missing - add it to the script a few lines above" 101 | ${TYPO3_CONSOLE_BIN} upgrade:list 102 | 103 | echo -e "* update language packs" 104 | ${TYPO3_CONSOLE_BIN} language:update 105 | 106 | #echo -e "* temporarily add admin user in case online users are unknown or not available locally" 107 | #${TYPO3_CONSOLE_BIN} backend:createadmin admin your..fancy..pwd 108 | 109 | echo -e "* exit for now, we suggest to run full script as soon as all extensions are ready and you start with the final iterations" 110 | exit; 111 | 112 | # activate only if you are pretty sure all is fine (after several successful migration runs) 113 | echo -e "* database:updateschema destructive" 114 | # first time rename to zzz_deleted 115 | ${TYPO3_CONSOLE_BIN} database:updateschema "destructive" 116 | # second time really drop 117 | ${TYPO3_CONSOLE_BIN} database:updateschema -v "destructive" 118 | 119 | echo -e "* and finally cleanup DB, might better be placed in bin/online/fullMigration.sh if you have that" 120 | bash ${SCRIPT_DIR}/cleanup.sh 121 | -------------------------------------------------------------------------------- /v13/sql/common/preUpgradeRun.sql: -------------------------------------------------------------------------------- 1 | -- if you have utf8mb4 now, you might get sql errors because index gets too long. The queries would be executed in the wrong order by DB compare 2 | -- alter table `sys_refindex` drop index lookup_string; 3 | -- ALTER TABLE sys_refindex CHANGE ref_string `ref_string` VARCHAR(1024) DEFAULT '' NOT NULL; 4 | -- CREATE INDEX `lookup_string` ON sys_refindex (ref_string(255)); 5 | 6 | -- there should not be pages in default language having l10n_parent set 7 | -- might break i.e. sitemap rendering 8 | update pages set l10n_parent = 0 where sys_language_uid = 0; 9 | 10 | -- in case you have no reason these two fields should differ, make them identical, this avoids issues in the backend in language compare mode 11 | -- find / test with 12 | -- SELECT trans.uid,trans.pid,trans.sys_language_uid,trans.title,trans.l10n_source,trans.l10n_parent FROM pages trans inner join pages orig on trans.l10n_parent=orig.uid WHERE trans.deleted=0 and trans.hidden=0 and orig.deleted=0 and orig.hidden=0 and trans.l10n_source != trans.l10n_parent; 13 | -- update pages set l10n_source = l10n_parent where sys_language_uid > 0 and l10n_source != l10n_parent; 14 | 15 | -- remove useless duplicates to allow DB compare to create new indexes 16 | ALTER TABLE sys_category_record_mm ADD COLUMN temp SERIAL; 17 | DELETE t1 FROM sys_category_record_mm t1, sys_category_record_mm t2 18 | WHERE t1.temp > t2.temp 19 | AND t1.uid_local = t2.uid_local 20 | AND t1.uid_foreign = t2.uid_foreign 21 | AND t1.tablenames = t2.tablenames 22 | AND t1.fieldname = t2.fieldname; 23 | ALTER TABLE sys_category_record_mm DROP COLUMN temp; 24 | 25 | -- remove zombie records, which where not found by cleanup commands 26 | select concat ('sys_file_metadata zombies, will be removed: ', count(*)) FROM sys_file_metadata WHERE file not in (select uid from sys_file); 27 | delete FROM sys_file_metadata WHERE file not in (select uid from sys_file); 28 | -------------------------------------------------------------------------------- /v13/sql/common/reduceSysLog.sql: -------------------------------------------------------------------------------- 1 | delete FROM `sys_log` where details like '%has cleared the cache%'; 2 | delete FROM `sys_log` where details like '[scheduler%'; 3 | delete FROM `sys_log` where details like 'User %s logged in from%'; 4 | delete FROM `sys_log` where details like 'User %s logged out from%'; 5 | delete FROM `sys_log` where details like 'Login-attempt from%'; 6 | delete FROM `sys_log` where details like '%switched to user%'; 7 | delete FROM `sys_log` where tstamp < UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 360 DAY)); 8 | delete FROM `sys_log` where details like '%was deleted unrecoverable%'; 9 | -- errors from old version should not be relevant in new version 10 | delete FROM `sys_log` where error>0; 11 | -------------------------------------------------------------------------------- /v13/sql/projectspecific/content.sql: -------------------------------------------------------------------------------- 1 | -- fill in whatever you need for your project 2 | -------------------------------------------------------------------------------- /v13/sql/projectspecific/preWizard.sql: -------------------------------------------------------------------------------- 1 | -- fill in whatever you need for your project done before the wizards run 2 | -------------------------------------------------------------------------------- /v13/sql/projectspecific/rights.sql: -------------------------------------------------------------------------------- 1 | -- fill in whatever you need for your project 2 | 3 | -- since v12 the submodules must be checked as own modules 4 | update be_groups set groupMods = replace(groupMods, 'web_info,', 'web_info,web_info_overview,web_info_translations,'); 5 | -- update be_groups set groupMods = replace(groupMods, 'web_brofix,', 'web_brofix,web_brofix_broken_links,web_brofix_manage_exclusions,'); 6 | -------------------------------------------------------------------------------- /v13/sql/projectspecific/template.sql: -------------------------------------------------------------------------------- 1 | -- fill in whatever you need for your project 2 | --------------------------------------------------------------------------------