├── .gitignore ├── README.md ├── pom.xml └── src └── main ├── java └── org │ └── thingsboard │ ├── client │ └── tools │ │ └── migrator │ │ ├── DictionaryParser.java │ │ ├── MigratorTool.java │ │ ├── NoSqlTsPartitionDate.java │ │ ├── PgCaMigrator.java │ │ ├── RelatedEntitiesParser.java │ │ ├── WriterBuilder.java │ │ ├── WriterUtils.java │ │ ├── exception │ │ └── EntityMissingException.java │ │ └── writer │ │ ├── AbstractTbWriter.java │ │ ├── TbLatestWriter.java │ │ ├── TbTsWriter.java │ │ └── TbWriter.java │ └── server │ └── common │ └── data │ └── EntityType.java └── resources ├── logback.xml ├── schema-ts-latest.cql └── schema-ts.cql /.gitignore: -------------------------------------------------------------------------------- 1 | *.toDelete 2 | output/** 3 | *.class 4 | *~ 5 | *.iml 6 | */.idea/** 7 | .idea/** 8 | .idea 9 | *.log 10 | *.log.[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] 11 | */.classpath 12 | .classpath 13 | */.project 14 | .project 15 | .cache/** 16 | target/ 17 | build/ 18 | tmp_deb_control/ 19 | tmp_rpm_control/ 20 | tmp_sh/ 21 | .gwt/ 22 | .settings/ 23 | /bin 24 | bin/ 25 | **/dependency-reduced-pom.xml 26 | pom.xml.versionsBackup 27 | .DS_Store 28 | **/.gradle 29 | **/local.properties 30 | **/build 31 | **/target 32 | **/Californium.properties 33 | **/.env 34 | .instance_id 35 | rebuild-docker.sh 36 | */.run/** 37 | .run/** 38 | .run 39 | instance-license.data -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | 3 | **IMPORTANT! Requires Java 8 SDK** 4 | 5 | Please make sure you have JDK 8 installed: 6 | 7 | ``` 8 | ubuntu@pc:~$ java -version 9 | openjdk version "1.8.0_292" 10 | OpenJDK Runtime Environment (build 1.8.0_292-8u292-b10-0ubuntu1~20.04-b10) 11 | OpenJDK 64-Bit Server VM (build 25.292-b10, mixed mode) 12 | ``` 13 | 14 | # Description 15 | This tool used for migrating ThingsBoard setup from PostgreSQL only into Hybrid mode. 16 | 17 | You can use this tool in two scenarios. 18 | 19 | ## Scenario #1 20 | This is a scenario when rule engine requires historical data to work properly (e.g. you have rule nodes that are fetching historical data during processing new messages from devices). 21 | In this case system downtime requires for the migration. 22 | 23 | ## Scenario #2 24 | This is a scenario when rule engine DOES NOT require historical data to work properly. 25 | In this case system downtime not required (in case you are running cluster you can do this migration with zero downtime), or you'll need to restart ThingsBoard service in case of monolith setup. 26 | You'll need to upgrade configuration of ThingsBoard to use Cassandra for timeseries instead PostgreSQL - old historical data will be added to Cassandra from PostgreSQL later. 27 | In this scenario historical data will not be available after the reconfiguration of ThingsBoard (for instance on dashboards), but you'll see after migration. 28 | 29 | # Performance 30 | 31 | Performance of this tool depends on disk type and instance type (mostly on CPU resources). 32 | Here are few benchmarks in general: 33 | 1. Creating Dump of the postgres ts_kv table (if it's size is 100 GB) ~ 1-2 hours 34 | 2. Generation SSTables from dump 100 GB ~ 3-4 hours 35 | 3. 100 GB dump file will be converted into SSTable with size about ~ 20-30 GB 36 | 37 | **NOTE** Recommended instance type in AWS is m5.xlarge or larger. Avoid using instance with burst CPUs. 38 | 39 | # Tool build instructions: 40 | To build the project execute: 41 | 42 | ``` 43 | mvn clean compile assembly:single 44 | ``` 45 | 46 | It will generate single jar file with all required dependencies inside `target` dir -> `database-migrator-1.0-SNAPSHOT-jar-with-dependencies.jar`. 47 | 48 | # Prepare required files and run tool 49 | 50 | ## Scenario #1 51 | 52 | ### Cassandra setup 53 | 54 | Install Cassandra - in this scenario you can install only single instance. 55 | 56 | Using `cqlsh` create `thingsboard` keyspace and required tables. 57 | 58 | **NOTE** You can use *schema-ts.cql* and *schema-ts-latest.cql* files, that are located in main/resources folder of this project. 59 | 60 | ### Dump data from the Postgres DB to files 61 | 62 | **Do not use compression if possible because tool can only work with uncompressed files** 63 | 64 | * Stop ThingsBoard instance 65 | * Dump related tables (entities) that used to validate telemetry: 66 | ```bash 67 | pg_dump -h localhost -U postgres -d thingsboard --exclude-table=admin_settings \ 68 | --exclude-table=attribute_kv --exclude-table=audit_log --exclude-table=component_discriptor \ 69 | --exclude-table=device_credentials --exclude-table=event --exclude-table=oauth2_client_registration \ 70 | --exclude-table=oauth2_client_registration_info --exclude-table=oauth2_client_registration_template \ 71 | --exclude-table=relation --exclude-table=rule_node_state --exclude-table=tb_schema_settings \ 72 | --exclude-table=user_credentials --exclude-table=ts_kv* --exclude-table=_timescaledb_internal.* > /home/user/dump/related_entities.dmp` 73 | ``` 74 | * Dump `ts_kv_dictionary`: 75 | ```bash 76 | pg_dump -h localhost -U postgres -d thingsboard --table=ts_kv_dictionary --table=key_dictionary > /home/user/dump/ts_kv_dictionary.dmp 77 | ``` 78 | 79 | * Dump `ts_kv` and all partitions: 80 | ```bash 81 | pg_dump -h localhost -U postgres -d thingsboard --load-via-partition-root --data-only \ 82 | --table=ts_kv* --table=_timescaledb_internal.* > /home/user/dump/ts_kv_all.dmp 83 | ``` 84 | - If only part of the data needs to be migrated, it will be necessary to create a custom table in Postgres: 85 | ``` 86 | create table ts_kv_custom as ( 87 | select ts_kv.* 88 | from ts_kv 89 | join ts_kv_dictionary on ts_kv.key=ts_kv_dictionary.key_id 90 | join device on device.id=ts_kv.entity_id 91 | join device_profile on device_profile.id=device.device_profile_id 92 | where ts_kv_dictionary.key in ('pulseCounter', 'temperature', 'flow') 93 | and device_profile.name in ('default') 94 | ); 95 | ``` 96 | Such tables can be created several times using different queries. The table name must start with `ts_kv_custom`. Like `ts_kv_custom1`, `ts_kv_custom_tenant_x` 97 | 98 | To create a dump, use the command: 99 | ```bash 100 | pg_dump -h localhost -U postgres -d thingsboard --load-via-partition-root --data-only \ 101 | --table=ts_kv_custom* > /home/user/dump/ts_kv_all.dmp 102 | ``` 103 | 104 | 105 | > [Optional] Move table dumps to the instance where cassandra will be hosted 106 | 107 | ### Prepare directory structure for SSTables 108 | Tool use 3 different directories for saving SSTables - `ts_kv_cf`, `ts_kv_latest_cf`, `ts_kv_partitions_cf`. 109 | 110 | Create 3 empty directories. For example: 111 | 112 | ``` 113 | /home/user/migration/ts 114 | /home/user/migration/ts_latest 115 | /home/user/migration/ts_partition 116 | ``` 117 | 118 | ### Run tool 119 | 120 | **IMPORTANT! If you run this tool on the remote instance - don't forget to execute this command in `screen` to avoid unexpected termination** 121 | 122 | ``` 123 | java -jar ./target/database-migrator-1.0-SNAPSHOT-jar-with-dependencies.jar \ 124 | -telemetryFrom /home/user/dump/ts_kv_all.dmp \ 125 | -relatedEntities /home/user/dump/related_entities.dmp \ 126 | -dictionary /home/user/dump/ts_kv_dictionary.dmp \ 127 | -latestOut /home/user/migration/ts_latest \ 128 | -tsOut /home/user/migration/ts \ 129 | -partitionsOut /home/user/migration/ts_partition \ 130 | -castEnable false \ 131 | -partitioning MONTHS \ 132 | -linesToSkip 0 > /tmp/migration.log & 133 | ``` 134 | 135 | *If you want to migrate just `ts_kv` without `ts_kv_latest`, execute next:* 136 | 137 | ``` 138 | java -jar ./target/database-migrator-1.0-SNAPSHOT-jar-with-dependencies.jar \ 139 | -telemetryFrom /home/user/dump/ts_kv_all.dmp \ 140 | -relatedEntities /home/user/dump/related_entities.dmp \ 141 | -dictionary /home/user/dump/ts_kv_dictionary.dmp \ 142 | -tsOut /home/user/migration/ts \ 143 | -partitionsOut /home/user/migration/ts_partition \ 144 | -castEnable false \ 145 | -partitioning MONTHS \ 146 | -linesToSkip 0 > /tmp/migration.log & 147 | ``` 148 | 149 | *Use your paths for program arguments* 150 | 151 | Tool execution time depends on DB size, CPU resources and disk throughput. 152 | 153 | ### Loading SSTables into Cassandra 154 | 155 | **Note that this part works only for single node Cassandra cluster** 156 | 157 | * Stop Cassandra 158 | * Look at `/var/lib/cassandra/data/thingsboard` and check for names of data folders 159 | * Copy generated SSTable files into cassandra data dir using next command: 160 | 161 | ``` 162 | sudo find /home/user/migration/ts -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_cf-0e9aaf00ee5511e9a5fa7d6f489ffd13/ \; 163 | sudo find /home/user/migration/ts_latest -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_latest_cf-161449d0ee5511e9a5fa7d6f489ffd13/ \; 164 | sudo find /home/user/migration/ts_partition -name '*.*' -exec mv {} /var/lib/cassandra/data/thingsboard/ts_kv_partitions_cf-12e8fa80ee5511e9a5fa7d6f489ffd13/ \; 165 | ``` 166 | 167 | *Pay attention! Data folders have similar name `ts_kv_cf-0e9aaf00ee5511e9a5fa7d6f489ffd13`, but you have to use own* 168 | 169 | * Start Cassandra service and trigger compaction 170 | * Trigger compactions: `nodetool compact thingsboard` 171 | * Check compaction status: `nodetool compactionstats` 172 | 173 | * Switch ThingsBoard into Hybrid Mode 174 | 175 | Modify ThingsBoard properties file `thingsboard.conf` and add next lines: 176 | ``` 177 | export DATABASE_TS_TYPE=cassandra 178 | export TS_KV_PARTITIONING=MONTHS 179 | export CASSANDRA_URL=YOUR_CASSANDRA_URL 180 | export CASSANDRA_CLUSTER_NAME=YOUR_CASSANDRA_CLUSTER_NAME 181 | export CASSANDRA_USE_CREDENTIALS=true # false if credentials not required 182 | export CASSANDRA_USERNAME=YOUR_CASSANDRA_USERNAME 183 | export CASSANDRA_PASSWORD=YOUR_CASSANDRA_PASSWORD 184 | ``` 185 | 186 | ### Final steps 187 | 188 | Start ThingsBoard instance and verify migration. 189 | 190 | ## Scenario #2 191 | 192 | ### Cassandra setup 193 | 194 | Install Cassandra - you can install single cluster, cluster of N nodes. Cluster can be in docker or k8s or bare metal. 195 | 196 | Using `cqlsh` create `thingsboard` keyspace and required tables. 197 | 198 | **NOTE** You can use *schema-ts.cql* and *schema-ts-latest.cql* files, that are located in main/resources folder of this project. 199 | 200 | ### Switch ThingsBoard into Hybrid Mode 201 | 202 | Modify ThingsBoard properties file `thingsboard.conf` and add next lines: 203 | ``` 204 | export DATABASE_TS_TYPE=cassandra 205 | export TS_KV_PARTITIONING=MONTHS 206 | export CASSANDRA_URL=YOUR_CASSANDRA_URL 207 | export CASSANDRA_CLUSTER_NAME=YOUR_CASSANDRA_CLUSTER_NAME 208 | export CASSANDRA_USE_CREDENTIALS=true # false if credentials not required 209 | export CASSANDRA_USERNAME=YOUR_CASSANDRA_USERNAME 210 | export CASSANDRA_PASSWORD=YOUR_CASSANDRA_PASSWORD 211 | ``` 212 | 213 | Re-start ThingsBoard and verify that new timeseries data written into Cassandra. 214 | 215 | ### Dump data from the Postgres DB to files 216 | 217 | **Do not use compression if possible because tool can only work with uncompressed files** 218 | 219 | * Dump related tables (entities) that used to validate telemetry: 220 | ```bash 221 | pg_dump -h localhost -U postgres -d thingsboard --exclude-table=admin_settings \ 222 | --exclude-table=attribute_kv --exclude-table=audit_log --exclude-table=component_discriptor \ 223 | --exclude-table=device_credentials --exclude-table=event --exclude-table=oauth2_client_registration \ 224 | --exclude-table=oauth2_client_registration_info --exclude-table=oauth2_client_registration_template \ 225 | --exclude-table=relation --exclude-table=rule_node_state --exclude-table=tb_schema_settings \ 226 | --exclude-table=user_credentials --exclude-table=ts_kv* --exclude-table=_timescaledb_internal.* > /home/user/dump/related_entities.dmp` 227 | ``` 228 | * Dump `ts_kv_dictionary`: 229 | ```bash 230 | pg_dump -h localhost -U postgres -d thingsboard --table=ts_kv_dictionary --table=key_dictionary > /home/user/dump/ts_kv_dictionary.dmp 231 | ``` 232 | 233 | * Dump `ts_kv` and all partitions: 234 | ```bash 235 | pg_dump -h localhost -U postgres -d thingsboard --load-via-partition-root --data-only \ 236 | --table=ts_kv* --table=_timescaledb_internal.* > /home/user/dump/ts_kv_all.dmp 237 | ``` 238 | - If only part of the data needs to be migrated, it will be necessary to create a custom table in Postgres: 239 | ``` 240 | create table ts_kv_custom as ( 241 | select ts_kv.* 242 | from ts_kv 243 | join ts_kv_dictionary on ts_kv.key=ts_kv_dictionary.key_id 244 | join device on device.id=ts_kv.entity_id 245 | join device_profile on device_profile.id=device.device_profile_id 246 | where ts_kv_dictionary.key in ('pulseCounter', 'temperature', 'flow') 247 | and device_profile.name in ('default') 248 | ); 249 | ``` 250 | Such tables can be created several times using different queries. The table name must start with `ts_kv_custom`. Like `ts_kv_custom1`, `ts_kv_custom_tenant_x` 251 | 252 | To create a dump, use the command: 253 | ```bash 254 | pg_dump -h localhost -U postgres -d thingsboard --load-via-partition-root --data-only \ 255 | --table=ts_kv_custom* > /home/user/dump/ts_kv_all.dmp 256 | ``` 257 | 258 | 259 | > [Optional] Move table dumps to the instance where cassandra will be hosted 260 | 261 | ### Prepare directory structure for SSTables 262 | Tool use 3 different directories for saving SSTables - `ts_kv_cf`, `ts_kv_latest_cf`, `ts_kv_partitions_cf` 263 | 264 | Create 3 empty directories. 265 | 266 | **IMPORTANT** directories MUST follow pattern *.../KEYSPACE_NAME/COLUMN_FAMILY_NAME/* 267 | 268 | For example: 269 | 270 | ``` 271 | /home/user/migration/thingsboard/ts_kv_cf/ 272 | /home/user/migration/thingsboard/ts_kv_partitions_cf/ 273 | /home/user/migration/thingsboard/ts_kv_latest_cf/ 274 | ``` 275 | 276 | ### Run tool 277 | 278 | **IMPORTANT! If you run this tool on the remote instance - don't forget to execute this command in `screen` to avoid unexpected termination** 279 | 280 | ``` 281 | java -jar ./target/database-migrator-1.0-SNAPSHOT-jar-with-dependencies.jar \ 282 | -telemetryFrom /home/user/dump/ts_kv_all.dmp \ 283 | -relatedEntities /home/user/dump/related_entities.dmp \ 284 | -dictionary /home/user/dump/ts_kv_dictionary.dmp \ 285 | -latestOut /home/user/migration/thingsboard/ts_kv_latest_cf \ 286 | -tsOut /home/user/migration/thingsboard/ts_kv_cf \ 287 | -partitionsOut /home/user/migration/thingsboard/ts_kv_partitions_cf \ 288 | -castEnable false \ 289 | -partitioning MONTHS \ 290 | -linesToSkip 0 > /tmp/migration.log & 291 | ``` 292 | 293 | *If you want to migrate just `ts_kv` without `ts_kv_latest`, execute next:* 294 | 295 | ``` 296 | java -jar ./target/database-migrator-1.0-SNAPSHOT-jar-with-dependencies.jar \ 297 | -telemetryFrom /home/user/dump/ts_kv_all.dmp \ 298 | -relatedEntities /home/user/dump/related_entities.dmp \ 299 | -dictionary /home/user/dump/ts_kv_dictionary.dmp \ 300 | -tsOut /home/user/migration/thingsboard/ts_kv_cf \ 301 | -partitionsOut /home/user/migration/thingsboard/ts_kv_partitions_cf \ 302 | -castEnable false \ 303 | -partitioning MONTHS \ 304 | -linesToSkip 0 > /tmp/migration.log & 305 | ``` 306 | 307 | *Use your paths for program arguments* 308 | 309 | Tool execution time depends on DB size, CPU resources and disk throughput. 310 | 311 | ### Loading SSTables into Cassandra 312 | 313 | Using [sstableloader](https://docs.datastax.com/en/cassandra-oss/3.x/cassandra/tools/toolsBulkloader.html) start loading data into Cassandra: 314 | 315 | ``` 316 | sstableloader --verbose --nodes CASSANDRA_NODES --username cassandra --password CASSANDRA_PASSWORD /home/user/migration/thingsboard/ts_kv_partitions_cf/ 317 | 318 | sstableloader --verbose --nodes CASSANDRA_NODES --username cassandra --password CASSANDRA_PASSWORD /home/user/migration/thingsboard/ts_kv_cf/ 319 | ``` 320 | 321 | ### Final steps 322 | 323 | Verify that historical data available in ThingsBoard. 324 | 325 | # Troubleshooting 326 | 327 | ## Continue migration in case of failure on particular migration line 328 | 329 | **IMPORTANT: works only in case of Scenario #2** 330 | 331 | Tool is able to continue creation of SSTables from the particular line. 332 | Let's image that tool has stopped at particular line - XXXXXXX: 333 | ``` 334 | 2021-11-30 13:55:22,648 [main] INFO o.t.c.t.m.writer.AbstractTbWriter - Lines processed 408000000, castOk 0, castErr 0, skippedLines 68956935 335 | 2021-11-30 13:55:25,625 [main] INFO o.t.c.t.m.writer.AbstractTbWriter - Lines migrated 738000000, castOk 0, castErr 0, skippedLines 68966040 336 | 2021-11-30 13:55:32,037 [main] INFO o.t.c.t.m.writer.AbstractTbWriter - Lines processed 409000000, castOk 0, castErr 0, skippedLines 68975104 337 | 2021-11-30 13:55:34,974 [main] INFO o.t.c.t.m.writer.AbstractTbWriter - Lines migrated 739000000, castOk 0, castErr 0, skippedLines 68984191 338 | 2021-11-30 13:55:37,762 [main] INFO o.t.c.t.m.writer.AbstractTbWriter - Lines processed 410000000, castOk 0, castErr 0, skippedLines 6899318 339 | ``` 340 | 341 | You'll need to find the latest entry of `Lines migrated XXXXXX`. You can start migration tool from XXXXXX line. 342 | 343 | To use this feature please do next steps: 344 | * Migrate already created SSTable to the Cassandra by following steps in the guide below for Scenario #2 345 | *You can copy already created SSTables into some other folder, and later migrate with sstableloader folder by folder* 346 | * Create new 3 different directories for saving SSTables, or reuse the ones used in the previous step 347 | ``` 348 | /home/user/migration/thingsboard/ts_kv_cf/ 349 | /home/user/migration/thingsboard/ts_kv_partitions_cf/ 350 | /home/user/migration/thingsboard/ts_kv_latest_cf/ 351 | ``` 352 | * Start tool according to instructions above, but with an additional parameter **linesToSkip**: 353 | 354 | ``` 355 | java -jar ./target/database-migrator-1.0-SNAPSHOT-jar-with-dependencies.jar \ 356 | -telemetryFrom /home/user/dump/ts_kv_all.dmp \ 357 | -relatedEntities /home/user/dump/related_entities.dmp \ 358 | -dictionary /home/user/dump/ts_kv_dictionary.dmp \ 359 | -latestOut /home/user/migration/thingsboard/ts_kv_latest_cf \ 360 | -tsOut /home/user/migration/thingsboard/ts_kv_cf \ 361 | -partitionsOut /home/user/migration/thingsboard/ts_kv_partitions_cf \ 362 | -castEnable false \ 363 | -partitioning MONTHS \ 364 | -linesToSkip XXXXXX > /tmp/migration.log & 365 | ``` 366 | 367 | where XXXXXXX is the number of the line, that script should continue. 368 | 369 | * Continue migration according to the steps above in Scenario #2 -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.example 8 | database-migrator 9 | 1.0-SNAPSHOT 10 | 11 | jar 12 | 13 | 14 | 1.18.18 15 | 2.3.12.RELEASE 16 | 2.11.0 17 | 30.0-jre 18 | 1.7.32 19 | 1.18.18 20 | 3.11.9 21 | 3.11.0 22 | 1.8 23 | 1.8 24 | 25 | 26 | 27 | 28 | org.springframework.boot 29 | spring-boot-starter-web 30 | ${spring-boot.version} 31 | 32 | 33 | com.google.guava 34 | guava 35 | ${guava.version} 36 | 37 | 38 | org.apache.cassandra 39 | cassandra-all 40 | ${cassandra-all.version} 41 | 42 | 43 | com.datastax.cassandra 44 | cassandra-driver-core 45 | ${cassandra-driver-core.version} 46 | 47 | 48 | commons-io 49 | commons-io 50 | ${commons-io.version} 51 | 52 | 53 | org.slf4j 54 | slf4j-api 55 | ${slf4j.version} 56 | 57 | 58 | org.slf4j 59 | log4j-over-slf4j 60 | ${slf4j.version} 61 | 62 | 63 | org.projectlombok 64 | lombok 65 | ${lombok.version} 66 | 67 | 68 | 69 | 70 | 71 | 72 | org.apache.maven.plugins 73 | maven-deploy-plugin 74 | 75 | true 76 | 77 | 78 | 79 | 80 | 81 | 82 | maven-assembly-plugin 83 | 84 | 85 | 86 | org.thingsboard.client.tools.migrator.MigratorTool 87 | 88 | 89 | 90 | jar-with-dependencies 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | thingsboard 101 | https://repo.thingsboard.io/artifactory/libs-release-public 102 | 103 | 104 | -------------------------------------------------------------------------------- /src/main/java/org/thingsboard/client/tools/migrator/DictionaryParser.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL 3 | * 4 | * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. 5 | * 6 | * NOTICE: All information contained herein is, and remains 7 | * the property of ThingsBoard, Inc. and its suppliers, 8 | * if any. The intellectual and technical concepts contained 9 | * herein are proprietary to ThingsBoard, Inc. 10 | * and its suppliers and may be covered by U.S. and Foreign Patents, 11 | * patents in process, and are protected by trade secret or copyright law. 12 | * 13 | * Dissemination of this information or reproduction of this material is strictly forbidden 14 | * unless prior written permission is obtained from COMPANY. 15 | * 16 | * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, 17 | * managers or contractors who have executed Confidentiality and Non-disclosure agreements 18 | * explicitly covering such access. 19 | * 20 | * The copyright notice above does not evidence any actual or intended publication 21 | * or disclosure of this source code, which includes 22 | * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. 23 | * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, 24 | * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT 25 | * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, 26 | * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. 27 | * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION 28 | * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, 29 | * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. 30 | */ 31 | package org.thingsboard.client.tools.migrator; 32 | 33 | import lombok.extern.slf4j.Slf4j; 34 | import org.apache.commons.io.FileUtils; 35 | import org.apache.commons.io.LineIterator; 36 | import org.apache.commons.lang3.StringUtils; 37 | 38 | import java.io.File; 39 | import java.io.IOException; 40 | import java.util.HashMap; 41 | import java.util.Map; 42 | 43 | @Slf4j 44 | public class DictionaryParser { 45 | private Map dictionaryParsed = new HashMap<>(); 46 | 47 | public DictionaryParser(File sourceFile) throws IOException { 48 | parseDictionaryDump(FileUtils.lineIterator(sourceFile)); 49 | } 50 | 51 | public String getKeyByKeyId(String keyId) { 52 | return dictionaryParsed.get(keyId); 53 | } 54 | 55 | private boolean isBlockStarted(String line) { 56 | return line.startsWith("COPY public.ts_kv_dictionary (") || line.startsWith("COPY public.key_dictionary ("); 57 | } 58 | 59 | private void parseDictionaryDump(LineIterator iterator) throws IOException { 60 | try { 61 | String tempLine; 62 | while (iterator.hasNext()) { 63 | tempLine = iterator.nextLine(); 64 | 65 | if (isBlockStarted(tempLine)) { 66 | log.info("START TO MIGRATE DICTIONARY"); 67 | processBlock(iterator); 68 | } 69 | } 70 | } finally { 71 | iterator.close(); 72 | } 73 | } 74 | 75 | private void processBlock(LineIterator lineIterator) { 76 | String tempLine; 77 | String[] lineSplited; 78 | while (lineIterator.hasNext()) { 79 | tempLine = lineIterator.nextLine(); 80 | if (WriterUtils.isBlockFinished(tempLine)) { 81 | return; 82 | } 83 | 84 | lineSplited = tempLine.split("\t"); 85 | dictionaryParsed.put(lineSplited[1], lineSplited[0]); 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/org/thingsboard/client/tools/migrator/MigratorTool.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL 3 | * 4 | * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. 5 | * 6 | * NOTICE: All information contained herein is, and remains 7 | * the property of ThingsBoard, Inc. and its suppliers, 8 | * if any. The intellectual and technical concepts contained 9 | * herein are proprietary to ThingsBoard, Inc. 10 | * and its suppliers and may be covered by U.S. and Foreign Patents, 11 | * patents in process, and are protected by trade secret or copyright law. 12 | * 13 | * Dissemination of this information or reproduction of this material is strictly forbidden 14 | * unless prior written permission is obtained from COMPANY. 15 | * 16 | * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, 17 | * managers or contractors who have executed Confidentiality and Non-disclosure agreements 18 | * explicitly covering such access. 19 | * 20 | * The copyright notice above does not evidence any actual or intended publication 21 | * or disclosure of this source code, which includes 22 | * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. 23 | * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, 24 | * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT 25 | * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, 26 | * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. 27 | * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION 28 | * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, 29 | * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. 30 | */ 31 | package org.thingsboard.client.tools.migrator; 32 | 33 | import lombok.extern.slf4j.Slf4j; 34 | import org.apache.commons.cli.BasicParser; 35 | import org.apache.commons.cli.CommandLine; 36 | import org.apache.commons.cli.CommandLineParser; 37 | import org.apache.commons.cli.HelpFormatter; 38 | import org.apache.commons.cli.Option; 39 | import org.apache.commons.cli.Options; 40 | import org.apache.commons.cli.ParseException; 41 | 42 | import java.io.File; 43 | 44 | @Slf4j 45 | public class MigratorTool { 46 | 47 | public static void main(String[] args) { 48 | CommandLine cmd = parseArgs(args); 49 | 50 | try { 51 | boolean castEnable = Boolean.parseBoolean(cmd.getOptionValue("castEnable")); 52 | File allTelemetrySource = new File(cmd.getOptionValue("telemetryFrom")); 53 | File tsSaveDir = null; 54 | File partitionsSaveDir = null; 55 | File latestSaveDir = null; 56 | 57 | RelatedEntitiesParser allEntityIdsAndTypes = 58 | new RelatedEntitiesParser(new File(cmd.getOptionValue("relatedEntities"))); 59 | DictionaryParser dictionaryParser = new DictionaryParser(new File(cmd.getOptionValue("dictionary"))); 60 | 61 | if (cmd.getOptionValue("latestTelemetryOut") != null) { 62 | latestSaveDir = new File(cmd.getOptionValue("latestTelemetryOut")); 63 | } 64 | if (cmd.getOptionValue("telemetryOut") != null) { 65 | tsSaveDir = new File(cmd.getOptionValue("telemetryOut")); 66 | partitionsSaveDir = new File(cmd.getOptionValue("partitionsOut")); 67 | } 68 | String partitioning = NoSqlTsPartitionDate.MONTHS.name(); 69 | if (cmd.getOptionValue("partitioning") != null) { 70 | partitioning = cmd.getOptionValue("partitioning"); 71 | } 72 | 73 | int linesToSkip = 0; 74 | if (cmd.getOptionValue("linesToSkip") != null) { 75 | linesToSkip = Integer.parseInt(cmd.getOptionValue("linesToSkip")); 76 | } 77 | 78 | new PgCaMigrator( 79 | allTelemetrySource, 80 | tsSaveDir, 81 | partitionsSaveDir, 82 | latestSaveDir, 83 | allEntityIdsAndTypes, 84 | dictionaryParser, 85 | castEnable, 86 | partitioning).migrate(linesToSkip); 87 | 88 | } catch (Throwable th) { 89 | log.error("Failed to migrate", th); 90 | } 91 | 92 | } 93 | 94 | private static CommandLine parseArgs(String[] args) { 95 | Options options = new Options(); 96 | 97 | Option telemetryAllFrom = new Option("telemetryFrom", "telemetryFrom", true, "telemetry source file"); 98 | telemetryAllFrom.setRequired(true); 99 | options.addOption(telemetryAllFrom); 100 | 101 | Option latestTsOutOpt = new Option("latestOut", "latestTelemetryOut", true, "latest telemetry save dir"); 102 | latestTsOutOpt.setRequired(false); 103 | options.addOption(latestTsOutOpt); 104 | 105 | Option tsOutOpt = new Option("tsOut", "telemetryOut", true, "sstable save dir"); 106 | tsOutOpt.setRequired(false); 107 | options.addOption(tsOutOpt); 108 | 109 | Option partitionOutOpt = new Option("partitionsOut", "partitionsOut", true, "partitions save dir"); 110 | partitionOutOpt.setRequired(false); 111 | options.addOption(partitionOutOpt); 112 | 113 | Option castOpt = new Option("castEnable", "castEnable", true, "cast String to Double if possible"); 114 | castOpt.setRequired(true); 115 | options.addOption(castOpt); 116 | 117 | Option relatedOpt = new Option("relatedEntities", "relatedEntities", true, "related entities source file path"); 118 | relatedOpt.setRequired(true); 119 | options.addOption(relatedOpt); 120 | 121 | Option dictionaryOpt = new Option("dictionary", "dictionary", true, "dictionary source file path"); 122 | dictionaryOpt.setRequired(true); 123 | options.addOption(dictionaryOpt); 124 | 125 | Option partitioningOpt = new Option("partitioning", "partitioning", true, 126 | "Specify partitioning size for timestamp key-value storage. Example: MINUTES, HOURS, DAYS, MONTHS, INDEFINITE"); 127 | partitioningOpt.setRequired(false); 128 | options.addOption(partitioningOpt); 129 | 130 | Option linesToSkipOpt = new Option("linesToSkip", "linesToSkip", true, 131 | "Specify number of lines to skip from dump file"); 132 | linesToSkipOpt.setRequired(false); 133 | options.addOption(linesToSkipOpt); 134 | 135 | HelpFormatter formatter = new HelpFormatter(); 136 | CommandLineParser parser = new BasicParser(); 137 | 138 | try { 139 | return parser.parse(options, args); 140 | } catch (ParseException e) { 141 | log.error("Parse exception", e); 142 | formatter.printHelp("utility-name", options); 143 | 144 | System.exit(1); 145 | } 146 | return null; 147 | } 148 | 149 | } 150 | -------------------------------------------------------------------------------- /src/main/java/org/thingsboard/client/tools/migrator/NoSqlTsPartitionDate.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL 3 | * 4 | * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. 5 | * 6 | * NOTICE: All information contained herein is, and remains 7 | * the property of ThingsBoard, Inc. and its suppliers, 8 | * if any. The intellectual and technical concepts contained 9 | * herein are proprietary to ThingsBoard, Inc. 10 | * and its suppliers and may be covered by U.S. and Foreign Patents, 11 | * patents in process, and are protected by trade secret or copyright law. 12 | * 13 | * Dissemination of this information or reproduction of this material is strictly forbidden 14 | * unless prior written permission is obtained from COMPANY. 15 | * 16 | * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, 17 | * managers or contractors who have executed Confidentiality and Non-disclosure agreements 18 | * explicitly covering such access. 19 | * 20 | * The copyright notice above does not evidence any actual or intended publication 21 | * or disclosure of this source code, which includes 22 | * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. 23 | * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, 24 | * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT 25 | * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, 26 | * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. 27 | * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION 28 | * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, 29 | * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. 30 | */ 31 | package org.thingsboard.client.tools.migrator; 32 | 33 | import java.time.LocalDateTime; 34 | import java.time.ZoneOffset; 35 | import java.time.temporal.ChronoUnit; 36 | import java.time.temporal.TemporalUnit; 37 | import java.util.Optional; 38 | 39 | public enum NoSqlTsPartitionDate { 40 | 41 | MINUTES("yyyy-MM-dd-HH-mm", ChronoUnit.MINUTES), HOURS("yyyy-MM-dd-HH", ChronoUnit.HOURS), DAYS("yyyy-MM-dd", ChronoUnit.DAYS), MONTHS("yyyy-MM", ChronoUnit.MONTHS), YEARS("yyyy", ChronoUnit.YEARS),INDEFINITE("", ChronoUnit.FOREVER); 42 | 43 | private final String pattern; 44 | private final transient TemporalUnit truncateUnit; 45 | public final static LocalDateTime EPOCH_START = LocalDateTime.ofEpochSecond(0,0, ZoneOffset.UTC); 46 | 47 | NoSqlTsPartitionDate(String pattern, TemporalUnit truncateUnit) { 48 | this.pattern = pattern; 49 | this.truncateUnit = truncateUnit; 50 | } 51 | 52 | public String getPattern() { 53 | return pattern; 54 | } 55 | 56 | public TemporalUnit getTruncateUnit() { 57 | return truncateUnit; 58 | } 59 | 60 | public LocalDateTime truncatedTo(LocalDateTime time) { 61 | switch (this){ 62 | case MONTHS: 63 | return time.truncatedTo(ChronoUnit.DAYS).withDayOfMonth(1); 64 | case YEARS: 65 | return time.truncatedTo(ChronoUnit.DAYS).withDayOfYear(1); 66 | case INDEFINITE: 67 | return EPOCH_START; 68 | default: 69 | return time.truncatedTo(truncateUnit); 70 | } 71 | } 72 | 73 | public static Optional parse(String name) { 74 | NoSqlTsPartitionDate partition = null; 75 | if (name != null) { 76 | for (NoSqlTsPartitionDate partitionDate : NoSqlTsPartitionDate.values()) { 77 | if (partitionDate.name().equalsIgnoreCase(name)) { 78 | partition = partitionDate; 79 | break; 80 | } 81 | } 82 | } 83 | return Optional.ofNullable(partition); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/org/thingsboard/client/tools/migrator/PgCaMigrator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL 3 | * 4 | * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. 5 | * 6 | * NOTICE: All information contained herein is, and remains 7 | * the property of ThingsBoard, Inc. and its suppliers, 8 | * if any. The intellectual and technical concepts contained 9 | * herein are proprietary to ThingsBoard, Inc. 10 | * and its suppliers and may be covered by U.S. and Foreign Patents, 11 | * patents in process, and are protected by trade secret or copyright law. 12 | * 13 | * Dissemination of this information or reproduction of this material is strictly forbidden 14 | * unless prior written permission is obtained from COMPANY. 15 | * 16 | * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, 17 | * managers or contractors who have executed Confidentiality and Non-disclosure agreements 18 | * explicitly covering such access. 19 | * 20 | * The copyright notice above does not evidence any actual or intended publication 21 | * or disclosure of this source code, which includes 22 | * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. 23 | * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, 24 | * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT 25 | * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, 26 | * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. 27 | * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION 28 | * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, 29 | * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. 30 | */ 31 | package org.thingsboard.client.tools.migrator; 32 | 33 | import lombok.extern.slf4j.Slf4j; 34 | import org.apache.commons.io.FileUtils; 35 | import org.apache.commons.io.LineIterator; 36 | import org.thingsboard.client.tools.migrator.writer.TbLatestWriter; 37 | import org.thingsboard.client.tools.migrator.writer.TbTsWriter; 38 | import org.thingsboard.client.tools.migrator.writer.TbWriter; 39 | 40 | import java.io.File; 41 | import java.io.IOException; 42 | 43 | @Slf4j 44 | public class PgCaMigrator { 45 | 46 | private final File sourceFile; 47 | private TbWriter tbLatestWriter; 48 | private TbWriter tbTsWriter; 49 | 50 | public PgCaMigrator(File sourceFile, 51 | File ourTsDir, 52 | File outTsPartitionDir, 53 | File outTsLatestDir, 54 | RelatedEntitiesParser allEntityIdsAndTypes, 55 | DictionaryParser dictionaryParser, 56 | boolean castStringsIfPossible, 57 | String partitioning) { 58 | this.sourceFile = sourceFile; 59 | if (outTsLatestDir != null) { 60 | this.tbLatestWriter = new TbLatestWriter(dictionaryParser, allEntityIdsAndTypes, outTsLatestDir, castStringsIfPossible, partitioning); 61 | } 62 | if (ourTsDir != null) { 63 | this.tbTsWriter = new TbTsWriter(dictionaryParser, allEntityIdsAndTypes, ourTsDir, outTsPartitionDir, castStringsIfPossible, partitioning); 64 | } 65 | } 66 | 67 | public void migrate(Integer linesToSkip) throws IOException { 68 | String line; 69 | LineIterator iterator = FileUtils.lineIterator(this.sourceFile); 70 | 71 | try { 72 | while (iterator.hasNext()) { 73 | line = iterator.nextLine(); 74 | if (this.tbLatestWriter != null && isBlockLatestStarted(line)) { 75 | log.info("START TO MIGRATE LATEST"); 76 | long start = System.currentTimeMillis(); 77 | linesToSkip = tbLatestWriter.processBlock(iterator, linesToSkip); 78 | log.info("TOTAL LINES MIGRATED: {}, FORMING OF SSL FOR LATEST TS FINISHED WITH TIME: {} ms, skipped lines {}", 79 | tbLatestWriter.getLinesMigrated(), (System.currentTimeMillis() - start), tbLatestWriter.getSkippedLines()); 80 | } 81 | 82 | if (this.tbTsWriter != null && isBlockTsStarted(line)) { 83 | log.info("START TO MIGRATE TS"); 84 | long start = System.currentTimeMillis(); 85 | linesToSkip = tbTsWriter.processBlock(iterator, linesToSkip); 86 | log.info("TOTAL LINES MIGRATED: {}, FORMING OF SSL FOR TS FINISHED WITH TIME: {} ms, skipped lines {}", 87 | tbTsWriter.getLinesMigrated(), (System.currentTimeMillis() - start), tbTsWriter.getSkippedLines()); 88 | } 89 | } 90 | 91 | log.info("Finished migrate Telemetry"); 92 | 93 | } finally { 94 | iterator.close(); 95 | if (this.tbTsWriter != null) { 96 | tbTsWriter.closeWriters(); 97 | } 98 | if (this.tbLatestWriter != null) { 99 | tbLatestWriter.closeWriters(); 100 | } 101 | } 102 | } 103 | 104 | private boolean isBlockTsStarted(String line) { 105 | return line.startsWith("COPY public.ts_kv (") || line.startsWith("COPY _timescaledb_internal._hyper") || line.startsWith("COPY public.ts_kv_custom"); 106 | } 107 | 108 | private boolean isBlockLatestStarted(String line) { 109 | return line.startsWith("COPY public.ts_kv_latest ("); 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /src/main/java/org/thingsboard/client/tools/migrator/RelatedEntitiesParser.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL 3 | * 4 | * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. 5 | * 6 | * NOTICE: All information contained herein is, and remains 7 | * the property of ThingsBoard, Inc. and its suppliers, 8 | * if any. The intellectual and technical concepts contained 9 | * herein are proprietary to ThingsBoard, Inc. 10 | * and its suppliers and may be covered by U.S. and Foreign Patents, 11 | * patents in process, and are protected by trade secret or copyright law. 12 | * 13 | * Dissemination of this information or reproduction of this material is strictly forbidden 14 | * unless prior written permission is obtained from COMPANY. 15 | * 16 | * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, 17 | * managers or contractors who have executed Confidentiality and Non-disclosure agreements 18 | * explicitly covering such access. 19 | * 20 | * The copyright notice above does not evidence any actual or intended publication 21 | * or disclosure of this source code, which includes 22 | * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. 23 | * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, 24 | * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT 25 | * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, 26 | * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. 27 | * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION 28 | * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, 29 | * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. 30 | */ 31 | package org.thingsboard.client.tools.migrator; 32 | 33 | import lombok.extern.slf4j.Slf4j; 34 | import org.apache.commons.io.FileUtils; 35 | import org.apache.commons.io.LineIterator; 36 | import org.thingsboard.server.common.data.EntityType; 37 | 38 | import java.io.File; 39 | import java.io.IOException; 40 | import java.util.HashMap; 41 | import java.util.Map; 42 | 43 | @Slf4j 44 | public class RelatedEntitiesParser { 45 | private final Map allEntityIdsAndTypes = new HashMap<>(); 46 | 47 | private Map tableNameAndEntityType; 48 | 49 | public RelatedEntitiesParser(File source) throws IOException { 50 | tableNameAndEntityType = 51 | new HashMap<>(); 52 | tableNameAndEntityType.put("COPY public.alarm ", EntityType.ALARM); 53 | tableNameAndEntityType.put("COPY public.asset ", EntityType.ASSET); 54 | tableNameAndEntityType.put("COPY public.customer ", EntityType.CUSTOMER); 55 | tableNameAndEntityType.put("COPY public.dashboard ", EntityType.DASHBOARD); 56 | tableNameAndEntityType.put("COPY public.device ", EntityType.DEVICE); 57 | tableNameAndEntityType.put("COPY public.rule_chain ", EntityType.RULE_CHAIN); 58 | tableNameAndEntityType.put("COPY public.rule_node ", EntityType.RULE_NODE); 59 | tableNameAndEntityType.put("COPY public.tenant ", EntityType.TENANT); 60 | tableNameAndEntityType.put("COPY public.tb_user ", EntityType.USER); 61 | tableNameAndEntityType.put("COPY public.entity_view ", EntityType.ENTITY_VIEW); 62 | tableNameAndEntityType.put("COPY public.widgets_bundle ", EntityType.WIDGETS_BUNDLE); 63 | tableNameAndEntityType.put("COPY public.widget_type ", EntityType.WIDGET_TYPE); 64 | tableNameAndEntityType.put("COPY public.tenant_profile ", EntityType.TENANT_PROFILE); 65 | tableNameAndEntityType.put("COPY public.device_profile ", EntityType.DEVICE_PROFILE); 66 | tableNameAndEntityType.put("COPY public.api_usage_state ", EntityType.API_USAGE_STATE); 67 | 68 | // PE 69 | tableNameAndEntityType.put("COPY public.entity_group ", EntityType.ENTITY_GROUP); 70 | tableNameAndEntityType.put("COPY public.converter ", EntityType.CONVERTER); 71 | tableNameAndEntityType.put("COPY public.integration ", EntityType.INTEGRATION); 72 | tableNameAndEntityType.put("COPY public.scheduler_event ", EntityType.SCHEDULER_EVENT); 73 | tableNameAndEntityType.put("COPY public.blob_entity ", EntityType.BLOB_ENTITY); 74 | tableNameAndEntityType.put("COPY public.role ", EntityType.ROLE); 75 | tableNameAndEntityType.put("COPY public.group_permission ", EntityType.GROUP_PERMISSION); 76 | tableNameAndEntityType.put("COPY public.resource ", EntityType.TB_RESOURCE); 77 | tableNameAndEntityType.put("COPY public.ota_package ", EntityType.OTA_PACKAGE); 78 | tableNameAndEntityType.put("COPY public.edge ", EntityType.EDGE); 79 | tableNameAndEntityType.put("COPY public.rpc ", EntityType.RPC); 80 | 81 | processAllTables(FileUtils.lineIterator(source)); 82 | } 83 | 84 | public String getEntityType(String uuid) { 85 | return this.allEntityIdsAndTypes.get(uuid); 86 | } 87 | 88 | private void processAllTables(LineIterator lineIterator) throws IOException { 89 | String currentLine; 90 | try { 91 | while (lineIterator.hasNext()) { 92 | currentLine = lineIterator.nextLine(); 93 | for (Map.Entry entry : tableNameAndEntityType.entrySet()) { 94 | if (currentLine.startsWith(entry.getKey())) { 95 | int idIdx = getIdIdx(currentLine); 96 | processBlock(lineIterator, entry.getValue(), idIdx); 97 | } 98 | } 99 | } 100 | } finally { 101 | lineIterator.close(); 102 | } 103 | } 104 | 105 | private int getIdIdx(String headerLine) { 106 | log.info("Going to process next headerLine: {}", headerLine); 107 | String columns = headerLine.substring(headerLine.indexOf("(") + 1, headerLine.indexOf(")")); 108 | String[] split = columns.split(", "); 109 | int idx = 0; 110 | for (String s : split) { 111 | if ("id".equalsIgnoreCase(s)) { 112 | return idx; 113 | } 114 | idx++; 115 | } 116 | throw new RuntimeException("ID column is not present in this table :" + headerLine); 117 | } 118 | 119 | private void processBlock(LineIterator lineIterator, EntityType entityType, int idIdx) { 120 | String currentLine; 121 | while (lineIterator.hasNext()) { 122 | currentLine = lineIterator.nextLine(); 123 | if (WriterUtils.isBlockFinished(currentLine)) { 124 | return; 125 | } 126 | allEntityIdsAndTypes.put(currentLine.split("\t")[idIdx], entityType.name()); 127 | } 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /src/main/java/org/thingsboard/client/tools/migrator/WriterBuilder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL 3 | * 4 | * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. 5 | * 6 | * NOTICE: All information contained herein is, and remains 7 | * the property of ThingsBoard, Inc. and its suppliers, 8 | * if any. The intellectual and technical concepts contained 9 | * herein are proprietary to ThingsBoard, Inc. 10 | * and its suppliers and may be covered by U.S. and Foreign Patents, 11 | * patents in process, and are protected by trade secret or copyright law. 12 | * 13 | * Dissemination of this information or reproduction of this material is strictly forbidden 14 | * unless prior written permission is obtained from COMPANY. 15 | * 16 | * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, 17 | * managers or contractors who have executed Confidentiality and Non-disclosure agreements 18 | * explicitly covering such access. 19 | * 20 | * The copyright notice above does not evidence any actual or intended publication 21 | * or disclosure of this source code, which includes 22 | * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. 23 | * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, 24 | * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT 25 | * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, 26 | * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. 27 | * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION 28 | * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, 29 | * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. 30 | */ 31 | package org.thingsboard.client.tools.migrator; 32 | 33 | import org.apache.cassandra.io.sstable.CQLSSTableWriter; 34 | 35 | import java.io.File; 36 | 37 | public class WriterBuilder { 38 | 39 | private static final String tsSchema = "CREATE TABLE thingsboard.ts_kv_cf (\n" + 40 | " entity_type text, // (DEVICE, CUSTOMER, TENANT)\n" + 41 | " entity_id timeuuid,\n" + 42 | " key text,\n" + 43 | " partition bigint,\n" + 44 | " ts bigint,\n" + 45 | " bool_v boolean,\n" + 46 | " str_v text,\n" + 47 | " long_v bigint,\n" + 48 | " dbl_v double,\n" + 49 | " json_v text,\n" + 50 | " PRIMARY KEY (( entity_type, entity_id, key, partition ), ts)\n" + 51 | ");"; 52 | 53 | private static final String latestSchema = "CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_latest_cf (\n" + 54 | " entity_type text, // (DEVICE, CUSTOMER, TENANT)\n" + 55 | " entity_id timeuuid,\n" + 56 | " key text,\n" + 57 | " ts bigint,\n" + 58 | " bool_v boolean,\n" + 59 | " str_v text,\n" + 60 | " long_v bigint,\n" + 61 | " dbl_v double,\n" + 62 | " json_v text,\n" + 63 | " PRIMARY KEY (( entity_type, entity_id ), key)\n" + 64 | ") WITH compaction = { 'class' : 'LeveledCompactionStrategy' };"; 65 | 66 | private static final String partitionSchema = "CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_partitions_cf (\n" + 67 | " entity_type text, // (DEVICE, CUSTOMER, TENANT)\n" + 68 | " entity_id timeuuid,\n" + 69 | " key text,\n" + 70 | " partition bigint,\n" + 71 | " PRIMARY KEY (( entity_type, entity_id, key ), partition)\n" + 72 | ") WITH CLUSTERING ORDER BY ( partition ASC )\n" + 73 | " AND compaction = { 'class' : 'LeveledCompactionStrategy' };"; 74 | 75 | public static CQLSSTableWriter getTsWriter(File dir) { 76 | return CQLSSTableWriter.builder() 77 | .inDirectory(dir) 78 | .forTable(tsSchema) 79 | .using("INSERT INTO thingsboard.ts_kv_cf (entity_type, entity_id, key, partition, ts, bool_v, str_v, long_v, dbl_v, json_v) " + 80 | "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") 81 | .build(); 82 | } 83 | 84 | public static CQLSSTableWriter getLatestWriter(File dir) { 85 | return CQLSSTableWriter.builder() 86 | .inDirectory(dir) 87 | .forTable(latestSchema) 88 | .using("INSERT INTO thingsboard.ts_kv_latest_cf (entity_type, entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) " + 89 | "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)") 90 | .build(); 91 | } 92 | 93 | public static CQLSSTableWriter getPartitionWriter(File dir) { 94 | return CQLSSTableWriter.builder() 95 | .inDirectory(dir) 96 | .forTable(partitionSchema) 97 | .using("INSERT INTO thingsboard.ts_kv_partitions_cf (entity_type, entity_id, key, partition) " + 98 | "VALUES (?, ?, ?, ?)") 99 | .build(); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/org/thingsboard/client/tools/migrator/WriterUtils.java: -------------------------------------------------------------------------------- 1 | package org.thingsboard.client.tools.migrator; 2 | 3 | import org.apache.commons.lang3.StringUtils; 4 | 5 | public class WriterUtils { 6 | 7 | private WriterUtils() {} 8 | 9 | public static boolean isBlockFinished(String line) { 10 | return StringUtils.isBlank(line) || line.equals("\\."); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/org/thingsboard/client/tools/migrator/exception/EntityMissingException.java: -------------------------------------------------------------------------------- 1 | package org.thingsboard.client.tools.migrator.exception; 2 | 3 | 4 | public class EntityMissingException extends RuntimeException { 5 | public EntityMissingException(String msg) { 6 | super(msg); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/org/thingsboard/client/tools/migrator/writer/AbstractTbWriter.java: -------------------------------------------------------------------------------- 1 | package org.thingsboard.client.tools.migrator.writer; 2 | 3 | import com.google.common.collect.Lists; 4 | import lombok.extern.slf4j.Slf4j; 5 | import org.apache.cassandra.io.sstable.CQLSSTableWriter; 6 | import org.apache.commons.io.LineIterator; 7 | import org.apache.commons.lang3.math.NumberUtils; 8 | import org.thingsboard.client.tools.migrator.DictionaryParser; 9 | import org.thingsboard.client.tools.migrator.NoSqlTsPartitionDate; 10 | import org.thingsboard.client.tools.migrator.RelatedEntitiesParser; 11 | import org.thingsboard.client.tools.migrator.WriterUtils; 12 | import org.thingsboard.client.tools.migrator.exception.EntityMissingException; 13 | 14 | import java.io.File; 15 | import java.io.IOException; 16 | import java.time.Instant; 17 | import java.time.LocalDateTime; 18 | import java.time.ZoneOffset; 19 | import java.time.temporal.ChronoUnit; 20 | import java.util.Arrays; 21 | import java.util.Date; 22 | import java.util.HashSet; 23 | import java.util.List; 24 | import java.util.Optional; 25 | import java.util.Set; 26 | import java.util.UUID; 27 | import java.util.stream.Collectors; 28 | 29 | @Slf4j 30 | public abstract class AbstractTbWriter implements TbWriter { 31 | 32 | private static final long LOG_BATCH = 1000000; 33 | private static final long ROW_PER_FILE = 10000000; 34 | 35 | private NoSqlTsPartitionDate tsFormat; 36 | 37 | private long castErrors = 0; 38 | private long castedOk = 0; 39 | 40 | protected long currentWriterCount = 1; 41 | 42 | protected final Set partitions = new HashSet<>(); 43 | 44 | protected long linesMigrated = 0; 45 | 46 | private long skippedLines = 0; 47 | 48 | private final DictionaryParser keyParser; 49 | private final RelatedEntitiesParser entityIdsAndTypes; 50 | protected CQLSSTableWriter currentWriter; 51 | protected final File outDir; 52 | 53 | private final boolean castStringIfPossible; 54 | 55 | public AbstractTbWriter(DictionaryParser keyParser, RelatedEntitiesParser entityIdsAndTypes, File outDir, 56 | boolean castStringIfPossible, String partitioning) { 57 | this.keyParser = keyParser; 58 | this.entityIdsAndTypes = entityIdsAndTypes; 59 | this.currentWriter = getWriter(outDir); 60 | this.outDir = outDir; 61 | this.castStringIfPossible = castStringIfPossible; 62 | 63 | Optional partition = NoSqlTsPartitionDate.parse(partitioning); 64 | if (partition.isPresent()) { 65 | tsFormat = partition.get(); 66 | } else { 67 | log.warn("Incorrect configuration of partitioning {}", partitioning); 68 | throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!"); 69 | } 70 | } 71 | 72 | public abstract List toValues(List raw); 73 | 74 | public abstract void reOpenWriter() throws IOException; 75 | 76 | public abstract CQLSSTableWriter getWriter(File outDir); 77 | 78 | @Override 79 | public long getLinesMigrated() { 80 | return linesMigrated; 81 | } 82 | 83 | @Override 84 | public long getSkippedLines() { 85 | return skippedLines; 86 | } 87 | 88 | @Override 89 | public void closeWriters() throws IOException { 90 | if (currentWriter != null) { 91 | currentWriter.close(); 92 | } 93 | } 94 | 95 | @Override 96 | public int processBlock(LineIterator iterator, int linesToSkip) { 97 | String currentLine; 98 | long linesProcessed = 0; 99 | while (iterator.hasNext()) { 100 | logLinesProcessed(linesProcessed++); 101 | currentLine = iterator.nextLine(); 102 | 103 | if (WriterUtils.isBlockFinished(currentLine)) { 104 | 105 | try { 106 | writePartitions(); 107 | } catch (IOException e) { 108 | throw new RuntimeException(e); 109 | } 110 | return linesToSkip; 111 | } 112 | 113 | if (linesToSkip > 0) { 114 | linesToSkip = linesToSkip - 1; 115 | continue; 116 | } 117 | 118 | List values = null; 119 | try { 120 | List raw = Arrays.stream(currentLine.trim().split("\t")) 121 | .map(String::trim) 122 | .collect(Collectors.toList()); 123 | try { 124 | values = toValues(raw); 125 | } catch (EntityMissingException e) { 126 | log.debug("Failed to process line [{}}, cause {}, skipping it ", currentLine, e.getMessage()); 127 | } 128 | 129 | if (values != null) { 130 | if (this.currentWriterCount == 0) { 131 | reOpenWriter(); 132 | log.info("Reopening writer"); 133 | 134 | writePartitions(); 135 | } 136 | 137 | if (this.castStringIfPossible) { 138 | currentWriter.addRow(castToNumericIfPossible(values)); 139 | } else { 140 | currentWriter.addRow(values); 141 | } 142 | 143 | currentWriterCount++; 144 | if (currentWriterCount >= ROW_PER_FILE) { 145 | currentWriterCount = 0; 146 | } 147 | } else { 148 | skippedLines++; 149 | } 150 | } catch (Exception ex) { 151 | String strValues = ""; 152 | if (values != null) { 153 | strValues = values.toString(); 154 | } 155 | log.error("Failed to process line [" + currentLine + "], skipping it , values = " + strValues + "", ex); 156 | } 157 | } 158 | try { 159 | writePartitions(); 160 | } catch (IOException e) { 161 | throw new RuntimeException(e); 162 | } 163 | return linesToSkip; 164 | } 165 | 166 | private void logLinesProcessed(long lines) { 167 | if (lines > 0 && lines % LOG_BATCH == 0) { 168 | log.info("Lines processed {}, castOk {}, castErr {}, skippedLines {}", 169 | lines, castedOk, castErrors, skippedLines); 170 | } 171 | } 172 | 173 | 174 | private List castToNumericIfPossible(List values) { 175 | try { 176 | if (values.get(6) != null && NumberUtils.isNumber(values.get(6).toString())) { 177 | Double casted = NumberUtils.createDouble(values.get(6).toString()); 178 | List numeric = Lists.newArrayList(); 179 | numeric.addAll(values); 180 | numeric.set(6, null); 181 | numeric.set(8, casted); 182 | castedOk++; 183 | return numeric; 184 | } 185 | } catch (Throwable th) { 186 | castErrors++; 187 | } 188 | 189 | processPartitions(values); 190 | 191 | return values; 192 | } 193 | 194 | protected void processPartitions(List values) { 195 | String key = values.get(0) + "|" + values.get(1) + "|" + values.get(2) + "|" + values.get(3); 196 | partitions.add(key); 197 | } 198 | 199 | protected void logLinesMigrated(long lines) { 200 | if (lines > 0 && lines % LOG_BATCH == 0) { 201 | log.info("Lines migrated {}, castOk {}, castErr {}, skippedLines {}", 202 | lines, castedOk, castErrors, skippedLines); 203 | } 204 | } 205 | 206 | protected void addTypeIdKey(List result, List raw) { 207 | String entityType = entityIdsAndTypes.getEntityType(raw.get(0)); 208 | if (entityType == null) { 209 | String errorMsg = String.format("Can't find entity type for ID [%s], most probably entity was removed from the system", 210 | raw.get(0)); 211 | throw new EntityMissingException(errorMsg); 212 | } 213 | result.add(entityType); 214 | result.add(UUID.fromString(raw.get(0))); 215 | result.add(keyParser.getKeyByKeyId(raw.get(1))); 216 | } 217 | 218 | protected void addPartitions(List result, List raw) { 219 | long ts = Long.parseLong(raw.get(2)); 220 | long partition = toPartitionTs(ts); 221 | result.add(partition); 222 | result.add(ts); 223 | } 224 | 225 | protected void addTimeseries(List result, List raw) { 226 | result.add(Long.parseLong(raw.get(2))); 227 | } 228 | 229 | protected void addValues(List result, List raw) { 230 | result.add(raw.get(3).equals("\\N") ? null : raw.get(3).equals("t") ? Boolean.TRUE : Boolean.FALSE); 231 | result.add(raw.get(4).equals("\\N") ? null : raw.get(4)); 232 | result.add(raw.get(5).equals("\\N") ? null : Long.parseLong(raw.get(5))); 233 | result.add(raw.get(6).equals("\\N") ? null : Double.parseDouble(raw.get(6))); 234 | result.add(raw.get(7).equals("\\N") ? null : raw.get(7)); 235 | } 236 | 237 | private long toPartitionTs(long ts) { 238 | LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC); 239 | return tsFormat.truncatedTo(time).toInstant(ZoneOffset.UTC).toEpochMilli(); 240 | } 241 | 242 | } 243 | -------------------------------------------------------------------------------- /src/main/java/org/thingsboard/client/tools/migrator/writer/TbLatestWriter.java: -------------------------------------------------------------------------------- 1 | package org.thingsboard.client.tools.migrator.writer; 2 | 3 | import org.apache.cassandra.io.sstable.CQLSSTableWriter; 4 | import org.thingsboard.client.tools.migrator.DictionaryParser; 5 | import org.thingsboard.client.tools.migrator.RelatedEntitiesParser; 6 | import org.thingsboard.client.tools.migrator.WriterBuilder; 7 | 8 | import java.io.File; 9 | import java.io.IOException; 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | 13 | public class TbLatestWriter extends AbstractTbWriter { 14 | 15 | public TbLatestWriter(DictionaryParser keyParser, RelatedEntitiesParser entityIdsAndTypes, File outDir, 16 | boolean castStringsIfPossible, String partitioning) { 17 | super(keyParser, entityIdsAndTypes, outDir, castStringsIfPossible, partitioning); 18 | } 19 | 20 | @Override 21 | public CQLSSTableWriter getWriter(File outDir) { 22 | return WriterBuilder.getLatestWriter(outDir); 23 | } 24 | 25 | @Override 26 | public List toValues(List raw) { 27 | List result = new ArrayList<>(); 28 | 29 | addTypeIdKey(result, raw); 30 | addTimeseries(result, raw); 31 | addValues(result, raw); 32 | 33 | logLinesMigrated(linesMigrated++); 34 | 35 | return result; 36 | } 37 | 38 | @Override 39 | public void reOpenWriter() throws IOException { 40 | currentWriter.close(); 41 | currentWriter = WriterBuilder.getLatestWriter(outDir); 42 | } 43 | 44 | @Override 45 | public void writePartitions() { 46 | // nothing todo 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/org/thingsboard/client/tools/migrator/writer/TbTsWriter.java: -------------------------------------------------------------------------------- 1 | package org.thingsboard.client.tools.migrator.writer; 2 | 3 | import com.google.common.collect.Lists; 4 | import lombok.extern.slf4j.Slf4j; 5 | import org.apache.cassandra.io.sstable.CQLSSTableWriter; 6 | import org.thingsboard.client.tools.migrator.DictionaryParser; 7 | import org.thingsboard.client.tools.migrator.RelatedEntitiesParser; 8 | import org.thingsboard.client.tools.migrator.WriterBuilder; 9 | 10 | import java.io.File; 11 | import java.io.IOException; 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | import java.util.UUID; 15 | 16 | @Slf4j 17 | public class TbTsWriter extends AbstractTbWriter { 18 | 19 | private final File outTsPartitionDir; 20 | 21 | public TbTsWriter(DictionaryParser keyParser, RelatedEntitiesParser entityIdsAndTypes, File outDir, 22 | File outTsPartitionDir, boolean castStringsIfPossible, String partitioning) { 23 | super(keyParser, entityIdsAndTypes, outDir, castStringsIfPossible, partitioning); 24 | this.outTsPartitionDir = outTsPartitionDir; 25 | } 26 | 27 | @Override 28 | public void closeWriters() throws IOException { 29 | super.closeWriters(); 30 | } 31 | 32 | @Override 33 | public List toValues(List raw) { 34 | List result = new ArrayList<>(); 35 | 36 | addTypeIdKey(result, raw); 37 | addPartitions(result, raw); 38 | addValues(result, raw); 39 | processPartitions(result); 40 | 41 | logLinesMigrated(linesMigrated++); 42 | 43 | return result; 44 | } 45 | 46 | @Override 47 | public void reOpenWriter() throws IOException { 48 | currentWriter.close(); 49 | currentWriter = WriterBuilder.getTsWriter(outDir); 50 | } 51 | 52 | @Override 53 | public CQLSSTableWriter getWriter(File outDir) { 54 | return WriterBuilder.getTsWriter(outDir); 55 | } 56 | 57 | @Override 58 | public void writePartitions() throws IOException { 59 | CQLSSTableWriter currentPartitionsWriter = null; 60 | try { 61 | currentPartitionsWriter = WriterBuilder.getPartitionWriter(outTsPartitionDir); 62 | log.info("Partitions collected " + partitions.size()); 63 | long startTs = System.currentTimeMillis(); 64 | for (String partition : partitions) { 65 | String[] split = partition.split("\\|"); 66 | List values = Lists.newArrayList(); 67 | values.add(split[0]); 68 | values.add(UUID.fromString(split[1])); 69 | values.add(split[2]); 70 | values.add(Long.parseLong(split[3])); 71 | currentPartitionsWriter.addRow(values); 72 | } 73 | 74 | log.info(" Migrated partitions " + partitions.size() + " in " + (System.currentTimeMillis() - startTs)); 75 | 76 | partitions.clear(); 77 | } finally { 78 | if (currentPartitionsWriter != null) { 79 | currentPartitionsWriter.close(); 80 | } 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/org/thingsboard/client/tools/migrator/writer/TbWriter.java: -------------------------------------------------------------------------------- 1 | package org.thingsboard.client.tools.migrator.writer; 2 | 3 | import org.apache.commons.io.LineIterator; 4 | 5 | import java.io.IOException; 6 | 7 | public interface TbWriter { 8 | 9 | int processBlock(LineIterator iterator, int linesToSkip); 10 | 11 | void writePartitions() throws IOException; 12 | 13 | void closeWriters() throws IOException; 14 | 15 | long getLinesMigrated(); 16 | 17 | long getSkippedLines(); 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/org/thingsboard/server/common/data/EntityType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL 3 | * 4 | * Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. 5 | * 6 | * NOTICE: All information contained herein is, and remains 7 | * the property of ThingsBoard, Inc. and its suppliers, 8 | * if any. The intellectual and technical concepts contained 9 | * herein are proprietary to ThingsBoard, Inc. 10 | * and its suppliers and may be covered by U.S. and Foreign Patents, 11 | * patents in process, and are protected by trade secret or copyright law. 12 | * 13 | * Dissemination of this information or reproduction of this material is strictly forbidden 14 | * unless prior written permission is obtained from COMPANY. 15 | * 16 | * Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, 17 | * managers or contractors who have executed Confidentiality and Non-disclosure agreements 18 | * explicitly covering such access. 19 | * 20 | * The copyright notice above does not evidence any actual or intended publication 21 | * or disclosure of this source code, which includes 22 | * information that is confidential and/or proprietary, and is a trade secret, of COMPANY. 23 | * ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, 24 | * OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT 25 | * THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, 26 | * AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. 27 | * THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION 28 | * DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, 29 | * OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. 30 | */ 31 | package org.thingsboard.server.common.data; 32 | 33 | /** 34 | * @author Andrew Shvayka 35 | */ 36 | public enum EntityType { 37 | TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, ENTITY_GROUP, 38 | CONVERTER, INTEGRATION, RULE_CHAIN, RULE_NODE, SCHEDULER_EVENT, BLOB_ENTITY, 39 | ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, ROLE, GROUP_PERMISSION, TENANT_PROFILE, 40 | DEVICE_PROFILE, API_USAGE_STATE, TB_RESOURCE, OTA_PACKAGE, EDGE, RPC; 41 | } 42 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 34 | 35 | 36 | 37 | 38 | 39 | %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /src/main/resources/schema-ts-latest.cql: -------------------------------------------------------------------------------- 1 | -- 2 | -- ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL 3 | -- 4 | -- Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. 5 | -- 6 | -- NOTICE: All information contained herein is, and remains 7 | -- the property of ThingsBoard, Inc. and its suppliers, 8 | -- if any. The intellectual and technical concepts contained 9 | -- herein are proprietary to ThingsBoard, Inc. 10 | -- and its suppliers and may be covered by U.S. and Foreign Patents, 11 | -- patents in process, and are protected by trade secret or copyright law. 12 | -- 13 | -- Dissemination of this information or reproduction of this material is strictly forbidden 14 | -- unless prior written permission is obtained from COMPANY. 15 | -- 16 | -- Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, 17 | -- managers or contractors who have executed Confidentiality and Non-disclosure agreements 18 | -- explicitly covering such access. 19 | -- 20 | -- The copyright notice above does not evidence any actual or intended publication 21 | -- or disclosure of this source code, which includes 22 | -- information that is confidential and/or proprietary, and is a trade secret, of COMPANY. 23 | -- ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, 24 | -- OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT 25 | -- THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, 26 | -- AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. 27 | -- THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION 28 | -- DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, 29 | -- OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. 30 | -- 31 | 32 | CREATE KEYSPACE IF NOT EXISTS thingsboard 33 | WITH replication = { 34 | 'class' : 'SimpleStrategy', 35 | 'replication_factor' : 1 36 | }; 37 | 38 | CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_latest_cf ( 39 | entity_type text, // (DEVICE, CUSTOMER, TENANT) 40 | entity_id timeuuid, 41 | key text, 42 | ts bigint, 43 | bool_v boolean, 44 | str_v text, 45 | long_v bigint, 46 | dbl_v double, 47 | json_v text, 48 | PRIMARY KEY (( entity_type, entity_id ), key) 49 | ) WITH compaction = { 'class' : 'LeveledCompactionStrategy' }; 50 | -------------------------------------------------------------------------------- /src/main/resources/schema-ts.cql: -------------------------------------------------------------------------------- 1 | -- 2 | -- ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL 3 | -- 4 | -- Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved. 5 | -- 6 | -- NOTICE: All information contained herein is, and remains 7 | -- the property of ThingsBoard, Inc. and its suppliers, 8 | -- if any. The intellectual and technical concepts contained 9 | -- herein are proprietary to ThingsBoard, Inc. 10 | -- and its suppliers and may be covered by U.S. and Foreign Patents, 11 | -- patents in process, and are protected by trade secret or copyright law. 12 | -- 13 | -- Dissemination of this information or reproduction of this material is strictly forbidden 14 | -- unless prior written permission is obtained from COMPANY. 15 | -- 16 | -- Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees, 17 | -- managers or contractors who have executed Confidentiality and Non-disclosure agreements 18 | -- explicitly covering such access. 19 | -- 20 | -- The copyright notice above does not evidence any actual or intended publication 21 | -- or disclosure of this source code, which includes 22 | -- information that is confidential and/or proprietary, and is a trade secret, of COMPANY. 23 | -- ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC PERFORMANCE, 24 | -- OR PUBLIC DISPLAY OF OR THROUGH USE OF THIS SOURCE CODE WITHOUT 25 | -- THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED, 26 | -- AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES. 27 | -- THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION 28 | -- DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS, 29 | -- OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT MAY DESCRIBE, IN WHOLE OR IN PART. 30 | -- 31 | 32 | CREATE KEYSPACE IF NOT EXISTS thingsboard 33 | WITH replication = { 34 | 'class' : 'SimpleStrategy', 35 | 'replication_factor' : 1 36 | }; 37 | 38 | CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_cf ( 39 | entity_type text, // (DEVICE, CUSTOMER, TENANT) 40 | entity_id timeuuid, 41 | key text, 42 | partition bigint, 43 | ts bigint, 44 | bool_v boolean, 45 | str_v text, 46 | long_v bigint, 47 | dbl_v double, 48 | json_v text, 49 | PRIMARY KEY (( entity_type, entity_id, key, partition ), ts) 50 | ); 51 | 52 | CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_partitions_cf ( 53 | entity_type text, // (DEVICE, CUSTOMER, TENANT) 54 | entity_id timeuuid, 55 | key text, 56 | partition bigint, 57 | PRIMARY KEY (( entity_type, entity_id, key ), partition) 58 | ) WITH CLUSTERING ORDER BY ( partition ASC ) 59 | AND compaction = { 'class' : 'LeveledCompactionStrategy' }; 60 | --------------------------------------------------------------------------------