├── images ├── env.png ├── oss2.png ├── pip.png ├── pysmb.png ├── pipEnv.png ├── UpdateVTPK.png ├── UpdateVTS.png ├── paramiko.png ├── UpdateVTOSS.png ├── CreateAdvVTPK.png ├── CreatePartVTPK.png ├── 20180720_ChangeLog1_OptimaizedErrorHandling.png └── 20180720_ChangeLog2_AddLODSpecifiedBoxandErrorHandling.png ├── python ├── advancedVectorTilePackage.tbx ├── updateVectorTileforOSS.py ├── updateVectorTilePackage.py ├── createAdvancedVectorTilePacakge.py ├── createPartVectorTilePackage.py └── updateVectorTileService.py ├── .idea ├── misc.xml ├── vcs.xml ├── modules.xml ├── PartiallyUpateArcGISVectorTilesTools.iml └── workspace.xml ├── LICENSE ├── .gitignore └── README.md /images/env.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makeling/PartiallyUpateArcGISVectorTilesTools/HEAD/images/env.png -------------------------------------------------------------------------------- /images/oss2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makeling/PartiallyUpateArcGISVectorTilesTools/HEAD/images/oss2.png -------------------------------------------------------------------------------- /images/pip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makeling/PartiallyUpateArcGISVectorTilesTools/HEAD/images/pip.png -------------------------------------------------------------------------------- /images/pysmb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makeling/PartiallyUpateArcGISVectorTilesTools/HEAD/images/pysmb.png -------------------------------------------------------------------------------- /images/pipEnv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makeling/PartiallyUpateArcGISVectorTilesTools/HEAD/images/pipEnv.png -------------------------------------------------------------------------------- /images/UpdateVTPK.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makeling/PartiallyUpateArcGISVectorTilesTools/HEAD/images/UpdateVTPK.png -------------------------------------------------------------------------------- /images/UpdateVTS.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makeling/PartiallyUpateArcGISVectorTilesTools/HEAD/images/UpdateVTS.png -------------------------------------------------------------------------------- /images/paramiko.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makeling/PartiallyUpateArcGISVectorTilesTools/HEAD/images/paramiko.png -------------------------------------------------------------------------------- /images/UpdateVTOSS.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makeling/PartiallyUpateArcGISVectorTilesTools/HEAD/images/UpdateVTOSS.png -------------------------------------------------------------------------------- /images/CreateAdvVTPK.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makeling/PartiallyUpateArcGISVectorTilesTools/HEAD/images/CreateAdvVTPK.png -------------------------------------------------------------------------------- /images/CreatePartVTPK.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makeling/PartiallyUpateArcGISVectorTilesTools/HEAD/images/CreatePartVTPK.png -------------------------------------------------------------------------------- /python/advancedVectorTilePackage.tbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makeling/PartiallyUpateArcGISVectorTilesTools/HEAD/python/advancedVectorTilePackage.tbx -------------------------------------------------------------------------------- /images/20180720_ChangeLog1_OptimaizedErrorHandling.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makeling/PartiallyUpateArcGISVectorTilesTools/HEAD/images/20180720_ChangeLog1_OptimaizedErrorHandling.png -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /images/20180720_ChangeLog2_AddLODSpecifiedBoxandErrorHandling.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/makeling/PartiallyUpateArcGISVectorTilesTools/HEAD/images/20180720_ChangeLog2_AddLODSpecifiedBoxandErrorHandling.png -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/PartiallyUpateArcGISVectorTilesTools.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 makeling 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | .idea/workspace.xml 103 | .idea/workspace.xml 104 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Read Me 2 | 3 | ## Installation Guild 4 | 5 | - Requirement ： ArcGIS Pro 2.0 or later; Pysmb; oss2; paramiko 6 | 7 | - Pip installation required. more information: https://pip.pypa.io/en/stable/installing/ 8 | 9 | - Optional: Add python and pip path to Windows Environments. 10 | 11 | - PYSMB, OSS2, PARAMIKO packages required. we strongly recommend the packages installed via pip. 12 | 13 | `pip install pysmb` 14 | 15 | `pip install oss2` 16 | 17 | `pip install paramiko` 18 | 19 | ​ 20 | 21 | ## Introduction 22 | 23 | **Partially Update ArcGIS Vector Tiles Toolbox** is based on ArcGIS Pro (version 2.0 or later), which is desktop end program that supports creating Vector Tile Package. This toolbox enables you to update vector tile package or services partially when the source data is in a pretty large amount. 24 | 25 | The toolbox includes 5 tools as below: 26 | 27 | ### Create Advanced Vector Tile Package 28 | 29 | This tool enables you to create an advanced vector tile package, which contains the Tiling Schema xml file and Tiling Indexed Polygon feature class. In the way , the schema is packaged with the tiles , so to ensure consistency for updating tiles partially. What's more, you can define Maximum Vertex Count to effect the levels of the tiles. 30 | 31 | ### Create Part Vector Tile Package 32 | 33 | This tool enables you to generate a partial vector tile package in the specified area of of the whole map extent. It requires that the Original Vector Tile Package created by the Create Advanced Vector Tile Package tool. The VTPK generated by the standard Create Vector Tile Package tool in ArcGIS Pro is not support for this tool. 34 | 35 | ### Update Vector Tile Package 36 | 37 | This tool enables you to update a vector tile package using the partial Vector Tile Package. 38 | 39 | ### Update Vector Tile Service 40 | 41 | This tool enables you to update a vector tile service using the partial Vector Tile Package. It doesn't require the tile layer service restarted, and the service can runs without interruption during the update . 42 | 43 | The ArcGIS Server Manager account and passwords, and tile service URL are required. 44 | 45 | ### Update Vector Tile for OSS 46 | 47 | This tool enables you to update a vector tile service on OSS using the partial Vector Tile Package. It doesn't require the tile layer service restarted, and the service can runs without interruption during the update . 48 | 49 | The Service Name, Access Key ID, Access Key Secret, Share Hostname, Bucket Name, Endpoint are required. 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | -------------------------------------------------------------------------------- /python/updateVectorTileforOSS.py: -------------------------------------------------------------------------------- 1 | # _*_ coding: utf-8 _*_ 2 | # !/usr/bin/python 3 | __author__ = 'ma_keling' 4 | # Version : 1.1.0 5 | # Start Time : 2017-12-07 6 | # Update Time : 2018-7-20 7 | # Change Log : 8 | ## 9 | 10 | import os 11 | import sys 12 | import zipfile 13 | import arcpy 14 | import oss2 15 | import shutil 16 | 17 | # Defines the entry point into the script 18 | def main(argv=None): 19 | # input new part vtpk name 20 | newPartVtpkPath = arcpy.GetParameterAsText(0) 21 | 22 | # input update service name 23 | serviceName = arcpy.GetParameterAsText(1) 24 | 25 | # input oss access key id 26 | access_key_id = arcpy.GetParameterAsText(2) 27 | 28 | # input oss key password 29 | access_key_secret = arcpy.GetParameterAsText(3) 30 | 31 | # input bucket name 32 | bucket_name = arcpy.GetParameterAsText(4) 33 | 34 | # input endpoint 35 | endpoint = arcpy.GetParameterAsText(5) 36 | 37 | # testing parameters 38 | # newPartVtpkPath = "/Users/maklmac/Desktop/newPart.vtpk" 39 | # serviceName = 'makltest' 40 | # access_key_id = 'LTAI95MbCPCvRXjZ' 41 | # access_key_secret = '1bZNmbZzT9v8sUUqXC65VlfNeQVcRn' 42 | # bucket_name = 'zhoun' 43 | # endpoint = 'http://oss-cn-shanghai.aliyuncs.com' 44 | 45 | execute(newPartVtpkPath, serviceName, access_key_id, access_key_secret, bucket_name, endpoint) 46 | 47 | # Change vtpk extension from .vtpk to .zip 48 | def retype(newPartVtpkPath,newtype): 49 | try: 50 | filename = os.path.splitext(newPartVtpkPath)[0]; # file name 51 | filetype = os.path.splitext(newPartVtpkPath)[1]; # file type 52 | olddir = newPartVtpkPath 53 | newdir = filename + newtype 54 | os.rename(olddir, newdir) 55 | print("has changed:" + newdir) 56 | 57 | return newdir 58 | except: 59 | arcpy.AddError("retype failed: please provide a validates path") 60 | 61 | #uncompress the .zip file to folder 62 | def unzip(newPartZipPath): 63 | try: 64 | file_zip = zipfile.ZipFile(newPartZipPath, 'r') 65 | for file in file_zip.namelist(): 66 | # print "unziping..." 67 | extractFolder = os.path.splitext(newPartZipPath)[0] 68 | file_zip.extract(file, extractFolder) 69 | file_zip.close() 70 | os.remove(newPartZipPath) 71 | print("unzip succeed!") 72 | return extractFolder 73 | except: 74 | arcpy.AddError("unzip failed, please provde a validates path") 75 | return False 76 | 77 | def zip_and_retype(newPartZipPath): 78 | try: 79 | prelen = len(newPartZipPath) 80 | # print(prelen) 81 | print("zip root folder: " + newPartZipPath) 82 | 83 | zipDir = newPartZipPath + ".zip" 84 | fp = zipfile.ZipFile(zipDir, mode='w') 85 | for parent, dirnames, filenames in os.walk(newPartZipPath): 86 | for filename in filenames: 87 | pathfile = os.path.join(parent, filename) 88 | arcname = pathfile[prelen:].strip(os.path.sep) 89 | fp.write(pathfile, arcname, compress_type=zipfile.ZIP_STORED) 90 | fp.close() 91 | print("zipDir:" + zipDir) 92 | retype(zipDir,".vtpk") 93 | return True 94 | except: 95 | arcpy.AddError("path or folderName not exit.") 96 | 97 | def delete_zip_folder(newPartZipPath): 98 | shutil.rmtree(newPartZipPath) 99 | 100 | # return the unzip local tile path include LODs 101 | def get_local_tile_path(newPartVtpkPath): 102 | newPartZipPath = retype(newPartVtpkPath,".zip") 103 | extractFolder = unzip(newPartZipPath) 104 | # for mac path 105 | # tilePath = extractFolder + '/p12/tile' 106 | 107 | # for windows path 108 | tilePath = os.path.join(extractFolder, 'p12\\tile') 109 | print("local new part tile path:", tilePath) 110 | return tilePath 111 | 112 | # connect OSS by access_key_id, access_key_secrest, bucket_name, endpoint 113 | def connect_OSS(access_key_id,access_key_secret,bucket_name,endpoint): 114 | # verify parameters 115 | for param in (access_key_id, access_key_secret, bucket_name, endpoint): 116 | assert '<' not in param, '请设置参数：' + param 117 | 118 | # create Bucket object 119 | bucket = oss2.Bucket(oss2.Auth(access_key_id, access_key_secret), endpoint, bucket_name) 120 | 121 | print("connect succeed: " + bucket_name) 122 | 123 | return bucket 124 | 125 | # upload bundles from local bundle files to OSS cache directories 126 | def oss_upload_bundles(bucket,filepath,bucket_path): 127 | file_list = os.listdir(filepath) 128 | for Level in file_list: 129 | lodPath = os.path.join(filepath,Level) 130 | bundles = os.listdir(lodPath) 131 | bucket_lod_path = os.path.join(bucket_path,Level) 132 | for bundle in bundles: 133 | local_bundle_path = os.path.join(lodPath,bundle) 134 | bucket_bundle_path = os.path.join(bucket_lod_path,bundle) 135 | result = bucket.put_object_from_file(bucket_bundle_path, local_bundle_path) 136 | print(local_bundle_path) 137 | print (bucket_bundle_path) 138 | print(result) 139 | return True 140 | 141 | # execute method for OSS 142 | def upload_bundle_In_OSS(filePath, bucketPath,access_key_id, access_key_secret, bucket_name,endpoint): 143 | 144 | bucket = connect_OSS(access_key_id, access_key_secret, bucket_name, endpoint) 145 | result = oss_upload_bundles(bucket, filePath, bucketPath) 146 | return result 147 | 148 | # get bucket cache path for specific service name 149 | def get_bucket_path(serviceName): 150 | bucket_path= 'agssitecache/VectorCache/Hosted/' + serviceName + '/VectorTileServer/tile' 151 | return bucket_path 152 | 153 | 154 | def execute(newPartVtpkPath, serviceName, access_key_id, access_key_secret, bucket_name,endpoint): 155 | 156 | try: 157 | #get local vector tile cache path 158 | filepath = get_local_tile_path(newPartVtpkPath) 159 | 160 | #get oss bucket path 161 | bucket_path = get_bucket_path(serviceName) 162 | 163 | #upload local vector tile cache to oss bucket store 164 | result = upload_bundle_In_OSS(filepath,bucket_path,access_key_id, access_key_secret, bucket_name,endpoint) 165 | 166 | if result: 167 | #zip newPathVTPKPath 168 | zip_and_retype(newPartVtpkPath) 169 | #delect zip folder 170 | delete_zip_folder(newPartVtpkPath) 171 | except: 172 | arcpy.AddError("execute failed!") 173 | 174 | if __name__ == "__main__": 175 | sys.exit(main(sys.argv[1:])) 176 | 177 | 178 | 179 | -------------------------------------------------------------------------------- /python/updateVectorTilePackage.py: -------------------------------------------------------------------------------- 1 | # _*_ coding: utf-8 _*_ 2 | # !/usr/bin/python 3 | __author__ = 'ma_keling' 4 | # Version : 1.1.0 5 | # Start Time : 2017-12-07 6 | # Update Time : 2018-7-20 7 | # Change Log : 8 | ## 9 | 10 | import os 11 | import zipfile 12 | import arcpy 13 | import shutil 14 | import sys 15 | 16 | 17 | # Defines the entry point into the script 18 | def main(argv=None): 19 | original_vtpk_path = arcpy.GetParameterAsText(0) 20 | update_vtpk_path = arcpy.GetParameterAsText(1) 21 | new_vtpk_path = arcpy.GetParameterAsText(2) 22 | 23 | # original_vtpk_path = r'D:\updateVT\workspace\china_100index.vtpk' 24 | # update_vtpk_path = r'D:\updateVT\workspace\newPart.vtpk' 25 | # new_vtpk_path = r'D:\updateVT\workspace\workspace\china_100index_new.vtpk' 26 | 27 | execute(new_vtpk_path, original_vtpk_path, update_vtpk_path) 28 | 29 | 30 | # Change vtpk extension from .vtpk to .zip 31 | def retype(workspace,newPartVtpkPath,newtype): 32 | try: 33 | filename = os.path.splitext(newPartVtpkPath)[0] # file name 34 | filetype = os.path.splitext(newPartVtpkPath)[1] # file type 35 | olddir = newPartVtpkPath 36 | newdir = filename + newtype 37 | os.rename(olddir, newdir) 38 | arcpy.AddMessage("has changed:" + newdir) 39 | return newdir 40 | 41 | except: 42 | delete_zip_folder(workspace) 43 | arcpy.AddError("retype failed: please provide a validates path") 44 | 45 | 46 | 47 | 48 | #uncompress the .zip file to folder 49 | def unzip(workspace,newPartZipPath): 50 | try: 51 | file_zip = zipfile.ZipFile(newPartZipPath, 'r') 52 | for file in file_zip.namelist(): 53 | # print "unziping..." 54 | extractFolder = os.path.splitext(newPartZipPath)[0] 55 | file_zip.extract(file, extractFolder) 56 | file_zip.close() 57 | os.remove(newPartZipPath) 58 | print("unzip succeed!") 59 | return extractFolder 60 | except: 61 | delete_zip_folder(workspace) 62 | arcpy.AddError("unzip failed, please provide a validates path") 63 | return "" 64 | 65 | def zip_and_retype(workspace,original_extract_path,new_vtpk_name): 66 | try: 67 | prelen = len(original_extract_path) 68 | # print(prelen) 69 | print("zip root folder: " + original_extract_path) 70 | 71 | zipDir = original_extract_path + ".zip" 72 | fp = zipfile.ZipFile(zipDir, mode='w') 73 | for parent, dirnames, filenames in os.walk(original_extract_path): 74 | for filename in filenames: 75 | pathfile = os.path.join(parent, filename) 76 | arcname = pathfile[prelen:].strip(os.path.sep) 77 | fp.write(pathfile, arcname, compress_type=zipfile.ZIP_STORED) 78 | fp.close() 79 | print("zipDir:" + zipDir) 80 | olddir = retype(workspace,zipDir,".vtpk") 81 | newdir = os.path.join(os.path.dirname(olddir), new_vtpk_name) 82 | print("new dir:", newdir) 83 | os.rename(olddir, newdir) 84 | return True 85 | except: 86 | delete_zip_folder(workspace) 87 | arcpy.AddError("path or folderName not exit.") 88 | 89 | def delete_zip_folder(delete_path): 90 | shutil.rmtree(delete_path) 91 | 92 | def copy_files(workspace,oldvtpath=None, newvtpath=None): 93 | try: 94 | sourceDir = newvtpath 95 | targetDir = oldvtpath 96 | 97 | for f in os.listdir(sourceDir): 98 | sourceF = os.path.join(sourceDir, f) 99 | targetF = os.path.join(targetDir, f) 100 | 101 | if os.path.isfile(sourceF): 102 | if os.path.exists(targetF): 103 | open(targetF, "wb").write(open(sourceF, "rb").read()) 104 | print("copy :" + targetF) 105 | arcpy.AddMessage("copy:"+ targetF) 106 | elif os.path.isdir(sourceF): 107 | if not os.path.exists(targetDir): 108 | os.mkdir(targetDir) 109 | 110 | copy_files(workspace, targetF, sourceF) 111 | 112 | print("Copy Succeed!") 113 | return True 114 | except: 115 | # delete_zip_folder(workspace) 116 | print("input vector tile path not exit") 117 | 118 | return False 119 | 120 | # return the unzip local tile path include LODs 121 | def get_tile_path(extract_folder): 122 | # # for windows path 123 | tilePath = os.path.join(extract_folder, 'p12\\tile') 124 | print("local new part tile path:", tilePath) 125 | return tilePath 126 | 127 | 128 | def make_workspace(workspace,original_vtpk_path,update_vtpk_path): 129 | original_vtpk_name = os.path.split(original_vtpk_path)[1] 130 | update_vtpk_name = os.path.split(update_vtpk_path)[1] 131 | original_vtpk_new_path = os.path.join(workspace,original_vtpk_name) 132 | update_vtpk_new_path = os.path.join(workspace,update_vtpk_name) 133 | open(original_vtpk_new_path,'wb').write(open(original_vtpk_path,'rb').read()) 134 | open(update_vtpk_new_path,'wb').write(open(update_vtpk_path,'rb').read()) 135 | print("new workspace created succeed!",workspace,original_vtpk_new_path,update_vtpk_new_path) 136 | return True 137 | # try: 138 | # 139 | # except: 140 | # print("create new workspace failed!") 141 | 142 | 143 | def execute(new_vtpk_path,original_vtpk_path,update_vtpk_path): 144 | 145 | # workspace = os.path.dirname(new_vtpk_path) 146 | dest_path = os.path.dirname(new_vtpk_path) 147 | workspace = os.path.join(os.path.dirname(original_vtpk_path),"workspace") 148 | if os.path.exists(workspace) == False: 149 | os.mkdir(workspace) 150 | # arcpy.CreateFolder_management(workspace) 151 | arcpy.AddMessage("Scratch!!!") 152 | original_vtpk_name = os.path.basename(original_vtpk_path) 153 | update_vtpk_name = os.path.basename(update_vtpk_path) 154 | new_vtpk_name = os.path.basename(new_vtpk_path) 155 | work_vtpk_path = os.path.join(workspace,new_vtpk_name) 156 | arcpy.AddMessage("creating workspace:" + workspace) 157 | result = make_workspace(workspace,original_vtpk_path, update_vtpk_path) 158 | arcpy.AddMessage("current workspace:" + workspace) 159 | if result: 160 | original_vtpk_new_path = os.path.join(workspace,original_vtpk_name) 161 | update_vtpk_new_path = os.path.join(workspace,update_vtpk_name) 162 | original_zip_path = retype(workspace,original_vtpk_new_path,".zip") 163 | update_zip_path = retype(workspace,update_vtpk_new_path,".zip") 164 | arcpy.AddMessage("retype the extension from .vtpk to .zip") 165 | print("original_zip_path:", original_zip_path) 166 | print("update_zip_path:", update_zip_path) 167 | original_unzip_path = unzip(workspace,original_zip_path) 168 | update_unzip_path = unzip(workspace,update_zip_path) 169 | arcpy.AddMessage("uncompress the zip files"+original_vtpk_path) 170 | print("original_unzip_path:" ,original_unzip_path) 171 | print("update_unzip_path:" ,update_unzip_path) 172 | original_tiles = get_tile_path(original_unzip_path) 173 | new_tiles = get_tile_path(update_unzip_path) 174 | arcpy.AddMessage("get the tiles path for update"+original_tiles+"\n"+new_tiles) 175 | # print("original_tiles", original_tiles) 176 | # print("new_tiles",new_tiles) 177 | arcpy.AddMessage("updating.......") 178 | copy_files(workspace,original_tiles,new_tiles) 179 | 180 | result = zip_and_retype(workspace,original_unzip_path,new_vtpk_name) 181 | 182 | arcpy.AddMessage("start copy...") 183 | shutil.copy(work_vtpk_path,new_vtpk_path) 184 | 185 | if result: 186 | delete_zip_folder(workspace) 187 | 188 | 189 | arcpy.AddMessage("update successfully!") 190 | 191 | if __name__ == "__main__": 192 | sys.exit(main(sys.argv[1:])) 193 | 194 | -------------------------------------------------------------------------------- /python/createAdvancedVectorTilePacakge.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # !/usr/bin/python 3 | __author__ = 'mu_xiaoyan' 4 | # Version : 1.1.0 5 | # Start Time : 2017-12-07 6 | # Update Time : 2018-7-20 7 | # Change Log : 8 | ## 1. Replace arcpy.AddMessage() with arcpy.AddError() so to let the tool throw an exception automatically. 9 | ## 2. Optimized Error handling mechanism in Function createVtpkIndexAndPackage() 10 | ## 3. Fixed wrong error text during to old Error Handling logic. 11 | 12 | 13 | import arcpy 14 | import os 15 | import xml.dom.minidom as DOM 16 | import zipfile 17 | import shutil 18 | import sys 19 | 20 | # Update 2018-9-21 Drop the old logic 21 | # # Create Vector Tile Package Scheme for Customize Geographic Coordinate System 22 | # # Using 20 fixed Level for the first Version. It can be better in future version, if necessary. 23 | # def GenerateVtpkTilingScheme(in_map,tileScheme): 24 | # try: 25 | # scales = "295829355.454565;147914677.727283;73957338.8636413;36978669.4318207;18489334.7159103;9244667.35795516;4622333.67897758;2311166.83948879;1155583.4197444;577791.709872198;288895.854936099;144447.927468049;72223.9637340247;36111.9818670124;18055.9909335062;9027.99546675309;4513.99773337654;2256.99886668827;1128.49943334414;564.249716672068" 26 | # arcpy.server.GenerateMapServerCacheTilingScheme(in_map=in_map, 27 | # tile_origin="-180.0 180.0", 28 | # output_tiling_scheme=tileScheme, 29 | # num_of_scales=20, 30 | # scales=scales, 31 | # dots_per_inch=96, 32 | # tile_size="512 x 512") 33 | # arcpy.AddMessage("tile scheme - ready.") 34 | # return tileScheme 35 | # except: 36 | # arcpy.AddError("input map for tiling scheme invalid.") 37 | # 38 | # # Modify Scheme File to Avoid the tile_Origin Specification Bug of the Pro Tool 39 | # def modifyTilingSchemeFile(tileScheme): 40 | # try: 41 | # doc = DOM.parse(tileScheme) 42 | # tileOriginX = doc.getElementsByTagName('X') 43 | # tileOriginY = doc.getElementsByTagName('Y') 44 | # tileOriginX[0].firstChild.data = '-180' 45 | # tileOriginY[0].firstChild.data = '180' 46 | # f = open(tileScheme, 'w+') 47 | # doc.writexml(f) 48 | # f.close() 49 | # return True 50 | # except: 51 | # arcpy.AddError("tile scheme XML file does not exist.") 52 | 53 | # Create Vector Tile Index and then Create Vector Tile Package 54 | def create_vtpk_index_and_package(in_map,service_type,tile_scheme,vertex_count,index_polygon,outVtpk): 55 | try: 56 | arcpy.management.CreateVectorTileIndex(in_map=in_map, 57 | out_featureclass=index_polygon, 58 | service_type=service_type, 59 | tiling_scheme=tile_scheme, 60 | vertex_count=vertex_count) 61 | arcpy.AddMessage("tile index - ready.") 62 | arcpy.management.CreateVectorTilePackage(in_map=in_map, 63 | output_file=outVtpk, 64 | service_type=service_type, 65 | tiling_scheme=tile_scheme, 66 | tile_structure="INDEXED", 67 | min_cached_scale="", 68 | max_cached_scale="", 69 | index_polygons=index_polygon, 70 | summary=None, 71 | tags=None) 72 | arcpy.AddMessage("Pro standard tile package - ready!") 73 | return outVtpk 74 | # Update at 2018-7-20, optimized Error handling mechanism 75 | # Previous Code: 76 | # except: 77 | # arcpy.AddError("input map for packaging invalid. Check the coordinate system of the input map.") 78 | except arcpy.ExecuteError: 79 | severity = arcpy.GetMaxSeverity() 80 | if severity == 2: 81 | # If the tool returned an error 82 | arcpy.AddError("Error occurred \n{0}".format(arcpy.GetMessages(2))) 83 | elif severity == 1: 84 | # If the tool returned no errors, but returned a warning 85 | arcpy.AddWarning("Warning raised \n{0}".format(arcpy.GetMessages(1))) 86 | else: 87 | # If the tool did not return an error or a warning 88 | arcpy.AddMessage(arcpy.GetMessages()) 89 | 90 | # Append the tiling scheme and index polygon to the standard vtpk 91 | def add_to_zip(original_zip, newfolder): 92 | try: 93 | # print("zip file: " + original_zip) 94 | folder = os.path.dirname(original_zip) 95 | prelen = len(folder) 96 | fp = zipfile.ZipFile(original_zip, mode='a') 97 | for parent, dirnames, filenames in os.walk(newfolder): 98 | for filename in filenames: 99 | pathfile = os.path.join(parent, filename) 100 | # print("pathfile",pathfile) 101 | arcname = pathfile[prelen:].strip(os.path.sep) 102 | # print("arcname",arcname) 103 | fp.write(pathfile, arcname) 104 | fp.close() 105 | return True 106 | except: 107 | arcpy.AddError("path or folderName not exist.") 108 | 109 | # Clear the scratch data 110 | def delete_zip_folder(delete_path): 111 | shutil.rmtree(delete_path) 112 | 113 | # Updated at 2018-9-21 114 | # Automatically finding xml file and allowing user to choose origin index polygon 115 | def get_tile_scheme_and_index(in_map): 116 | try: 117 | arcpy.AddMessage("# Starting finding existing XML file ...") 118 | aprx = arcpy.mp.ArcGISProject("CURRENT") 119 | map = aprx.listMaps(in_map)[0] 120 | # getting xml file 121 | map_sr = map.defaultCamera.getExtent().spatialReference 122 | map_sr_name = map_sr.name 123 | map_sr_wkid = map_sr.factoryCode 124 | vtpk_xml_name = "VTTS_"+str(map_sr_wkid)+"_"+map_sr_name+".xml" 125 | local_path = os.path.join(os.path.expanduser("~"), "AppData\Local\ESRI\Geoprocessing") 126 | vtpk_xml_path = os.path.join(local_path,vtpk_xml_name) 127 | arcpy.AddMessage("# Bingo! " + vtpk_xml_path + " has been found!") 128 | tile_scheme = vtpk_xml_path 129 | # getting service type 130 | if map_sr_wkid == 3857: 131 | service_type = "ONLINE" 132 | else: 133 | service_type = "EXISTING" 134 | # building index polygons 135 | common_aux_paras = [tile_scheme, service_type] 136 | return common_aux_paras 137 | except: 138 | arcpy.AddError("Failed to get tile scheme") 139 | 140 | 141 | # the main method of this script 142 | def execute(in_map,outVtpk,isOnline,vertex_count): 143 | vtpkDir = os.path.join(os.path.dirname(outVtpk), "AdvVtpkAuxFiles") 144 | # arcpy.AddMessage("TempFolder"+vtpkDir) 145 | if not os.path.exists(vtpkDir): 146 | os.makedirs(vtpkDir) 147 | 148 | # Updated 2018-9-21 149 | common_aux_paras = get_tile_scheme_and_index(in_map) 150 | tile_scheme = common_aux_paras[0] 151 | arcpy.AddMessage(tile_scheme) 152 | service_type = common_aux_paras[1] 153 | shutil.copy(tile_scheme,vtpkDir) 154 | index_polygon = vtpkDir + "\originMasterIndex.shp" 155 | 156 | # # Update 2018-9-21 Drop the old logic 157 | # if isOnline: 158 | # service_type = "ONLINE" 159 | # tileScheme = "" 160 | # arcpy.AddMessage("service type:"+service_type) 161 | # else: 162 | # service_type = "EXISTING" 163 | # arcpy.AddMessage("service type:"+service_type) 164 | # GenerateVtpkTilingScheme(in_map,tileScheme) 165 | # arcpy.AddMessage(tileScheme) 166 | # modifyTilingSchemeFile(tileScheme) 167 | 168 | originalVTPK = create_vtpk_index_and_package(in_map,service_type,tile_scheme,vertex_count,index_polygon,outVtpk) 169 | if add_to_zip(originalVTPK, vtpkDir): 170 | delete_zip_folder(vtpkDir) 171 | 172 | arcpy.AddMessage("advanced vector tile package has been generated.") 173 | 174 | 175 | def main(argv=None): 176 | # Input map in current project 177 | in_map = arcpy.GetParameterAsText(0) 178 | arcpy.AddMessage("Input map : {0}.".format(in_map)) 179 | 180 | # Specify name and workspace for the output vtpk 181 | outVtpk = arcpy.GetParameterAsText(1) 182 | arcpy.AddMessage("Output advanced vtpk : {0}.".format(outVtpk)) 183 | 184 | # Specify service type 185 | isOnline = arcpy.GetParameter(2) 186 | arcpy.AddMessage("isOnline:"+str(isOnline)) 187 | 188 | # Specify maximum vertex count 189 | vertex_count = arcpy.GetParameterAsText(3) 190 | arcpy.AddMessage("Maximum vertex count : {0}.".format(vertex_count)) 191 | 192 | execute(in_map,outVtpk,isOnline,vertex_count) 193 | 194 | if __name__ == "__main__": 195 | sys.exit(main(sys.argv[1:])) -------------------------------------------------------------------------------- /python/createPartVectorTilePackage.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # !/usr/bin/python 3 | __author__ = 'mu_xiaoyan' 4 | # Version : 2.1.0 5 | # Start Time : 2017-12-07 6 | 7 | # Update Time : 2018-09-20 8 | # Change Log : 9 | ## Add support for getting known XML 10 | 11 | # Update Time : 2018-8-13 12 | # Change Log : 13 | ## Added new logical to calculate affected bundle extent in each level to avoid vector tile lost 14 | ## New function - calculate_affected_indexes_layer(AOI, index_polygon) 15 | ## Perfectly maintained spatial relationship internal 16 | ## Obsoleted parameter LOD, which now is calculated automatically 17 | ## Automatically removed intermediate data and workspace 18 | 19 | 20 | import time, zipfile, arcpy 21 | import sys, os, shutil 22 | 23 | 24 | # uncompress the .zip file to folder 25 | def unzip(newPartZipPath): 26 | try: 27 | file_zip = zipfile.ZipFile(newPartZipPath, 'r') 28 | for file in file_zip.namelist(): 29 | # print "unziping..." 30 | extractFolder = os.path.splitext(newPartZipPath)[0] 31 | file_zip.extract(file, extractFolder) 32 | file_zip.close() 33 | os.remove(newPartZipPath) 34 | #arcpy.AddMessage("unzip succeed!") 35 | return extractFolder 36 | except: 37 | arcpy.AddError("unzip failed, please provde a validates path") 38 | return "" 39 | 40 | # Analyzing Original vtpk file to get the tiling scheme and index polygon and also get the service type 41 | def analysis_original_vtpk(origin_vtpk_path): 42 | 43 | origin_workspace = os.path.dirname(origin_vtpk_path) 44 | origin_vtpk_name = os.path.basename(origin_vtpk_path) 45 | # Create temp workspace 46 | timeStamp = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time())) 47 | temp_workspace = os.path.join(origin_workspace, str(timeStamp)) 48 | os.mkdir(temp_workspace) 49 | try: 50 | # copy original vtpk 51 | bak_original_vtpk = shutil.copy(origin_vtpk_path, temp_workspace) 52 | unzip(bak_original_vtpk) 53 | # Locate vtpk aux files 54 | tile_scheme_name = "customizedScheme.xml" 55 | index_polygon_name = "originMasterIndex.shp" 56 | vtpk_extract_dir = origin_vtpk_name.split(".")[0] 57 | arcpy.AddMessage(vtpk_extract_dir) 58 | aux_files_path = temp_workspace+ r"\\"+ vtpk_extract_dir+ r"\AdvVtpkAuxFiles" 59 | index_polygon = os.path.join(aux_files_path, index_polygon_name) 60 | tile_scheme = os.path.join(aux_files_path, tile_scheme_name) 61 | # get the service type 62 | if os.path.exists(tile_scheme): 63 | service_type = "EXISTING" 64 | else: 65 | tile_scheme = "" 66 | service_type = "ONLINE" 67 | adv_aux_paras = [index_polygon, tile_scheme, service_type,temp_workspace] 68 | return adv_aux_paras 69 | except: 70 | arcpy.AddError("Original vtpk does not exist.") 71 | 72 | # Updated at 2018-9-21 73 | # Automatically finding xml file and allowing user to choose origin index polygon 74 | def get_tile_scheme_and_index(in_map): 75 | try: 76 | arcpy.AddMessage("# Starting finding existing XML file ...") 77 | aprx = arcpy.mp.ArcGISProject("CURRENT") 78 | map = aprx.listMaps(in_map)[0] 79 | # getting xml file 80 | map_sr = map.defaultCamera.getExtent().spatialReference 81 | map_sr_name = map_sr.name 82 | map_sr_wkid = map_sr.factoryCode 83 | vtpk_xml_name = "VTTS_"+str(map_sr_wkid)+"_"+map_sr_name+".xml" 84 | local_path = os.path.join(os.path.expanduser("~"), "AppData\Local\ESRI\Geoprocessing") 85 | vtpk_xml_path = os.path.join(local_path,vtpk_xml_name) 86 | arcpy.AddMessage("# Bingo! " + vtpk_xml_path + " has been found!") 87 | tile_scheme = vtpk_xml_path 88 | # getting service type 89 | if map_sr_wkid == 3857: 90 | service_type = "ONLINE" 91 | else: 92 | service_type = "EXISTING" 93 | # building index polygons 94 | common_aux_paras = [tile_scheme, service_type] 95 | return common_aux_paras 96 | except: 97 | arcpy.AddError("Failed to get tile scheme") 98 | 99 | # Updated at 2018-8-9 100 | # Added new function calculating the exact bundle extent in each level of vector tile 101 | def calculate_affected_indexes_layer(AOI,index_polygon): 102 | try: 103 | # in memory variables 104 | affected_indexes_fc = "in_memory/base_affected_indexes" 105 | statistics_LOD_tbl = "in_memory/statistics_LOD" 106 | bundle_polygon_each_level = "in_memory/bundle_index_polygon_each_level" 107 | timeStamp = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time())) 108 | bundle_index_polygons = 'intermediate_indexes_' + str(timeStamp) + '.shp' 109 | arcpy.AddMessage(bundle_index_polygons + " Created ! Do not delete it whiling the tool is running.") 110 | AOI_lyr = arcpy.MakeFeatureLayer_management(AOI, "AOI_lyr_"+ str(timeStamp)) 111 | index_polygon_lyr = arcpy.MakeFeatureLayer_management(index_polygon, "index_lyr_" + str(timeStamp)) 112 | 113 | # Finding the max LOD 114 | arcpy.SelectLayerByLocation_management(index_polygon_lyr, 'INTERSECT', AOI_lyr) 115 | arcpy.Statistics_analysis(index_polygon_lyr, statistics_LOD_tbl, [["LOD", "MAX"]]) 116 | statistics_LOD = [row[0] for row in arcpy.da.SearchCursor(statistics_LOD_tbl, ['MAX_LOD'])] 117 | max_lod = statistics_LOD[0] 118 | arcpy.AddMessage("- Max affected LOD detected "+str(max_lod)) 119 | arcpy.AddMessage("- Partial vector indexes Creating ... ") 120 | # Finding indexed polygons overlaped AOI 121 | arcpy.CopyFeatures_management(index_polygon_lyr, affected_indexes_fc) 122 | arcpy.SelectLayerByAttribute_management(index_polygon_lyr,'CLEAR_SELECTION') 123 | base_affected_indexes_lyr = arcpy.MakeFeatureLayer_management(affected_indexes_fc, "base_affected_indexes_lyr") 124 | # Calculating bundle extent in each level 125 | arcpy.Select_analysis(index_polygon, bundle_index_polygons, '"LOD"<7') 126 | with arcpy.da.SearchCursor(affected_indexes_fc, ['SHAPE@','LOD']) as cursor: 127 | for row in cursor: 128 | if row[1]<=max_lod and row[1]>=7: 129 | bundle_lod = row[1]-7 130 | arcpy.SelectLayerByAttribute_management(base_affected_indexes_lyr, 'NEW_SELECTION', 131 | '"LOD" = '+str(bundle_lod)) 132 | arcpy.SelectLayerByAttribute_management(index_polygon_lyr, 'NEW_SELECTION', 133 | '"LOD" = ' + str(row[1])) 134 | arcpy.SelectLayerByLocation_management(index_polygon_lyr, "WITHIN", base_affected_indexes_lyr, None, 135 | "SUBSET_SELECTION", "NOT_INVERT") 136 | arcpy.CopyFeatures_management(index_polygon_lyr, bundle_polygon_each_level) 137 | arcpy.Append_management([bundle_polygon_each_level], bundle_index_polygons, "NO_TEST") 138 | arcpy.Delete_management("in_memory") 139 | arcpy.Delete_management(AOI_lyr) 140 | arcpy.Delete_management(index_polygon_lyr) 141 | arcpy.Delete_management(base_affected_indexes_lyr) 142 | arcpy.AddMessage("- Partial vector indexes Generated !") 143 | return bundle_index_polygons 144 | 145 | except arcpy.ExecuteError: 146 | severity = arcpy.GetMaxSeverity() 147 | if severity == 2: 148 | # If the tool returned an error 149 | arcpy.AddError("Error occurred \n{0}".format(arcpy.GetMessages(2))) 150 | elif severity == 1: 151 | # If the tool returned no errors, but returned a warning 152 | arcpy.AddWarning("Warning raised \n{0}".format(arcpy.GetMessages(1))) 153 | else: 154 | # If the tool did not return an error or a warning 155 | arcpy.AddMessage(arcpy.GetMessages()) 156 | return "" 157 | 158 | def main(argv=None): 159 | # Input map in the current project 160 | in_map = arcpy.GetParameterAsText(0) 161 | arcpy.AddMessage("# Input map : {0}.".format(in_map)) 162 | 163 | # Specify the area where the delta new part vtpk need to be created 164 | AOI = arcpy.GetParameterAsText(1) 165 | arcpy.AddMessage("# AOI : {0}.".format(AOI)) 166 | 167 | # Choose the existing original adv vtpk 168 | origin_vtpk = arcpy.GetParameterAsText(2) 169 | arcpy.AddMessage("# Original vtpk : {0}.".format(origin_vtpk)) 170 | 171 | # Update 2018-9-20 add boolean parameter to check if the vtpk is advanced 172 | # Check if the vtpk is advanced 173 | is_adv_vtpk = arcpy.GetParameter(3) 174 | arcpy.AddMessage("# Is Advanced Vtpk : {0}.".format(is_adv_vtpk)) 175 | # Specify the index polygon 176 | index_polygon = arcpy.GetParameter(4) 177 | arcpy.AddMessage("# Index Polygon path : {0}.".format(index_polygon)) 178 | 179 | # Specify name and workspace for new part vtpk 180 | out_part_vtpk = arcpy.GetParameterAsText(5) 181 | arcpy.AddMessage("# New part vtpk : {0}.".format(out_part_vtpk)) 182 | 183 | execute(in_map, AOI, origin_vtpk, is_adv_vtpk, index_polygon, out_part_vtpk) 184 | 185 | # Update 2018-8-10, Calling the new function 186 | # calculating the exact bundle extent in each level of vector tile 187 | def execute(in_map, AOI, origin_vtpk, is_adv_vtpk, index_polygon, out_part_vtpk): 188 | 189 | arcpy.env.workspace = os.path.dirname(out_part_vtpk) 190 | temp_workspace = None 191 | 192 | # Update 2018-9-20, add support for user defined xml file 193 | if is_adv_vtpk: 194 | # analyzing original_vtpk to get the scheme and index polygon feature class 195 | adv_aux_paras = analysis_original_vtpk(origin_vtpk) 196 | index_polygon = adv_aux_paras[0] 197 | tile_scheme = adv_aux_paras[1] 198 | service_type = adv_aux_paras[2] 199 | temp_workspace = adv_aux_paras[3] 200 | arcpy.AddMessage("- Analyzing origin vtpk succesfully! " 201 | + "\n - " + index_polygon + "\n - " + tile_scheme + "\n - " + service_type) 202 | else: 203 | common_aux_paras = get_tile_scheme_and_index(in_map) 204 | tile_scheme = common_aux_paras[0] 205 | service_type = common_aux_paras[1] 206 | arcpy.AddMessage("- Servcie Type: " + service_type) 207 | 208 | # Calculating affected bundle indexes 209 | bundle_index_polygons = calculate_affected_indexes_layer(AOI, index_polygon) 210 | arcpy.AddMessage("# Affected exactly index polygons created! ") 211 | arcpy.AddMessage("# Starting creating partial vector tiles ... ") 212 | arcpy.CreateVectorTilePackage_management(in_map=in_map, 213 | output_file=out_part_vtpk, 214 | service_type=service_type, 215 | tiling_scheme=tile_scheme, 216 | tile_structure="INDEXED", 217 | index_polygons=bundle_index_polygons) 218 | arcpy.AddMessage("# Partial vector tiles created! ") 219 | 220 | # Remove temporary intermediate data 221 | if os.path.exists(out_part_vtpk) and arcpy.Exists(out_part_vtpk): 222 | arcpy.Delete_management(os.path.join(arcpy.env.workspace, bundle_index_polygons)) 223 | arcpy.AddMessage("Temp Data deleted - " + bundle_index_polygons) 224 | if os.path.exists(out_part_vtpk) and temp_workspace is not None: 225 | shutil.rmtree(temp_workspace) 226 | arcpy.AddMessage("Temp Data deleted - " + temp_workspace) 227 | 228 | 229 | 230 | if __name__ == "__main__": 231 | sys.exit(main(sys.argv[1:])) 232 | -------------------------------------------------------------------------------- /python/updateVectorTileService.py: -------------------------------------------------------------------------------- 1 | # _*_ coding: utf-8 _*_ 2 | # !/usr/bin/python 3 | __author__ = 'ma_keling' 4 | # Version : 1.1.0 5 | # Start Time : 2017-12-07 6 | # Update Time : 2018-7-20 7 | # Change Log : 8 | ## 9 | 10 | import os 11 | import sys 12 | import zipfile 13 | import shutil 14 | import requests 15 | import json 16 | import paramiko 17 | from smb.SMBConnection import * 18 | import arcpy 19 | 20 | 21 | def main(argv=None): 22 | update_vtpk_path = arcpy.GetParameterAsText(0) 23 | service_url = arcpy.GetParameterAsText(1) 24 | ags_username = arcpy.GetParameterAsText(2) 25 | ags_password = arcpy.GetParameterAsText(3) 26 | share_hostname=arcpy.GetParameterAsText(4) 27 | share_admin = arcpy.GetParameterAsText(5) 28 | share_password = arcpy.GetParameterAsText(6) 29 | 30 | # local part vtpk from windows system 31 | # update_cache_path = r'C:\makl\testdata\NewPart.vtpk' 32 | 33 | # for vector tile service from arcgis server 10.5 for windows 34 | # service_url = r'https://120win105.esrichina.com/server/rest/services/Hosted/china400W_vtpk/VectorTileServer' 35 | 36 | # for vector tile service from arcgis server 10.5 for linux 37 | # service_url = r"https://123linux106.esrichina.com:6443/arcgis/rest/services/Hosted/china400W_vtpk/VectorTileServer" 38 | 39 | # for arcgis server admin account 40 | # ags_username = 'arcgis' 41 | # ags_password = 'Super123' 42 | 43 | # cache directory stored in windows file system 44 | # share_hostname = '192.168.220.120' 45 | # share_admin = "administrator" 46 | # share_password = "Super123" 47 | 48 | # cache directory stored in linux file system 49 | # share_hostname = '192.168.220.123' 50 | # share_admin = "arcgis" 51 | # share_password = "Super123" 52 | 53 | execute(update_vtpk_path, service_url, ags_username, ags_password, share_hostname, 54 | share_admin, share_password) 55 | 56 | #common http tools 57 | #assistant method for submit request 58 | def submit_request(url,params,item=""): 59 | err_flag = 'failed' 60 | try: 61 | r = requests.post(url, data=params, verify=False) 62 | 63 | if (r.status_code != 200): 64 | r.raise_for_status() 65 | print('request failed.') 66 | return err_flag 67 | else: 68 | data = r.text 69 | elapse_time = str(r.elapsed.microseconds/1000/1000) + 's' 70 | # Check that data returned is not an error object 71 | if not assertJsonSuccess(data): 72 | return 73 | # Extract service list from data 74 | result = json.loads(data) 75 | 76 | if item != "": 77 | last_result = result[item] 78 | else: 79 | last_result = result 80 | 81 | # arcpy.AddMessage("response:"+str(last_result)) 82 | return elapse_time,last_result 83 | except : 84 | arcpy.AddError("request failed") 85 | return err_flag 86 | 87 | # assert response json 88 | def assertJsonSuccess(data): 89 | obj = json.loads(data) 90 | if 'status' in obj and obj['status'] == "error": 91 | arcpy.AddError('Error: JSON object returns an error.' + str(obj)) 92 | sys.exit(False) 93 | else: 94 | return True 95 | 96 | # connect_remote_linux_path(hostname, username, password, filepath, cache_path) 97 | def connect_remote_linux_path(hostname, username, password, filepath, cache_path): 98 | try: 99 | ##连接远程主机 100 | client = paramiko.SSHClient() 101 | client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 102 | client.connect(hostname, 22, username, password) 103 | 104 | sftp = client.open_sftp() 105 | 106 | file_list = os.listdir(filepath) 107 | print(filepath) 108 | for Level in file_list: 109 | lodPath = os.path.join(filepath, Level) 110 | bundles = os.listdir(lodPath) 111 | server_lod_path = cache_path + '/' + Level 112 | for bundle in bundles: 113 | local_bundle_path = lodPath + os.sep + bundle 114 | server_bundle_path = server_lod_path + '/' + bundle 115 | print("local:",local_bundle_path) 116 | print("server:",server_bundle_path) 117 | sftp.put(local_bundle_path, server_bundle_path) 118 | 119 | 120 | client.close() 121 | return True 122 | except: 123 | arcpy.AddError("upload bundle failed!") 124 | return False 125 | 126 | # connect_remote_win_path(hostname,username,password,filepath,cache_path) 127 | def connect_remote_win_path(hostname,username,password,filepath,cache_path): 128 | localserver = "local" 129 | domain = "" 130 | service_name = '' 131 | 132 | try: 133 | if cache_path[:2] == r"\\": 134 | ll = cache_path.split('\\') 135 | service_name = ll[3] 136 | print("service_name") 137 | new_cache_path = "" 138 | for i in range(len(ll)): 139 | if i > 3: 140 | new_cache_path += "/"+ ll[i] 141 | print("new_cache_path:",new_cache_path) 142 | 143 | else: 144 | service_name = cache_path[:1] + "$" 145 | ll = cache_path.split('\\') 146 | new_cache_path = "" 147 | for i in range(len(ll)): 148 | if i > 0: 149 | new_cache_path += "/" + ll[i] 150 | print("new_cache_path:", new_cache_path) 151 | 152 | print("connecting server") 153 | conn = SMBConnection(username,password,localserver,hostname,domain=domain, 154 | use_ntlm_v2=True,sign_options=SMBConnection.SIGN_WHEN_SUPPORTED,is_direct_tcp=True) 155 | 156 | conn.connect(hostname, 445) 157 | 158 | response = conn.listShares(timeout=30) # obtain a list of shares 159 | print('Shares on: ' + hostname) 160 | arcpy.AddMessage('Shares on: ' + hostname) 161 | 162 | for i in range(len(response)): # iterate through the list of shares 163 | print(" Share[", i, "] =", response[i].name) 164 | 165 | print("filepath:", filepath) 166 | 167 | try: 168 | 169 | file_list = os.listdir(filepath) 170 | 171 | for Level in file_list: 172 | lodPath = os.path.join(filepath, Level) 173 | bundles = os.listdir(lodPath) 174 | server_lod_path = new_cache_path + "/" + Level 175 | for bundle in bundles: 176 | local_bundle_path = lodPath + os.sep + bundle 177 | server_bundle_path = server_lod_path + "/" + bundle 178 | print("local:",local_bundle_path) 179 | # print("server:", server_bundle_path) 180 | file_obj = open(local_bundle_path,'rb') 181 | 182 | # service_name = response[2].name 183 | print("service_name:",service_name) 184 | conn.storeFile(service_name, server_bundle_path, file_obj, timeout=60) 185 | print("uploaded:", server_bundle_path) 186 | arcpy.AddMessage("uploaded:"+server_bundle_path) 187 | file_obj.close() 188 | 189 | 190 | # print("upload over") 191 | conn.close() 192 | return True 193 | except: 194 | arcpy.AddError("upload failed! Maybe there is no any tiles in the dir: "+filepath) 195 | conn.close() 196 | return False 197 | except: 198 | conn.close() 199 | return False 200 | 201 | # get token by arcgis server 202 | def generateToken(url, username, password): 203 | tokenUrl = url + '/admin/generateToken' 204 | print(tokenUrl) 205 | # , 'ip':'192.168.100.85' 206 | params = {'username': username, 'password': password, 'client': 'requestip', 'f': 'json'} 207 | 208 | item = 'token' 209 | 210 | r = submit_request(tokenUrl, params, item) 211 | 212 | # arcpy.AddMessage("token:"+str(r[1])) 213 | return r[1] 214 | 215 | def get_cahces_list(url,token): 216 | directory_url = url + '/admin/system/directories' 217 | print('directory_url', directory_url) 218 | 219 | d_params = {'token': token, 'f': 'json'} 220 | 221 | result = submit_request(directory_url, d_params) 222 | 223 | directories = result[1]['directories'] 224 | 225 | cache_dirs = [] 226 | 227 | for dir in directories: 228 | if dir['directoryType'] == 'CACHE': 229 | print(dir) 230 | cache_dirs.append(dir['physicalPath']) 231 | print(dir['physicalPath']) 232 | 233 | return cache_dirs 234 | 235 | def get_cache_dir(url,token,service,item): 236 | # https://120win105.esrichina.com:6443/arcgis/admin/services/Hosted/mytest5000.VectorTileServer 237 | directory_url = url + '/admin/services/Hosted/'+service+".VectorTileServer" 238 | print('directory_url', directory_url) 239 | 240 | d_params = {'token': token, 'f': 'json'} 241 | 242 | result = submit_request(directory_url, d_params) 243 | 244 | # print("cache_result",result[1]) 245 | cache_dir = "" 246 | 247 | cache_dir_main = result[1]['properties']['cacheDir'] 248 | if cache_dir_main.__contains__('\\'): 249 | cache_dir = cache_dir_main + '\\VectorCache\\Hosted\\'+ service + '\\VectorTileServer\\tile' 250 | else: 251 | cache_dir = cache_dir_main + '/VectorCache/Hosted/'+ service + '/VectorTileServer/tile' 252 | 253 | # arcpy.AddMessage("token:" + str(cache_dir)) 254 | return cache_dir 255 | 256 | # Defines the entry point into the script 257 | 258 | def parse_service_url(service_url): 259 | node_list = str(service_url).split('/') 260 | print(node_list) 261 | if len(node_list[2].split(":")) > 1: 262 | server_url = node_list[0]+'//'+node_list[2]+'/'+node_list[3] 263 | service_name = node_list[7] 264 | else: 265 | server_url = node_list[0]+'//'+node_list[2]+':6443/arcgis' 266 | service_name = node_list[7] 267 | 268 | 269 | return service_name,server_url 270 | 271 | # Change vtpk extension from .vtpk to .zip 272 | def retype(newPartVtpkPath,newtype): 273 | try: 274 | filename = os.path.splitext(newPartVtpkPath)[0]; # file name 275 | filetype = os.path.splitext(newPartVtpkPath)[1]; # file type 276 | 277 | olddir = newPartVtpkPath 278 | newdir = filename + newtype 279 | 280 | os.rename(olddir, newdir) 281 | 282 | return newdir 283 | except: 284 | arcpy.AddError("retype failed: please provide a validates path") 285 | 286 | #uncompress the .zip file to folder 287 | def unzip(newPartZipPath): 288 | try: 289 | file_zip = zipfile.ZipFile(newPartZipPath, 'r') 290 | for file in file_zip.namelist(): 291 | # print "unziping..." 292 | extractFolder = os.path.splitext(newPartZipPath)[0] 293 | file_zip.extract(file, extractFolder) 294 | file_zip.close() 295 | os.remove(newPartZipPath) 296 | print("unzip succeed!") 297 | return extractFolder 298 | except: 299 | arcpy.AddError("unzip failed, please provde a validates path") 300 | return "" 301 | 302 | #zip and retype file 303 | def zip_and_retype(original_extract_path,new_vtpk_name): 304 | try: 305 | prelen = len(original_extract_path) 306 | # print(prelen) 307 | # print("zip root folder: " + original_extract_path) 308 | 309 | zipDir = original_extract_path + ".zip" 310 | fp = zipfile.ZipFile(zipDir, mode='w') 311 | for parent, dirnames, filenames in os.walk(original_extract_path): 312 | for filename in filenames: 313 | pathfile = os.path.join(parent, filename) 314 | arcname = pathfile[prelen:].strip(os.path.sep) 315 | fp.write(pathfile, arcname, compress_type=zipfile.ZIP_STORED) 316 | fp.close() 317 | # print("zipDir:" + zipDir) 318 | olddir = retype(zipDir,".vtpk") 319 | newdir = os.path.join(os.path.dirname(olddir), new_vtpk_name) 320 | # print("new dir:", newdir) 321 | os.rename(olddir, newdir) 322 | return True 323 | except: 324 | arcpy.AddError("path or folderName not exit.") 325 | 326 | def delete_zip_folder(delete_path): 327 | shutil.rmtree(delete_path) 328 | 329 | # upload bundles from local bundle files to OSS cache directories 330 | def upload_bundles(filepath, cache_path, hostname, username, password): 331 | result = False 332 | 333 | if cache_path[:1] == '/': 334 | result = connect_remote_linux_path(hostname, username, password, filepath, cache_path) 335 | elif cache_path[:2] == '\\': 336 | hostname_w = cache_path.split('\\')[2] 337 | result = connect_remote_win_path(hostname_w.username, password, filepath, cache_path) 338 | else: 339 | result = connect_remote_win_path(hostname, username, password, filepath, cache_path) 340 | 341 | return result 342 | 343 | # get unzip local .vtpk file and return local cache path 344 | def get_local_cache_path(upate_vtpk_path): 345 | zip_path = retype(upate_vtpk_path,'.zip') 346 | print("zip path:", zip_path) 347 | unzip_path = unzip(zip_path) 348 | 349 | if unzip_path[:1] == r"/": 350 | cache_path = os.path.join(unzip_path,'p12/tile') 351 | else: 352 | cache_path = os.path.join(unzip_path, 'p12/tile') 353 | 354 | return cache_path 355 | 356 | # execute method for arcgis gp tool 357 | def execute(update_vtpk_path,service_url,ags_username,ags_password,share_hostname,admin_username,admin_password): 358 | # arcpy.AddMessage('update starting...') 359 | 360 | update_vtpk_name = os.path.split(update_vtpk_path)[1] 361 | # arcpy.AddMessage('get update vtpk name: '+update_vtpk_name) 362 | print('get update vtpk name: '+update_vtpk_name) 363 | 364 | local_cache_path = get_local_cache_path(update_vtpk_path) 365 | # arcpy.AddMessage('get local part vector tile cache dir:'+local_cache_path) 366 | print('get local part vector tile cache dir:'+local_cache_path) 367 | 368 | service_name, server_url = parse_service_url(service_url) 369 | print("get service name : "+ service_name) 370 | print("get service url: " + server_url) 371 | 372 | token = generateToken(server_url, username=ags_username, password=ags_password) 373 | 374 | # arcpy.AddMessage('get service token: '+ token) 375 | print('get service token: ' + token) 376 | 377 | vt_path = get_cache_dir(server_url, token, service_name, 'cacheDir') 378 | 379 | # arcpy.AddMessage('get service tile cache dir: '+vt_path) 380 | print('get service tile cache dir: '+vt_path) 381 | 382 | # arcpy.AddMessage('uploading service tiles ...') 383 | print('uploading service tiles ...') 384 | 385 | if upload_bundles(local_cache_path, vt_path, share_hostname,admin_username,admin_password): 386 | # arcpy.AddMessage('upload cache successfully!') 387 | print('upload cache successfully!') 388 | unzip_path = update_vtpk_path.split(sep='.')[0] 389 | if zip_and_retype(unzip_path,update_vtpk_name): 390 | delete_zip_folder(unzip_path) 391 | # arcpy.AddMessage('All the update task finished!') 392 | print('All the update task finished!') 393 | 394 | 395 | if __name__ == "__main__": 396 | # sys.exit(main(sys.argv[1:])) 397 | sys.exit(main()) 398 | 399 | 400 | -------------------------------------------------------------------------------- /.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | <<<<<<< HEAD 6 | <<<<<<< HEAD 7 | ======= 8 | 9 | 10 | 11 | ======= 12 | >>>>>>> 5611968ea3870391f7845ed27030be38f9393dde 13 | 14 | 15 | 16 | 17 | <<<<<<< HEAD 18 | ======= 19 | 20 | 21 | 22 | 23 | >>>>>>> 137ea50a2b2444a3035f23d241c37f2e796eb822 24 | ======= 25 | >>>>>>> origin/master 26 | >>>>>>> 5611968ea3870391f7845ed27030be38f9393dde 27 | 28 | 35 | 36 | 38 | 39 | <<<<<<< HEAD 40 | 41 | 42 | 43 | 44 | <<<<<<< HEAD 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | ======= 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | <<<<<<< HEAD 81 | ======= 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | >>>>>>> 137ea50a2b2444a3035f23d241c37f2e796eb822 113 | ======= 114 | >>>>>>> origin/master 115 | >>>>>>> 5611968ea3870391f7845ed27030be38f9393dde 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 126 | 127 | 156 | 157 | 158 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | <<<<<<< HEAD 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 |