├── .gitignore ├── LICENSE ├── README.md ├── settings.ini ├── settings_image.png ├── update.py └── update_directory.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.sd 2 | *.sddraft -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # update-hosted-feature-service 2 | 3 | This Python script turns an MXD into a service definition (SD) file. Your ArcGIS.com account is searched for a matching service. The SD file is uploaded to your ArcGIS.com account, the existing service is overwritten with new content. 4 | 5 | #### Note! Update as of August, 2015 - A major overhaul to the entire workflow, but it'll continue updating your hosted feature service just the same as it did previously! The script no longer requires [Requests](http://docs.python-requests.org/en/latest/), it uses built-in Python libraries. It will now push your SD file to arcgis.com using a multipart upload (10mb chunks). This means it should support really big uploads without running out of memory! 6 | 7 | ###### (old)Note! This script was updated August 29, 2014 - The script no longer deletes and re-creates the feature service. It now will overwrite it in place. This means the original itemID of the feature service is maintained and any webmaps that referenced the feature service will continue to work. 8 | 9 | See more information on the [associated ArcGIS Blog post](http://blogs.esri.com/esri/arcgis/2014/01/24/updating-your-hosted-feature-service-for-10-2/). 10 | The original blog post for ArcGIS 10.1 can be found [here](http://blogs.esri.com/esri/arcgis/2013/04/23/updating-arcgis-com-hosted-feature-services-with-python/). 11 | 12 | ## Instructions: 13 | ### To update a single service 14 | 1. Download the update.py and settings.ini files. (Hint: Click the `Download ZIP` button on the right) 15 | 2. Save these files to your local working directory 16 | 3. __Note!__ You no longer need to download requests. The script will run using built-in Python modules now! 17 | 4. Update the settings.ini file to values for your service. 18 | ![App](settings_image.png) 19 | 5. Run the python script 20 | 21 | ``` 22 | c:\>c:\Python27\ArcGIS10.2\python.exe c:\myLocalDirectory\update.py 23 | ... 24 | Starting Feature Service publish process 25 | found Feature Service : 7ac9e68e9cd341e0a0a217590e4f6265 26 | found Service Definition : bb2261fe2d684e7cb950c6d29372642e 27 | Created D:\myLocalFolder\MyMaps\tempDir\MyMapService.sd 28 | updated SD: bb2261fe24684e7cb950c6d29372642e 29 | successfully deleted...7ac9e68e9cd34fe0a0f217590e4f6265... 30 | successfully updated...[{u'encodedServiceURL': u'http://services1.arcgis.com/hLJbHVsas2rDIzK0I/arcgis/rest/services/MyMapService/FeatureServer', u'jobId': u'c886b86c-46d4-4be2-95ab-32b284b72dfb', u'serviceurl': u'http://services1.arcgis.com/hLJbHVsas2rDIzK0I/arcgis/rest/services/MyMapService/FeatureServer', u'type': u'Feature Service', u'serviceItemId': u'7df087dfea1b4c7bad2ba372faeefc1c', u'size': 75344}]... 31 | successfully shared...7df087dfea1b4c7bad2ba372faeefc1c... 32 | finished. 33 | ``` 34 | ### To update multiple services 35 | 1. Download the update_directory.py and settings.ini files. (Hint: Click the `Download ZIP` button on the right) 36 | 2. Save these files to your local working directory 37 | 3. Update a settings.ini file for each of your services. Create one .ini file for each service. Place these multiple .ini files in a single directory. The file names do not matter as long as you keep the .ini extension. 38 | ![App](settings_image.png) 39 | 4. Update the directory variable on line 25 of update_directory.py to point to the directory holding your multiple .ini files (e.g. `directory = r"C:\path\to\ini_files"` ). 40 | 5. Run the update_directory.py python script. 41 | 42 | ## Requirements 43 | 44 | * ArcGIS 10.2, 10.2.1, 10.3, 10.3.1, 10.4, 10.4.1 45 | * Python 2.7 (not currently configured to work with Python 3+) 46 | 47 | ## Resources 48 | 49 | * [ArcGIS REST API](http://resources.arcgis.com/en/help/arcgis-rest-api/index.html#/The_ArcGIS_REST_API/02r300000054000000/) 50 | 51 | 52 | ## Issues 53 | 54 | Find a bug or want to request a new feature? Please let us know by submitting an issue. 55 | 56 | ## Contributing 57 | 58 | Esri welcomes contributions from anyone and everyone. Please see our [guidelines for contributing](https://github.com/esri/contributing). 59 | 60 | ## Licensing 61 | Copyright 2013 Esri 62 | 63 | Licensed under the Apache License, Version 2.0 (the "License"); 64 | you may not use this file except in compliance with the License. 65 | You may obtain a copy of the License at 66 | 67 | http://www.apache.org/licenses/LICENSE-2.0 68 | 69 | Unless required by applicable law or agreed to in writing, software 70 | distributed under the License is distributed on an "AS IS" BASIS, 71 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 72 | See the License for the specific language governing permissions and 73 | limitations under the License. 74 | 75 | A copy of the license is available in the repository's [license.txt]( https://github.com/update-hosted-feature-service/master/license.txt) file. 76 | 77 | [](Esri Tags: ArcGIS.com Online Update Hosted Feature Services Automate Python Publish) 78 | [](Esri Language: Python)​ 79 | -------------------------------------------------------------------------------- /settings.ini: -------------------------------------------------------------------------------- 1 | [FS_INFO] 2 | SERVICENAME = MyMapService 3 | FOLDERNAME = None 4 | MXD = D:\nightly_updates\maps\MyMap.mxd 5 | TAGS = points, dots, places 6 | DESCRIPTION = This is the description text 7 | MAXRECORDS = 1000 8 | 9 | [FS_SHARE] 10 | SHARE = True 11 | EVERYONE = true 12 | ORG = true 13 | GROUPS = None 14 | 15 | [AGOL] 16 | USER = user_name 17 | PASS = pass_word1 18 | 19 | [PROXY] 20 | USEPROXY = False 21 | SERVER = proxysrvr 22 | PORT = 8888 23 | USER = bandit 24 | PASS = ##### 25 | -------------------------------------------------------------------------------- /settings_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/khibma/update-hosted-feature-service/3bbd6037cbf6f74e2fd1aaabaf6bfeab4b238a7f/settings_image.png -------------------------------------------------------------------------------- /update.py: -------------------------------------------------------------------------------- 1 | # Update.py - update hosted feature services by replacing the .SD file 2 | # and calling publishing (with overwrite) to update the feature service 3 | # 4 | 5 | import ConfigParser 6 | import ast 7 | import os 8 | import sys 9 | import time 10 | 11 | import urllib2 12 | import urllib 13 | import json 14 | import mimetypes 15 | import gzip 16 | from io import BytesIO 17 | import string 18 | import random 19 | 20 | from xml.etree import ElementTree as ET 21 | import arcpy 22 | 23 | 24 | class AGOLHandler(object): 25 | 26 | def __init__(self, username, password, serviceName, folderName, proxyDict): 27 | 28 | self.headers = { 29 | 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 30 | 'User-Agent': ('updatehostedfeatureservice') 31 | } 32 | self.username = username 33 | self.password = password 34 | self.base_url = "https://www.arcgis.com/sharing/rest" 35 | self.proxyDict = proxyDict 36 | self.serviceName = serviceName 37 | self.token = self.getToken(username, password) 38 | self.itemID = self.findItem("Feature Service") 39 | self.SDitemID = self.findItem("Service Definition") 40 | self.folderName = folderName 41 | self.folderID = self.findFolder() 42 | 43 | def getToken(self, username, password, exp=60): 44 | 45 | referer = "http://www.arcgis.com/" 46 | query_dict = {'username': username, 47 | 'password': password, 48 | 'expiration': str(exp), 49 | 'client': 'referer', 50 | 'referer': referer, 51 | 'f': 'json'} 52 | 53 | token_url = '{}/generateToken'.format(self.base_url) 54 | 55 | token_response = self.url_request(token_url, query_dict, 'POST') 56 | 57 | if "token" not in token_response: 58 | print(token_response['error']) 59 | sys.exit() 60 | else: 61 | return token_response['token'] 62 | 63 | def findItem(self, findType): 64 | """ Find the itemID of whats being updated 65 | """ 66 | 67 | searchURL = self.base_url + "/search" 68 | 69 | query_dict = {'f': 'json', 70 | 'token': self.token, 71 | 'q': "title:\"" + self.serviceName + "\"AND owner:\"" + 72 | self.username + "\" AND type:\"" + findType + "\""} 73 | 74 | jsonResponse = self.url_request(searchURL, query_dict, 'POST') 75 | 76 | if jsonResponse['total'] == 0: 77 | print("\nCould not find a service to update. Check the service name in the settings.ini") 78 | sys.exit() 79 | else: 80 | resultList = jsonResponse['results'] 81 | for it in resultList: 82 | if it["title"] == self.serviceName: 83 | print("found {} : {}").format(findType, it["id"]) 84 | return it["id"] 85 | 86 | def findFolder(self, folderName=None): 87 | """ Find the ID of the folder containing the service 88 | """ 89 | 90 | if self.folderName == "None": 91 | return "" 92 | 93 | findURL = "{}/content/users/{}".format(self.base_url, self.username) 94 | 95 | query_dict = {'f': 'json', 96 | 'num': 1, 97 | 'token': self.token} 98 | 99 | jsonResponse = self.url_request(findURL, query_dict, 'POST') 100 | 101 | for folder in jsonResponse['folders']: 102 | if folder['title'] == self.folderName: 103 | return folder['id'] 104 | 105 | print("\nCould not find the specified folder name provided in the settings.ini") 106 | print("-- If your content is in the root folder, change the folder name to 'None'") 107 | sys.exit() 108 | 109 | def upload(self, fileName, tags, description): 110 | """ 111 | Overwrite the SD on AGOL with the new SD. 112 | This method uses 3rd party module: requests 113 | """ 114 | 115 | updateURL = '{}/content/users/{}/{}/items/{}/update'.format(self.base_url, self.username, 116 | self.folderID, self.SDitemID) 117 | 118 | query_dict = {"filename": fileName, 119 | "type": "Service Definition", 120 | "title": self.serviceName, 121 | "tags": tags, 122 | "description": description, 123 | "f": "json", 124 | 'multipart': 'true', 125 | "token": self.token} 126 | 127 | details = {'filename': fileName} 128 | add_item_res = self.url_request(updateURL, query_dict, "POST", "", details) 129 | 130 | itemPartJSON = self._add_part(fileName, add_item_res['id'], "Service Definition") 131 | 132 | if "success" in itemPartJSON: 133 | itemPartID = itemPartJSON['id'] 134 | 135 | commit_response = self.commit(itemPartID) 136 | 137 | # valid states: partial | processing | failed | completed 138 | status = 'processing' 139 | while status == 'processing' or status == 'partial': 140 | status = self.item_status(itemPartID)['status'] 141 | time.sleep(1.5) 142 | 143 | print("updated SD: {}".format(itemPartID)) 144 | return True 145 | 146 | else: 147 | print("\n.sd file not uploaded. Check the errors and try again.\n") 148 | print(itemPartJSON) 149 | sys.exit() 150 | 151 | def _add_part(self, file_to_upload, item_id, upload_type=None): 152 | """ Add the item to the portal in chunks. 153 | """ 154 | 155 | def read_in_chunks(file_object, chunk_size=10000000): 156 | """Generate file chunks of 10MB""" 157 | while True: 158 | data = file_object.read(chunk_size) 159 | if not data: 160 | break 161 | yield data 162 | 163 | url = '{}/content/users/{}/items/{}/addPart'.format(self.base_url, self.username, item_id) 164 | 165 | with open(file_to_upload, 'rb') as f: 166 | for part_num, piece in enumerate(read_in_chunks(f), start=1): 167 | title = os.path.basename(file_to_upload) 168 | files = {"file": {"filename": file_to_upload, "content": piece}} 169 | params = { 170 | 'f': "json", 171 | 'token': self.token, 172 | 'partNum': part_num, 173 | 'title': title, 174 | 'itemType': 'file', 175 | 'type': upload_type 176 | } 177 | 178 | request_data, request_headers = self.multipart_request(params, files) 179 | resp = self.url_request(url, request_data, "MULTIPART", request_headers) 180 | 181 | return resp 182 | 183 | def item_status(self, item_id, jobId=None): 184 | """ Gets the status of an item. 185 | Returns: 186 | The item's status. (partial | processing | failed | completed) 187 | """ 188 | 189 | url = '{}/content/users/{}/items/{}/status'.format(self.base_url, self.username, item_id) 190 | parameters = {'token': self.token, 191 | 'f': 'json'} 192 | 193 | if jobId: 194 | parameters['jobId'] = jobId 195 | 196 | return self.url_request(url, parameters) 197 | 198 | def commit(self, item_id): 199 | """ Commits an item that was uploaded as multipart 200 | """ 201 | 202 | url = '{}/content/users/{}/items/{}/commit'.format(self.base_url, self.username, item_id) 203 | parameters = {'token': self.token, 204 | 'f': 'json'} 205 | 206 | return self.url_request(url, parameters) 207 | 208 | def publish(self): 209 | """ Publish the existing SD on AGOL (it will be turned into a Feature Service) 210 | """ 211 | 212 | publishURL = '{}/content/users/{}/publish'.format(self.base_url, self.username) 213 | 214 | query_dict = {'itemID': self.SDitemID, 215 | 'filetype': 'serviceDefinition', 216 | 'overwrite': 'true', 217 | 'f': 'json', 218 | 'token': self.token} 219 | 220 | jsonResponse = self.url_request(publishURL, query_dict, 'POST') 221 | try: 222 | if 'jobId' in jsonResponse['services'][0]: 223 | jobID = jsonResponse['services'][0]['jobId'] 224 | 225 | # valid states: partial | processing | failed | completed 226 | status = 'processing' 227 | print("Checking the status of publish..") 228 | while status == 'processing' or status == 'partial': 229 | status = self.item_status(self.SDitemID, jobID)['status'] 230 | print(" {}".format(status)) 231 | time.sleep(2) 232 | 233 | if status == 'completed': 234 | print("item finished published") 235 | return jsonResponse['services'][0]['serviceItemId'] 236 | if status == 'failed': 237 | raise("Status of publishing returned FAILED.") 238 | 239 | except Exception as e: 240 | print("Problem trying to check publish status. Might be further errors.") 241 | print("Returned error Python:\n {}".format(e)) 242 | print("Message from publish call:\n {}".format(jsonResponse)) 243 | print(" -- quit --") 244 | sys.exit() 245 | 246 | 247 | def enableSharing(self, newItemID, everyone, orgs, groups): 248 | """ Share an item with everyone, the organization and/or groups 249 | """ 250 | 251 | shareURL = '{}/content/users/{}/{}/items/{}/share'.format(self.base_url, self.username, 252 | self.folderID, newItemID) 253 | 254 | if groups is None: 255 | groups = '' 256 | 257 | query_dict = {'f': 'json', 258 | 'everyone': everyone, 259 | 'org': orgs, 260 | 'groups': groups, 261 | 'token': self.token} 262 | 263 | jsonResponse = self.url_request(shareURL, query_dict, 'POST') 264 | 265 | print("successfully shared...{}...".format(jsonResponse['itemId'])) 266 | 267 | def url_request(self, in_url, request_parameters, request_type='GET', 268 | additional_headers=None, files=None, repeat=0): 269 | """ 270 | Make a request to the portal, provided a portal URL 271 | and request parameters, returns portal response. 272 | 273 | Arguments: 274 | in_url -- portal url 275 | request_parameters -- dictionary of request parameters. 276 | request_type -- HTTP verb (default: GET) 277 | additional_headers -- any headers to pass along with the request. 278 | files -- any files to send. 279 | repeat -- repeat the request up to this number of times. 280 | 281 | Returns: 282 | dictionary of response from portal instance. 283 | """ 284 | 285 | if request_type == 'GET': 286 | req = urllib2.Request('?'.join((in_url, urllib.urlencode(request_parameters)))) 287 | elif request_type == 'MULTIPART': 288 | req = urllib2.Request(in_url, request_parameters) 289 | else: 290 | req = urllib2.Request( 291 | in_url, urllib.urlencode(request_parameters), self.headers) 292 | 293 | if additional_headers: 294 | for key, value in list(additional_headers.items()): 295 | req.add_header(key, value) 296 | req.add_header('Accept-encoding', 'gzip') 297 | 298 | if self.proxyDict: 299 | p = urllib2.ProxyHandler(self.proxyDict) 300 | auth = urllib2.HTTPBasicAuthHandler() 301 | opener = urllib2.build_opener(p, auth, urllib2.HTTPHandler) 302 | urllib2.install_opener(opener) 303 | 304 | response = urllib2.urlopen(req) 305 | 306 | if response.info().get('Content-Encoding') == 'gzip': 307 | buf = BytesIO(response.read()) 308 | with gzip.GzipFile(fileobj=buf) as gzip_file: 309 | response_bytes = gzip_file.read() 310 | else: 311 | response_bytes = response.read() 312 | 313 | response_text = response_bytes.decode('UTF-8') 314 | response_json = json.loads(response_text) 315 | 316 | if not response_json or "error" in response_json: 317 | rerun = False 318 | if repeat > 0: 319 | repeat -= 1 320 | rerun = True 321 | 322 | if rerun: 323 | time.sleep(2) 324 | response_json = self.url_request( 325 | in_url, request_parameters, request_type, 326 | additional_headers, files, repeat) 327 | 328 | return response_json 329 | 330 | def multipart_request(self, params, files): 331 | """ Uploads files as multipart/form-data. files is a dict and must 332 | contain the required keys "filename" and "content". The "mimetype" 333 | value is optional and if not specified will use mimetypes.guess_type 334 | to determine the type or use type application/octet-stream. params 335 | is a dict containing the parameters to be passed in the HTTP 336 | POST request. 337 | 338 | content = open(file_path, "rb").read() 339 | files = {"file": {"filename": "some_file.sd", "content": content}} 340 | params = {"f": "json", "token": token, "type": item_type, 341 | "title": title, "tags": tags, "description": description} 342 | data, headers = multipart_request(params, files) 343 | """ 344 | # Get mix of letters and digits to form boundary. 345 | letters_digits = "".join(string.digits + string.ascii_letters) 346 | boundary = "----WebKitFormBoundary{}".format("".join(random.choice(letters_digits) for i in range(16))) 347 | file_lines = [] 348 | # Parse the params and files dicts to build the multipart request. 349 | for name, value in params.iteritems(): 350 | file_lines.extend(("--{}".format(boundary), 351 | 'Content-Disposition: form-data; name="{}"'.format(name), 352 | "", str(value))) 353 | for name, value in files.items(): 354 | if "filename" in value: 355 | filename = value.get("filename") 356 | else: 357 | raise Exception("The filename key is required.") 358 | if "mimetype" in value: 359 | mimetype = value.get("mimetype") 360 | else: 361 | mimetype = mimetypes.guess_type(filename)[0] or "application/octet-stream" 362 | if "content" in value: 363 | file_lines.extend(("--{}".format(boundary), 364 | 'Content-Disposition: form-data; name="{}"; filename="{}"'.format(name, filename), 365 | "Content-Type: {}".format(mimetype), "", 366 | (value.get("content")))) 367 | else: 368 | raise Exception("The content key is required.") 369 | # Create the end of the form boundary. 370 | file_lines.extend(("--{}--".format(boundary), "")) 371 | 372 | request_data = "\r\n".join(file_lines) 373 | request_headers = {"Content-Type": "multipart/form-data; boundary={}".format(boundary), 374 | "Content-Length": str(len(request_data))} 375 | return request_data, request_headers 376 | 377 | 378 | def makeSD(MXD, serviceName, tempDir, outputSD, maxRecords, tags, summary): 379 | """ create a draft SD and modify the properties to overwrite an existing FS 380 | """ 381 | 382 | arcpy.env.overwriteOutput = True 383 | # All paths are built by joining names to the tempPath 384 | SDdraft = os.path.join(tempDir, "tempdraft.sddraft") 385 | newSDdraft = os.path.join(tempDir, "updatedDraft.sddraft") 386 | 387 | # Check the MXD for summary and tags, if empty, push them in. 388 | try: 389 | mappingMXD = arcpy.mapping.MapDocument(MXD) 390 | if mappingMXD.tags == "": 391 | mappingMXD.tags = tags 392 | mappingMXD.save() 393 | if mappingMXD.summary == "": 394 | mappingMXD.summary = summary 395 | mappingMXD.save() 396 | except IOError: 397 | print("IOError on save, do you have the MXD open? Summary/tag info not pushed to MXD, publishing may fail.") 398 | 399 | arcpy.mapping.CreateMapSDDraft(MXD, SDdraft, serviceName, "MY_HOSTED_SERVICES") 400 | 401 | # Read the contents of the original SDDraft into an xml parser 402 | doc = ET.parse(SDdraft) 403 | 404 | root_elem = doc.getroot() 405 | if root_elem.tag != "SVCManifest": 406 | raise ValueError("Root tag is incorrect. Is {} a .sddraft file?".format(SDDraft)) 407 | 408 | # The following 6 code pieces modify the SDDraft from a new MapService 409 | # with caching capabilities to a FeatureService with Query,Create, 410 | # Update,Delete,Uploads,Editing capabilities as well as the ability 411 | # to set the max records on the service. 412 | # The first two lines (commented out) are no longer necessary as the FS 413 | # is now being deleted and re-published, not truly overwritten as is the 414 | # case when publishing from Desktop. 415 | # The last three pieces change Map to Feature Service, disable caching 416 | # and set appropriate capabilities. You can customize the capabilities by 417 | # removing items. 418 | # Note you cannot disable Query from a Feature Service. 419 | 420 | # doc.find("./Type").text = "esriServiceDefinitionType_Replacement" 421 | # doc.find("./State").text = "esriSDState_Published" 422 | 423 | # Change service type from map service to feature service 424 | for config in doc.findall("./Configurations/SVCConfiguration/TypeName"): 425 | if config.text == "MapServer": 426 | config.text = "FeatureServer" 427 | 428 | # Turn off caching 429 | for prop in doc.findall("./Configurations/SVCConfiguration/Definition/" + 430 | "ConfigurationProperties/PropertyArray/" + 431 | "PropertySetProperty"): 432 | if prop.find("Key").text == 'isCached': 433 | prop.find("Value").text = "false" 434 | if prop.find("Key").text == 'maxRecordCount': 435 | prop.find("Value").text = maxRecords 436 | 437 | # Turn on feature access capabilities 438 | for prop in doc.findall("./Configurations/SVCConfiguration/Definition/Info/PropertyArray/PropertySetProperty"): 439 | if prop.find("Key").text == 'WebCapabilities': 440 | prop.find("Value").text = "Query,Create,Update,Delete,Uploads,Editing" 441 | 442 | # Add the namespaces which get stripped, back into the .SD 443 | root_elem.attrib["xmlns:typens"] = 'http://www.esri.com/schemas/ArcGIS/10.1' 444 | root_elem.attrib["xmlns:xs"] = 'http://www.w3.org/2001/XMLSchema' 445 | 446 | # Write the new draft to disk 447 | with open(newSDdraft, 'w') as f: 448 | doc.write(f, 'utf-8') 449 | 450 | # Analyze the service 451 | analysis = arcpy.mapping.AnalyzeForSD(newSDdraft) 452 | 453 | if analysis['errors'] == {}: 454 | # Stage the service 455 | arcpy.StageService_server(newSDdraft, outputSD) 456 | print("Created {}".format(outputSD)) 457 | 458 | else: 459 | # If the sddraft analysis contained errors, display them and quit. 460 | print("Errors in analyze: \n {}".format(analysis['errors'])) 461 | sys.exit() 462 | 463 | 464 | if __name__ == "__main__": 465 | # 466 | # start 467 | 468 | print("Starting Feature Service publish process") 469 | 470 | # Find and gather settings from the ini file 471 | localPath = sys.path[0] 472 | settingsFile = os.path.join(localPath, "settings.ini") 473 | 474 | if os.path.isfile(settingsFile): 475 | config = ConfigParser.ConfigParser() 476 | config.read(settingsFile) 477 | else: 478 | print("INI file not found. \nMake sure a valid 'settings.ini' file exists in the same directory as this script.") 479 | sys.exit() 480 | 481 | # AGOL Credentials 482 | inputUsername = config.get('AGOL', 'USER') 483 | inputPswd = config.get('AGOL', 'PASS') 484 | 485 | # FS values 486 | MXD = config.get('FS_INFO', 'MXD') 487 | serviceName = config.get('FS_INFO', 'SERVICENAME') 488 | folderName = config.get('FS_INFO', 'FOLDERNAME') 489 | tags = config.get('FS_INFO', 'TAGS') 490 | summary = config.get('FS_INFO', 'DESCRIPTION') 491 | maxRecords = config.get('FS_INFO', 'MAXRECORDS') 492 | 493 | # Share FS to: everyone, org, groups 494 | shared = config.get('FS_SHARE', 'SHARE') 495 | everyone = config.get('FS_SHARE', 'EVERYONE') 496 | orgs = config.get('FS_SHARE', 'ORG') 497 | groups = config.get('FS_SHARE', 'GROUPS') # Groups are by ID. Multiple groups comma separated 498 | 499 | use_prxy = config.get('PROXY', 'USEPROXY') 500 | pxy_srvr = config.get('PROXY', 'SERVER') 501 | pxy_port = config.get('PROXY', 'PORT') 502 | pxy_user = config.get('PROXY', 'USER') 503 | pxy_pass = config.get('PROXY', 'PASS') 504 | 505 | proxyDict = {} 506 | if ast.literal_eval(use_prxy): 507 | http_proxy = "http://" + pxy_user + ":" + pxy_pass + "@" + pxy_srvr + ":" + pxy_port 508 | https_proxy = "http://" + pxy_user + ":" + pxy_pass + "@" + pxy_srvr + ":" + pxy_port 509 | ftp_proxy = "http://" + pxy_user + ":" + pxy_pass + "@" + pxy_srvr + ":" + pxy_port 510 | proxyDict = {"http": http_proxy, "https": https_proxy, "ftp": ftp_proxy} 511 | 512 | # create a temp directory under the script 513 | tempDir = os.path.join(localPath, "tempDir") 514 | if not os.path.isdir(tempDir): 515 | os.mkdir(tempDir) 516 | finalSD = os.path.join(tempDir, serviceName + ".sd") 517 | 518 | # initialize AGOLHandler class 519 | agol = AGOLHandler(inputUsername, inputPswd, serviceName, folderName, proxyDict) 520 | 521 | # Turn map document into .SD file for uploading 522 | makeSD(MXD, serviceName, tempDir, finalSD, maxRecords, tags, summary) 523 | 524 | # overwrite the existing .SD on arcgis.com 525 | if agol.upload(finalSD, tags, summary): 526 | 527 | # publish the sd which was just uploaded 528 | fsID = agol.publish() 529 | 530 | # share the item 531 | if ast.literal_eval(shared): 532 | agol.enableSharing(fsID, everyone, orgs, groups) 533 | 534 | print("\nfinished.") 535 | -------------------------------------------------------------------------------- /update_directory.py: -------------------------------------------------------------------------------- 1 | # update_directory.py - update entire directory of hosted feature services by replacing the .SD file 2 | # and calling publishing (with overwrite) to update the feature service 3 | # 4 | 5 | import ConfigParser 6 | import ast 7 | import os 8 | import sys 9 | import time 10 | 11 | import urllib2 12 | import urllib 13 | import json 14 | import mimetypes 15 | import gzip 16 | from io import BytesIO 17 | import string 18 | import random 19 | 20 | from xml.etree import ElementTree as ET 21 | import arcpy 22 | 23 | #Directory of .ini files - Change to your directory 24 | #Create one .ini file per feature service 25 | directory = r"G:\GIS_Projects\python_arcpy_scripts\FmGis.Lib.ArcPy\updateFS" 26 | 27 | class AGOLHandler(object): 28 | 29 | def __init__(self, username, password, serviceName, folderName, proxyDict): 30 | 31 | self.headers = { 32 | 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 33 | 'User-Agent': ('updatehostedfeatureservice') 34 | } 35 | self.username = username 36 | self.password = password 37 | self.base_url = "https://www.arcgis.com/sharing/rest" 38 | self.proxyDict = proxyDict 39 | self.serviceName = serviceName 40 | self.token = self.getToken(username, password) 41 | self.itemID = self.findItem("Feature Service") 42 | self.SDitemID = self.findItem("Service Definition") 43 | self.folderName = folderName 44 | self.folderID = self.findFolder() 45 | 46 | def getToken(self, username, password, exp=60): 47 | 48 | referer = "http://www.arcgis.com/" 49 | query_dict = {'username': username, 50 | 'password': password, 51 | 'expiration': str(exp), 52 | 'client': 'referer', 53 | 'referer': referer, 54 | 'f': 'json'} 55 | 56 | token_url = '{}/generateToken'.format(self.base_url) 57 | 58 | token_response = self.url_request(token_url, query_dict, 'POST') 59 | 60 | if "token" not in token_response: 61 | print(token_response['error']) 62 | sys.exit() 63 | else: 64 | return token_response['token'] 65 | 66 | def findItem(self, findType): 67 | """ Find the itemID of whats being updated 68 | """ 69 | 70 | searchURL = self.base_url + "/search" 71 | 72 | query_dict = {'f': 'json', 73 | 'token': self.token, 74 | 'q': "title:\"" + self.serviceName + "\"AND owner:\"" + 75 | self.username + "\" AND type:\"" + findType + "\""} 76 | 77 | jsonResponse = self.url_request(searchURL, query_dict, 'POST') 78 | 79 | if jsonResponse['total'] == 0: 80 | print("\nCould not find a service to update. Check the service name in the settings.ini") 81 | sys.exit() 82 | else: 83 | resultList = jsonResponse['results'] 84 | for it in resultList: 85 | if it["title"] == self.serviceName: 86 | print("found {} : {}").format(findType, it["id"]) 87 | return it["id"] 88 | 89 | def findFolder(self, folderName=None): 90 | """ Find the ID of the folder containing the service 91 | """ 92 | 93 | if self.folderName == "None": 94 | return "" 95 | 96 | findURL = "{}/content/users/{}".format(self.base_url, self.username) 97 | 98 | query_dict = {'f': 'json', 99 | 'num': 1, 100 | 'token': self.token} 101 | 102 | jsonResponse = self.url_request(findURL, query_dict, 'POST') 103 | 104 | for folder in jsonResponse['folders']: 105 | if folder['title'] == self.folderName: 106 | return folder['id'] 107 | 108 | print("\nCould not find the specified folder name provided in the settings.ini") 109 | print("-- If your content is in the root folder, change the folder name to 'None'") 110 | sys.exit() 111 | 112 | def upload(self, fileName, tags, description): 113 | """ 114 | Overwrite the SD on AGOL with the new SD. 115 | This method uses 3rd party module: requests 116 | """ 117 | 118 | updateURL = '{}/content/users/{}/{}/items/{}/update'.format(self.base_url, self.username, 119 | self.folderID, self.SDitemID) 120 | 121 | query_dict = {"filename": fileName, 122 | "type": "Service Definition", 123 | "title": self.serviceName, 124 | "tags": tags, 125 | "description": description, 126 | "f": "json", 127 | 'multipart': 'true', 128 | "token": self.token} 129 | 130 | details = {'filename': fileName} 131 | add_item_res = self.url_request(updateURL, query_dict, "POST", "", details) 132 | 133 | itemPartJSON = self._add_part(fileName, add_item_res['id'], "Service Definition") 134 | 135 | if "success" in itemPartJSON: 136 | itemPartID = itemPartJSON['id'] 137 | 138 | commit_response = self.commit(itemPartID) 139 | 140 | # valid states: partial | processing | failed | completed 141 | status = 'processing' 142 | while status == 'processing' or status == 'partial': 143 | status = self.item_status(itemPartID)['status'] 144 | time.sleep(1.5) 145 | 146 | print("updated SD: {}".format(itemPartID)) 147 | return True 148 | 149 | else: 150 | print("\n.sd file not uploaded. Check the errors and try again.\n") 151 | print(itemPartJSON) 152 | sys.exit() 153 | 154 | def _add_part(self, file_to_upload, item_id, upload_type=None): 155 | """ Add the item to the portal in chunks. 156 | """ 157 | 158 | def read_in_chunks(file_object, chunk_size=10000000): 159 | """Generate file chunks of 10MB""" 160 | while True: 161 | data = file_object.read(chunk_size) 162 | if not data: 163 | break 164 | yield data 165 | 166 | url = '{}/content/users/{}/items/{}/addPart'.format(self.base_url, self.username, item_id) 167 | 168 | with open(file_to_upload, 'rb') as f: 169 | for part_num, piece in enumerate(read_in_chunks(f), start=1): 170 | title = os.path.basename(file_to_upload) 171 | files = {"file": {"filename": file_to_upload, "content": piece}} 172 | params = { 173 | 'f': "json", 174 | 'token': self.token, 175 | 'partNum': part_num, 176 | 'title': title, 177 | 'itemType': 'file', 178 | 'type': upload_type 179 | } 180 | 181 | request_data, request_headers = self.multipart_request(params, files) 182 | resp = self.url_request(url, request_data, "MULTIPART", request_headers) 183 | 184 | return resp 185 | 186 | def item_status(self, item_id, jobId=None): 187 | """ Gets the status of an item. 188 | Returns: 189 | The item's status. (partial | processing | failed | completed) 190 | """ 191 | 192 | url = '{}/content/users/{}/items/{}/status'.format(self.base_url, self.username, item_id) 193 | parameters = {'token': self.token, 194 | 'f': 'json'} 195 | 196 | if jobId: 197 | parameters['jobId'] = jobId 198 | 199 | return self.url_request(url, parameters) 200 | 201 | def commit(self, item_id): 202 | """ Commits an item that was uploaded as multipart 203 | """ 204 | 205 | url = '{}/content/users/{}/items/{}/commit'.format(self.base_url, self.username, item_id) 206 | parameters = {'token': self.token, 207 | 'f': 'json'} 208 | 209 | return self.url_request(url, parameters) 210 | 211 | def publish(self): 212 | """ Publish the existing SD on AGOL (it will be turned into a Feature Service) 213 | """ 214 | 215 | publishURL = '{}/content/users/{}/publish'.format(self.base_url, self.username) 216 | 217 | query_dict = {'itemID': self.SDitemID, 218 | 'filetype': 'serviceDefinition', 219 | 'overwrite': 'true', 220 | 'f': 'json', 221 | 'token': self.token} 222 | 223 | jsonResponse = self.url_request(publishURL, query_dict, 'POST') 224 | try: 225 | if 'jobId' in jsonResponse['services'][0]: 226 | jobID = jsonResponse['services'][0]['jobId'] 227 | 228 | # valid states: partial | processing | failed | completed 229 | status = 'processing' 230 | print("Checking the status of publish..") 231 | while status == 'processing' or status == 'partial': 232 | status = self.item_status(self.SDitemID, jobID)['status'] 233 | print(" {}".format(status)) 234 | time.sleep(2) 235 | 236 | if status == 'completed': 237 | print("item finished published") 238 | return jsonResponse['services'][0]['serviceItemId'] 239 | if status == 'failed': 240 | raise("Status of publishing returned FAILED.") 241 | 242 | except Exception as e: 243 | print("Problem trying to check publish status. Might be further errors.") 244 | print("Returned error Python:\n {}".format(e)) 245 | print("Message from publish call:\n {}".format(jsonResponse)) 246 | print(" -- quit --") 247 | sys.exit() 248 | 249 | 250 | def enableSharing(self, newItemID, everyone, orgs, groups): 251 | """ Share an item with everyone, the organization and/or groups 252 | """ 253 | 254 | shareURL = '{}/content/users/{}/{}/items/{}/share'.format(self.base_url, self.username, 255 | self.folderID, newItemID) 256 | 257 | if groups is None: 258 | groups = '' 259 | 260 | query_dict = {'f': 'json', 261 | 'everyone': everyone, 262 | 'org': orgs, 263 | 'groups': groups, 264 | 'token': self.token} 265 | 266 | jsonResponse = self.url_request(shareURL, query_dict, 'POST') 267 | 268 | print("successfully shared...{}...".format(jsonResponse['itemId'])) 269 | 270 | def url_request(self, in_url, request_parameters, request_type='GET', 271 | additional_headers=None, files=None, repeat=0): 272 | """ 273 | Make a request to the portal, provided a portal URL 274 | and request parameters, returns portal response. 275 | 276 | Arguments: 277 | in_url -- portal url 278 | request_parameters -- dictionary of request parameters. 279 | request_type -- HTTP verb (default: GET) 280 | additional_headers -- any headers to pass along with the request. 281 | files -- any files to send. 282 | repeat -- repeat the request up to this number of times. 283 | 284 | Returns: 285 | dictionary of response from portal instance. 286 | """ 287 | 288 | if request_type == 'GET': 289 | req = urllib2.Request('?'.join((in_url, urllib.urlencode(request_parameters)))) 290 | elif request_type == 'MULTIPART': 291 | req = urllib2.Request(in_url, request_parameters) 292 | else: 293 | req = urllib2.Request( 294 | in_url, urllib.urlencode(request_parameters), self.headers) 295 | 296 | if additional_headers: 297 | for key, value in list(additional_headers.items()): 298 | req.add_header(key, value) 299 | req.add_header('Accept-encoding', 'gzip') 300 | 301 | if self.proxyDict: 302 | p = urllib2.ProxyHandler(self.proxyDict) 303 | auth = urllib2.HTTPBasicAuthHandler() 304 | opener = urllib2.build_opener(p, auth, urllib2.HTTPHandler) 305 | urllib2.install_opener(opener) 306 | 307 | response = urllib2.urlopen(req) 308 | 309 | if response.info().get('Content-Encoding') == 'gzip': 310 | buf = BytesIO(response.read()) 311 | with gzip.GzipFile(fileobj=buf) as gzip_file: 312 | response_bytes = gzip_file.read() 313 | else: 314 | response_bytes = response.read() 315 | 316 | response_text = response_bytes.decode('UTF-8') 317 | response_json = json.loads(response_text) 318 | 319 | if not response_json or "error" in response_json: 320 | rerun = False 321 | if repeat > 0: 322 | repeat -= 1 323 | rerun = True 324 | 325 | if rerun: 326 | time.sleep(2) 327 | response_json = self.url_request( 328 | in_url, request_parameters, request_type, 329 | additional_headers, files, repeat) 330 | 331 | return response_json 332 | 333 | def multipart_request(self, params, files): 334 | """ Uploads files as multipart/form-data. files is a dict and must 335 | contain the required keys "filename" and "content". The "mimetype" 336 | value is optional and if not specified will use mimetypes.guess_type 337 | to determine the type or use type application/octet-stream. params 338 | is a dict containing the parameters to be passed in the HTTP 339 | POST request. 340 | 341 | content = open(file_path, "rb").read() 342 | files = {"file": {"filename": "some_file.sd", "content": content}} 343 | params = {"f": "json", "token": token, "type": item_type, 344 | "title": title, "tags": tags, "description": description} 345 | data, headers = multipart_request(params, files) 346 | """ 347 | # Get mix of letters and digits to form boundary. 348 | letters_digits = "".join(string.digits + string.ascii_letters) 349 | boundary = "----WebKitFormBoundary{}".format("".join(random.choice(letters_digits) for i in range(16))) 350 | file_lines = [] 351 | # Parse the params and files dicts to build the multipart request. 352 | for name, value in params.iteritems(): 353 | file_lines.extend(("--{}".format(boundary), 354 | 'Content-Disposition: form-data; name="{}"'.format(name), 355 | "", str(value))) 356 | for name, value in files.items(): 357 | if "filename" in value: 358 | filename = value.get("filename") 359 | else: 360 | raise Exception("The filename key is required.") 361 | if "mimetype" in value: 362 | mimetype = value.get("mimetype") 363 | else: 364 | mimetype = mimetypes.guess_type(filename)[0] or "application/octet-stream" 365 | if "content" in value: 366 | file_lines.extend(("--{}".format(boundary), 367 | 'Content-Disposition: form-data; name="{}"; filename="{}"'.format(name, filename), 368 | "Content-Type: {}".format(mimetype), "", 369 | (value.get("content")))) 370 | else: 371 | raise Exception("The content key is required.") 372 | # Create the end of the form boundary. 373 | file_lines.extend(("--{}--".format(boundary), "")) 374 | 375 | request_data = "\r\n".join(file_lines) 376 | request_headers = {"Content-Type": "multipart/form-data; boundary={}".format(boundary), 377 | "Content-Length": str(len(request_data))} 378 | return request_data, request_headers 379 | 380 | 381 | def makeSD(MXD, serviceName, tempDir, outputSD, maxRecords, tags, summary): 382 | """ create a draft SD and modify the properties to overwrite an existing FS 383 | """ 384 | 385 | arcpy.env.overwriteOutput = True 386 | # All paths are built by joining names to the tempPath 387 | SDdraft = os.path.join(tempDir, "tempdraft.sddraft") 388 | newSDdraft = os.path.join(tempDir, "updatedDraft.sddraft") 389 | 390 | # Check the MXD for summary and tags, if empty, push them in. 391 | try: 392 | mappingMXD = arcpy.mapping.MapDocument(MXD) 393 | if mappingMXD.tags == "": 394 | mappingMXD.tags = tags 395 | mappingMXD.save() 396 | if mappingMXD.summary == "": 397 | mappingMXD.summary = summary 398 | mappingMXD.save() 399 | except IOError: 400 | print("IOError on save, do you have the MXD open? Summary/tag info not pushed to MXD, publishing may fail.") 401 | 402 | arcpy.mapping.CreateMapSDDraft(MXD, SDdraft, serviceName, "MY_HOSTED_SERVICES") 403 | 404 | # Read the contents of the original SDDraft into an xml parser 405 | doc = ET.parse(SDdraft) 406 | 407 | root_elem = doc.getroot() 408 | if root_elem.tag != "SVCManifest": 409 | raise ValueError("Root tag is incorrect. Is {} a .sddraft file?".format(SDDraft)) 410 | 411 | # The following 6 code pieces modify the SDDraft from a new MapService 412 | # with caching capabilities to a FeatureService with Query,Create, 413 | # Update,Delete,Uploads,Editing capabilities as well as the ability 414 | # to set the max records on the service. 415 | # The first two lines (commented out) are no longer necessary as the FS 416 | # is now being deleted and re-published, not truly overwritten as is the 417 | # case when publishing from Desktop. 418 | # The last three pieces change Map to Feature Service, disable caching 419 | # and set appropriate capabilities. You can customize the capabilities by 420 | # removing items. 421 | # Note you cannot disable Query from a Feature Service. 422 | 423 | # doc.find("./Type").text = "esriServiceDefinitionType_Replacement" 424 | # doc.find("./State").text = "esriSDState_Published" 425 | 426 | # Change service type from map service to feature service 427 | for config in doc.findall("./Configurations/SVCConfiguration/TypeName"): 428 | if config.text == "MapServer": 429 | config.text = "FeatureServer" 430 | 431 | # Turn off caching 432 | for prop in doc.findall("./Configurations/SVCConfiguration/Definition/" + 433 | "ConfigurationProperties/PropertyArray/" + 434 | "PropertySetProperty"): 435 | if prop.find("Key").text == 'isCached': 436 | prop.find("Value").text = "false" 437 | if prop.find("Key").text == 'maxRecordCount': 438 | prop.find("Value").text = maxRecords 439 | 440 | # Turn on feature access capabilities 441 | for prop in doc.findall("./Configurations/SVCConfiguration/Definition/Info/PropertyArray/PropertySetProperty"): 442 | if prop.find("Key").text == 'WebCapabilities': 443 | prop.find("Value").text = "Query,Create,Update,Delete,Uploads,Editing" 444 | 445 | # Add the namespaces which get stripped, back into the .SD 446 | root_elem.attrib["xmlns:typens"] = 'http://www.esri.com/schemas/ArcGIS/10.1' 447 | root_elem.attrib["xmlns:xs"] = 'http://www.w3.org/2001/XMLSchema' 448 | 449 | # Write the new draft to disk 450 | with open(newSDdraft, 'w') as f: 451 | doc.write(f, 'utf-8') 452 | 453 | # Analyze the service 454 | analysis = arcpy.mapping.AnalyzeForSD(newSDdraft) 455 | 456 | if analysis['errors'] == {}: 457 | # Stage the service 458 | arcpy.StageService_server(newSDdraft, outputSD) 459 | print("Created {}".format(outputSD)) 460 | 461 | else: 462 | # If the sddraft analysis contained errors, display them and quit. 463 | print("Errors in analyze: \n {}".format(analysis['errors'])) 464 | sys.exit() 465 | 466 | 467 | for file in os.listdir(directory): 468 | if file.endswith(".ini") and __name__ == "__main__": 469 | 470 | print("Now exporting " + file) 471 | setfile = os.path.join(directory, file) 472 | 473 | print("Starting Feature Service publish process") 474 | 475 | # Find and gather settings from the ini file 476 | localPath = sys.path[0] 477 | settingsFile = setfile 478 | 479 | if os.path.isfile(settingsFile): 480 | config = ConfigParser.ConfigParser() 481 | config.read(settingsFile) 482 | else: 483 | print("INI file not found. \nMake sure a valid 'settings.ini' file exists in the same directory as this script.") 484 | sys.exit() 485 | 486 | # AGOL Credentials 487 | inputUsername = config.get('AGOL', 'USER') 488 | inputPswd = config.get('AGOL', 'PASS') 489 | 490 | # FS values 491 | MXD = config.get('FS_INFO', 'MXD') 492 | serviceName = config.get('FS_INFO', 'SERVICENAME') 493 | folderName = config.get('FS_INFO', 'FOLDERNAME') 494 | tags = config.get('FS_INFO', 'TAGS') 495 | summary = config.get('FS_INFO', 'DESCRIPTION') 496 | maxRecords = config.get('FS_INFO', 'MAXRECORDS') 497 | 498 | # Share FS to: everyone, org, groups 499 | shared = config.get('FS_SHARE', 'SHARE') 500 | everyone = config.get('FS_SHARE', 'EVERYONE') 501 | orgs = config.get('FS_SHARE', 'ORG') 502 | groups = config.get('FS_SHARE', 'GROUPS') # Groups are by ID. Multiple groups comma separated 503 | 504 | use_prxy = config.get('PROXY', 'USEPROXY') 505 | pxy_srvr = config.get('PROXY', 'SERVER') 506 | pxy_port = config.get('PROXY', 'PORT') 507 | pxy_user = config.get('PROXY', 'USER') 508 | pxy_pass = config.get('PROXY', 'PASS') 509 | 510 | proxyDict = {} 511 | if ast.literal_eval(use_prxy): 512 | http_proxy = "http://" + pxy_user + ":" + pxy_pass + "@" + pxy_srvr + ":" + pxy_port 513 | https_proxy = "http://" + pxy_user + ":" + pxy_pass + "@" + pxy_srvr + ":" + pxy_port 514 | ftp_proxy = "http://" + pxy_user + ":" + pxy_pass + "@" + pxy_srvr + ":" + pxy_port 515 | proxyDict = {"http": http_proxy, "https": https_proxy, "ftp": ftp_proxy} 516 | 517 | # create a temp directory under the script 518 | tempDir = os.path.join(localPath, "tempDir") 519 | if not os.path.isdir(tempDir): 520 | os.mkdir(tempDir) 521 | finalSD = os.path.join(tempDir, serviceName + ".sd") 522 | 523 | # initialize AGOLHandler class 524 | agol = AGOLHandler(inputUsername, inputPswd, serviceName, folderName, proxyDict) 525 | 526 | # Turn map document into .SD file for uploading 527 | makeSD(MXD, serviceName, tempDir, finalSD, maxRecords, tags, summary) 528 | 529 | # overwrite the existing .SD on arcgis.com 530 | if agol.upload(finalSD, tags, summary): 531 | 532 | # publish the sd which was just uploaded 533 | fsID = agol.publish() 534 | 535 | # share the item 536 | if ast.literal_eval(shared): 537 | agol.enableSharing(fsID, everyone, orgs, groups) 538 | 539 | print("\nfinished.") 540 | 541 | print("Congratulations! All feature services in " + directory + " have been updated.") --------------------------------------------------------------------------------