├── .env
├── package.json
├── index.js
├── LICENSE
├── .gitignore
├── README.md
├── ocapi-calls.js
├── webdav-calls.js
└── form-file-util.js
/.env:
--------------------------------------------------------------------------------
1 | BASE_URL=https://dev00-store-X.demandware.net
2 | CLIENT_ID=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
3 | CLIENT_KEY=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
4 | BM_USER=the.user
5 | BM_PASS=the$Pass
6 | SITE_ID=The-Site
7 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "sfcc-form-fields-to-csv",
3 | "version": "0.0.1",
4 | "description": "A tool to collect sites form field info from XML files and export them in one CSV file. While gathering info considers cartridge path and site locales.",
5 | "main": "index.js",
6 | "scripts": {
7 | "test": "echo \"Error: no test specified\" && exit 1"
8 | },
9 | "author": "Serdar Büyüktemiz",
10 | "license": "MIT",
11 | "dependencies": {
12 | "dotenv": "^6.2.0",
13 | "fs": "0.0.1-security",
14 | "request": "^2.87.0",
15 | "request-promise": "^4.2.2",
16 | "webdav": "^2.2.0",
17 | "xml2js": "^0.4.19"
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/index.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | const { load } = require('dotenv');
4 |
5 | const ocapi = require('./ocapi-calls');
6 | const webdav = require('./webdav-calls');
7 | const formFileUtil = require('./form-file-util');
8 |
9 | load();
10 |
11 | (async ({ BASE_URL, BM_USER, BM_PASS, CLIENT_ID, CLIENT_KEY, SITE_ID }) => {
12 | try {
13 | const token = await ocapi.getToken(BASE_URL, BM_USER, BM_PASS, CLIENT_ID, CLIENT_KEY);
14 | const activeCodeVersion = await ocapi.getActiveCodeVersion(BASE_URL, token);
15 | const cartridges = await ocapi.getSiteCartridges(BASE_URL, SITE_ID, token);
16 | const locales = await ocapi.getSiteLocales(BASE_URL, SITE_ID, token);
17 |
18 | await webdav.getFormXMLsOfSiteCartridges(BASE_URL, BM_USER, BM_PASS, activeCodeVersion, cartridges, locales);
19 | await formFileUtil.processFiles();
20 |
21 | console.log('\r\ncompleted!\r\n');
22 | }
23 | catch(err) {
24 | console.error(err);
25 | process.exit(1);
26 | }
27 | })(dotenv.config().parsed);
28 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2019 Serdar Büyüktemiz
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .env
2 | files/
3 |
4 | # Logs
5 | logs
6 | *.log
7 | npm-debug.log*
8 | yarn-debug.log*
9 | yarn-error.log*
10 |
11 | # Runtime data
12 | pids
13 | *.pid
14 | *.seed
15 | *.pid.lock
16 |
17 | # Directory for instrumented libs generated by jscoverage/JSCover
18 | lib-cov
19 |
20 | # Coverage directory used by tools like istanbul
21 | coverage
22 |
23 | # nyc test coverage
24 | .nyc_output
25 |
26 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
27 | .grunt
28 |
29 | # Bower dependency directory (https://bower.io/)
30 | bower_components
31 |
32 | # node-waf configuration
33 | .lock-wscript
34 |
35 | # Compiled binary addons (https://nodejs.org/api/addons.html)
36 | build/Release
37 |
38 | # Dependency directories
39 | node_modules/
40 | jspm_packages/
41 |
42 | # TypeScript v1 declaration files
43 | typings/
44 |
45 | # Optional npm cache directory
46 | .npm
47 |
48 | # Optional eslint cache
49 | .eslintcache
50 |
51 | # Optional REPL history
52 | .node_repl_history
53 |
54 | # Output of 'npm pack'
55 | *.tgz
56 |
57 | # Yarn Integrity file
58 | .yarn-integrity
59 |
60 | # dotenv environment variables file
61 | .env
62 |
63 | # next.js build output
64 | .next
65 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # sfcc-form-fields-to-csv
2 | A tool to collect sites form field info from XML files and export them in one CSV file. While gathering info considers cartridge path and site locales.
3 |
4 | *Please set the settings in the settings in .env file before running the tool.*
5 |
6 | ### OCAPI Roles & Permissions
7 |
8 |
9 | {
10 | "client_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
11 | "resources": [{
12 | "resource_id": "/code_versions",
13 | "methods": ["get"],
14 | "read_attributes": "(**)",
15 | "write_attributes": "(**)"
16 | },
17 | {
18 | "resource_id": "/sites/**",
19 | "methods": ["get"],
20 | "read_attributes": "(**)",
21 | "write_attributes": "(**)"
22 | }]
23 | }
24 |
25 |
26 | ### How to run this tool?
27 |
28 | you should have git, node and npm installed so that you can run the tool with these commands.
29 |
30 |
31 | git clone https://github.com/serdarb/sfcc-form-fields-to-csv.git
32 | cd .\sfcc-field-info-export\
33 | npm install
34 | node .\index.js
35 |
36 |
37 |
--------------------------------------------------------------------------------
/ocapi-calls.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | const request = require('request-promise');
4 |
5 | const getToken = async (baseURL, bmUser, bmPass, clientId, clientKey) => {
6 |
7 | let response = await request.post(`${baseURL}/dw/oauth2/access_token`, {
8 | headers: {
9 | 'Authorization': 'Basic ' + Buffer.from(`${bmUser}:${bmPass}:${clientKey}`).toString('base64'),
10 | 'Content-Type': 'application/x-www-form-urlencoded'
11 | },
12 | form: { 'grant_type': 'urn:demandware:params:oauth:grant-type:client-id:dwsid:dwsecuretoken' },
13 | qs: { 'client_id': clientId },
14 | json: true
15 | });
16 |
17 | let access_token = response.access_token;
18 | if(!access_token) {
19 | throw new Error('(getToken) access_token is empty, response: ' + JSON.stringify(response));
20 | }
21 | else {
22 | return access_token;
23 | }
24 | };
25 |
26 | const getActiveCodeVersion = async (baseURL, token) => {
27 |
28 | let response = await request.get(`${baseURL}/s/-/dw/data/v19_1/code_versions`, {
29 | headers: {
30 | 'Authorization': 'Bearer ' + token,
31 | 'Content-Type': 'application/json'
32 | },
33 | json: true
34 | });
35 |
36 | let activeCodeVersion = response.data.filter(x => x.active)[0].id;
37 | if(!activeCodeVersion) {
38 | throw new Error('(getActiveCodeVersion) activeCodeVersion is empty, response: ' + JSON.stringify(response));
39 | }
40 | else {
41 | return activeCodeVersion;
42 | }
43 | };
44 |
45 | const getSiteCartridges = async (baseURL, siteId, token) => {
46 |
47 | let response = await request.get(`${baseURL}/s/-/dw/data/v19_1/sites/${siteId}`, {
48 | headers: {
49 | 'Authorization': 'Bearer ' + token,
50 | 'Content-Type': 'application/json'
51 | },
52 | json: true
53 | });
54 |
55 | let cartridges = response.cartridges.split(':');
56 | if(!cartridges) {
57 | throw new Error('(getSiteCartridges) cartridges is empty, response: ' + JSON.stringify(response));
58 | }
59 | else {
60 | return cartridges;
61 | }
62 | };
63 |
64 | const getSiteLocales = async (baseURL, siteId, token) => {
65 |
66 | let response = await request.get(`${baseURL}/s/-/dw/data/v19_1/sites/${siteId}/locale_info/locales`, {
67 | headers: {
68 | 'Authorization': 'Bearer ' + token,
69 | 'Content-Type': 'application/json'
70 | },
71 | json: true
72 | });
73 |
74 | let locales = response.hits;
75 | if(!locales) {
76 | throw new Error('(getSiteLocales) locales is empty, response: ' + JSON.stringify(response));
77 | }
78 | else {
79 |
80 | var result = [];
81 |
82 | locales.forEach(locale => {
83 | var idInfo = locale.id.split('-');
84 | var language = idInfo[0];
85 | var country = idInfo[1];
86 | result.push({'language':language, 'country':country});
87 | });
88 |
89 | return result;
90 | }
91 | };
92 |
93 | module.exports = {
94 | getToken : getToken,
95 | getActiveCodeVersion : getActiveCodeVersion,
96 | getSiteCartridges : getSiteCartridges,
97 | getSiteLocales : getSiteLocales
98 | };
99 |
--------------------------------------------------------------------------------
/webdav-calls.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | const { createClient } = require('webdav');
4 | const fs = require('fs');
5 |
6 | const getFormFilesIfPathExists = async (client, path, locales, foundFormXMlFileNames, localeFolderName) => {
7 | let foundFiles = [];
8 |
9 | for (let j = 0, jj = locales.length; j !== jj; j++) {
10 | try {
11 | let contents = await client.getDirectoryContents(path + '/' + locales[j]);
12 |
13 | for (let i = 0, ii = contents.length; i !== ii; i++) {
14 | if (contents[i].filename.indexOf('xml') !== -1) {
15 | let filePath = contents[i].filename;
16 | let fileName = filePath.substring(filePath.lastIndexOf('/') + 1);
17 |
18 | if (!foundFormXMlFileNames.includes(fileName)) {
19 | foundFiles.push(filePath);
20 | foundFormXMlFileNames.push(fileName);
21 |
22 | fs.appendFileSync(localeFolderName + '\\fullpaths.txt', fileName + ',' + path + '/' + locales[j] + '/' + fileName + '\r\n');
23 | }
24 | } else {
25 | let val = await getFormFilesIfPathExists(client, contents[i].filename, foundFormXMlFileNames, localeFolderName);
26 | foundFiles = foundFiles.concat(val);
27 | }
28 | }
29 | } catch (e) {
30 | if (!e.response)
31 | {
32 | console.log(e);
33 | }
34 | else if (e.response.status !== 404) {
35 | console.log(path + '\r\n' + e + '\r\n');
36 | }
37 | }
38 | }
39 |
40 | return foundFiles;
41 | }
42 |
43 | const getFormXMLsOfSiteCartridges = async (baseURL, bmUser, bmPass, activeCodeVersion, cartridges, locales) => {
44 | console.log('starting to get form xmls\r\n');
45 |
46 | let client = createClient(baseURL + '/on/demandware.servlet/webdav/Sites/Cartridges/', { username: bmUser, password: bmPass });
47 |
48 | let preparedLocales = locales.map(locale => [locale.language + '_' + locale.country, locale.language, 'default']);
49 | let paths = cartridges.map(cartridge => '/' + activeCodeVersion + '/' + cartridge + '/cartridge/forms');
50 |
51 | for (let j = 0, jj = preparedLocales.length; j !== jj; j++) {
52 | let foundFromXMLFilePaths = [];
53 | let foundFormXMlFileNames = [];
54 |
55 | var localeName = preparedLocales[j][1];
56 | var localeFolderName = __dirname + '\\files\\' + localeName;
57 | if (!fs.existsSync(localeFolderName)) {
58 | fs.mkdirSync(localeFolderName, { recursive: true }, (r) => { if (r != null) { console.log(r); } });
59 | }
60 |
61 | for (let i = 0, ii = paths.length; i !== ii; i++) {
62 | foundFromXMLFilePaths = foundFromXMLFilePaths.concat(await getFormFilesIfPathExists(client, paths[i], preparedLocales[j], foundFormXMlFileNames, localeFolderName));
63 | }
64 |
65 | console.log('\r\n' + foundFromXMLFilePaths.length + ' form found for ' + localeName + ' locale\r\n');
66 |
67 | for (let i = 0, ii = foundFromXMLFilePaths.length; i !== ii; i++) {
68 | var formXMLContent = await client.getFileContents(foundFromXMLFilePaths[i], { format: 'text' });
69 | let fileName = foundFromXMLFilePaths[i].substring(foundFromXMLFilePaths[i].lastIndexOf('/') + 1);
70 |
71 | await fs.writeFile(localeFolderName + '\\' + fileName, formXMLContent, (r) => { if (r != null) { console.log(r); } });
72 | }
73 | }
74 | };
75 |
76 | module.exports = {
77 | getFormXMLsOfSiteCartridges: getFormXMLsOfSiteCartridges
78 | };
79 |
--------------------------------------------------------------------------------
/form-file-util.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | const fs = require('fs');
4 | const xmlParser = require('xml2js').parseString;
5 |
6 | const filesFolder = __dirname + '\\files';
7 |
8 | const processFiles = () => {
9 |
10 | fs.readdir(filesFolder, (err, localeFolders) => {
11 | if (localeFolders == null) { return; }
12 |
13 | for (var i = 0; i < localeFolders.length; i++) {
14 | let localeFolder = localeFolders[i];
15 |
16 | let localeFolderPath = filesFolder + '\\' + localeFolder;
17 | fs.readdir(localeFolderPath, (err, fileNames) => {
18 | if (fileNames == null) { return; }
19 |
20 | let fullPathFileArray = fs.readFileSync(localeFolderPath + '\\fullpaths.txt').toString().split("\r\n");
21 | let fullPaths = fullPathFileArray.map(p => { var items = p.split(',');
22 | return { 'key': items[0], 'value': items[1] }; });
23 |
24 | for (let j = 0; j < fileNames.length; j++) {
25 | let fileName = fileNames[j];
26 | if(!fileName.endsWith('xml')) {
27 | continue;
28 | }
29 |
30 | let fileNamePretty = fileName.replace('.xml', '');
31 | let fullPath = fullPaths.filter(x => x.key === fileName)[0].value;
32 |
33 | fs.readFile(localeFolderPath + '\\' + fileName, 'utf8', (err, xmlContent) => {
34 | xmlParser(xmlContent, (err, xmlObject) => {
35 |
36 | let fields = xmlObject.form.field;
37 | if (fields) {
38 | fields.forEach(fld => { getFieldInfo(fullPath, localeFolder, fileNamePretty, fld); });
39 | }
40 |
41 | let groups = xmlObject.form.group;
42 | if (groups) {
43 | groups.forEach(group => { getGroupFields(fullPath, localeFolder, fileNamePretty, group); });
44 | }
45 |
46 | let lists = xmlObject.form.list;
47 | if (lists) {
48 | lists.forEach(list => { getListFields(fullPath, localeFolder, fileNamePretty, list); });
49 | }
50 | });
51 | });
52 | }
53 | });
54 | }
55 | });
56 | }
57 |
58 | const getGroupFields = async (path, locale, fileName, group, prefix) => {
59 | let groupFields = group.field;
60 | if (!groupFields) { return; }
61 |
62 | prefix = prefix != null || prefix !== undefined ? prefix + '.' + group.$['formid'] : group.$['formid'];
63 |
64 | groupFields.forEach(groupField => { getFieldInfo(path, locale, fileName, groupField, prefix); });
65 |
66 | let groupGroups = group.group;
67 | if (groupGroups) {
68 | groupGroups.forEach(grp => { getGroupFields(path, locale, fileName, grp, prefix); });
69 | }
70 | }
71 |
72 | const getListFields = async (path, locale, fileName, list, prefix) => {
73 | let listFields = list.field;
74 | if (!listFields) { return; }
75 |
76 | prefix = prefix != null || prefix !== undefined ? prefix + '.' + list.$['formid'] : list.$['formid'];
77 |
78 | listFields.forEach(listField => { getFieldInfo(path, locale, fileName, listField, prefix); });
79 |
80 | let listLists = list.list;
81 | if (listLists) {
82 | listLists.forEach(lst => { getListFields(path, locale, fileName, lst, prefix); });
83 | }
84 |
85 | let listGroups = list.group;
86 | if (listGroups) {
87 | listGroups.forEach(grp => { getGroupFields(path, locale, fileName, grp, prefix); });
88 | }
89 | }
90 |
91 | const getFieldInfo = async (path, locale, fileName, field, prefix) => {
92 | let info = {
93 | 'path': path,
94 | 'locale': locale,
95 | 'fileName': fileName,
96 | 'name': prefix != null ? prefix + '.' + field.$['formid'] : field.$['formid'],
97 | 'mandatory': field.$['mandatory'] || '',
98 | 'regexp': field.$['regexp'] || '',
99 | 'max-length': field.$['max-length'] || '',
100 | 'min-length': field.$['min-length'] || ''
101 | };
102 | //let infoObjectString = JSON.stringify(info);
103 |
104 | fs.appendFileSync(filesFolder + '\\export-' + locale + '.csv',
105 | info.fileName + ',' +
106 | info.name + ',' +
107 | info.mandatory + ',' +
108 | info.regexp + ',' +
109 | info['max-length'] + ',' +
110 | info['min-length'] + ','+
111 | info.path +
112 | '\r\n');
113 | return info;
114 | }
115 |
116 | module.exports = {
117 | processFiles : processFiles,
118 | getFieldInfo : getFieldInfo,
119 | getListFields : getListFields,
120 | getGroupFields : getGroupFields
121 | };
122 |
--------------------------------------------------------------------------------