From f6218ec595522181d5de9a54b1e459434fd3e3b4 Mon Sep 17 00:00:00 2001 From: Ivan Duplenskikh <115665590+ivanduplenskikh@users.noreply.github.com> Date: Mon, 27 May 2024 18:02:48 +0200 Subject: [PATCH] CopyFilesOverSSHV0 change sequential files sending to continuous queue (#19798) * Change sequential exectuion to Queue mechanism * Update ssh2 sftp client in node20 handler package * Revert dependencies to their versions as they were in CopyFilesOverSSHV0 0.231.0 --- .../resources.resjson/en-US/resources.resjson | 13 +- .../Node20/Tests/package-lock.json | 14 +- .../_buildConfigs/Node20/package-lock.json | 565 +++++++++++------- .../_buildConfigs/Node20/package.json | 4 +- Tasks/CopyFilesOverSSHV0/copyfilesoverssh.ts | 238 +++++++- Tasks/CopyFilesOverSSHV0/package-lock.json | 41 +- Tasks/CopyFilesOverSSHV0/package.json | 8 +- Tasks/CopyFilesOverSSHV0/queue.ts | 98 +++ Tasks/CopyFilesOverSSHV0/sshhelper.ts | 213 ++++--- Tasks/CopyFilesOverSSHV0/task.json | 29 +- Tasks/CopyFilesOverSSHV0/task.loc.json | 27 +- Tasks/CopyFilesOverSSHV0/tsconfig.json | 2 +- _generated/CopyFilesOverSSHV0.versionmap.txt | 4 +- .../resources.resjson/en-US/resources.resjson | 13 +- .../CopyFilesOverSSHV0/copyfilesoverssh.ts | 238 +++++++- .../CopyFilesOverSSHV0/package-lock.json | 41 +- _generated/CopyFilesOverSSHV0/package.json | 8 +- _generated/CopyFilesOverSSHV0/queue.ts | 98 +++ _generated/CopyFilesOverSSHV0/sshhelper.ts | 213 ++++--- _generated/CopyFilesOverSSHV0/task.json | 33 +- _generated/CopyFilesOverSSHV0/task.loc.json | 31 +- _generated/CopyFilesOverSSHV0/tsconfig.json | 2 +- .../resources.resjson/en-US/resources.resjson | 13 +- .../Tests/package-lock.json | 14 +- .../copyfilesoverssh.ts | 238 +++++++- .../package-lock.json | 565 +++++++++++------- .../CopyFilesOverSSHV0_Node20/package.json | 4 +- _generated/CopyFilesOverSSHV0_Node20/queue.ts | 98 +++ .../CopyFilesOverSSHV0_Node20/sshhelper.ts | 213 ++++--- .../CopyFilesOverSSHV0_Node20/task.json | 33 +- .../CopyFilesOverSSHV0_Node20/task.loc.json | 31 +- .../CopyFilesOverSSHV0_Node20/tsconfig.json | 2 +- 32 files changed, 2353 insertions(+), 791 deletions(-) create mode 100644 Tasks/CopyFilesOverSSHV0/queue.ts create mode 100644 _generated/CopyFilesOverSSHV0/queue.ts create mode 100644 _generated/CopyFilesOverSSHV0_Node20/queue.ts diff --git a/Tasks/CopyFilesOverSSHV0/Strings/resources.resjson/en-US/resources.resjson b/Tasks/CopyFilesOverSSHV0/Strings/resources.resjson/en-US/resources.resjson index 56323de2ea28..c7755eb1f5c9 100644 --- a/Tasks/CopyFilesOverSSHV0/Strings/resources.resjson/en-US/resources.resjson +++ b/Tasks/CopyFilesOverSSHV0/Strings/resources.resjson/en-US/resources.resjson @@ -26,15 +26,23 @@ "loc.input.help.failOnEmptySource": "Fail if no matching files to be copied are found under the source folder.", "loc.input.label.flattenFolders": "Flatten folders", "loc.input.help.flattenFolders": "Flatten the folder structure and copy all files into the specified target folder on the remote machine.", + "loc.input.label.concurrentUploads": "Number of concurrent uploads when copying files", + "loc.input.help.concurrentUploads": "Number of concurrent uploads when copying files. Default is 10.", + "loc.input.label.delayBetweenUploads": "Delay between queueing uploads (in milliseconds)", + "loc.input.help.delayBetweenUploads": "Delay between queueing uploads (in milliseconds). Default is 50.", "loc.messages.CheckLogForStdErr": "Check the build log for STDERR from the command.", "loc.messages.CleanTargetFolder": "Cleaning target folder %s on the remote machine", "loc.messages.CleanTargetFolderFailed": "Failed to clean the target folder on the remote machine. %s", "loc.messages.ConnectionFailed": "Failed to connect to remote machine. Verify the SSH service connection details. %s.", "loc.messages.ConnectionNotSetup": "SSH service connection is not set up.", "loc.messages.CopyCompleted": "Completed copying %s files to the remote machine.", + "loc.messages.CopyDirectoryCompleted": "Completed copying %s directory to the remote machine.", + "loc.messages.CopyDirectoryFailed": "Failed to copy %s directory to the remote machine. %s", "loc.messages.CopyingFiles": "Found %s files to copy to the remote machine.", "loc.messages.FailedOnFile": "Failed to copy %s. %s", "loc.messages.FileExists": "File %s cannot be copied to the remote machine because it already exists and the 'Overwrite' option is disabled.", + "loc.messages.FolderCreated": "Created folder %s on the remote machine.", + "loc.messages.FoldersCreated": "Created %s folders on the remote machine.", "loc.messages.NothingToCopy": "No files were found matching the patterns specified to copy to the remote machine.", "loc.messages.NumberFailed": "Failed to copy %d files", "loc.messages.RemoteCmdExecutionErr": "Command %s failed with errors on remote machine. %s.", @@ -44,8 +52,9 @@ "loc.messages.StartedFileCopy": "Copying file %s to %s on remote machine.", "loc.messages.UploadFileFailed": "Failed to upload %s to %s on remote machine. %s.", "loc.messages.UseDefaultPort": "Using port 22 which is the default for SSH since no port was specified.", - "loc.messages.TargetNotCreated": "Unable to create target folder %s.", + "loc.messages.TargetNotCreated": "Unable to create target folder %s. %s.", "loc.messages.CheckingPathExistance": "Checking if %s on the remote machine exists.", "loc.messages.PathExists": "%s exists on the remote machine", - "loc.messages.PathNotExists": "%s doesn't exist on the remote machine" + "loc.messages.PathNotExists": "%s doesn't exist on the remote machine", + "loc.messages.UploadFolderFailed": "Failed to upload folder %s to %s on remote machine. %s." } \ No newline at end of file diff --git a/Tasks/CopyFilesOverSSHV0/_buildConfigs/Node20/Tests/package-lock.json b/Tasks/CopyFilesOverSSHV0/_buildConfigs/Node20/Tests/package-lock.json index ba6519c9013a..d1b155f0581e 100644 --- a/Tasks/CopyFilesOverSSHV0/_buildConfigs/Node20/Tests/package-lock.json +++ b/Tasks/CopyFilesOverSSHV0/_buildConfigs/Node20/Tests/package-lock.json @@ -1,10 +1,18 @@ { "name": "copy-files-over-ssh-tests", "version": "1.0.0", - "lockfileVersion": 1, + "lockfileVersion": 3, "requires": true, - "dependencies": { - "@types/mocha": { + "packages": { + "": { + "name": "copy-files-over-ssh-tests", + "version": "1.0.0", + "license": "MIT", + "devDependencies": { + "@types/mocha": "^5.2.7" + } + }, + "node_modules/@types/mocha": { "version": "5.2.7", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-5.2.7.tgz", "integrity": "sha512-NYrtPht0wGzhwe9+/idPaBB+TqkY9AhTvOLMkThm0IoEfLaiVQZwBwyJ5puCkO3AUCWrmcoePjp2mbFocKy4SQ==", diff --git a/Tasks/CopyFilesOverSSHV0/_buildConfigs/Node20/package-lock.json b/Tasks/CopyFilesOverSSHV0/_buildConfigs/Node20/package-lock.json index 8c721d54b2db..8c3e7035c1f5 100644 --- a/Tasks/CopyFilesOverSSHV0/_buildConfigs/Node20/package-lock.json +++ b/Tasks/CopyFilesOverSSHV0/_buildConfigs/Node20/package-lock.json @@ -1,230 +1,276 @@ { "name": "vsts-copyssh-task", "version": "1.0.0", - "lockfileVersion": 1, + "lockfileVersion": 3, "requires": true, - "dependencies": { - "@types/concat-stream": { + "packages": { + "": { + "name": "vsts-copyssh-task", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "@types/mocha": "^5.2.7", + "@types/node": "^20.3.1", + "azure-pipelines-task-lib": "^5.0.0-preview.0", + "minimatch": "^3.0.4", + "ssh2": "^1.4.0", + "ssh2-sftp-client": "^7.0.4" + }, + "devDependencies": { + "typescript": "5.1.6" + } + }, + "node_modules/@types/concat-stream": { "version": "1.6.1", "resolved": "https://registry.npmjs.org/@types/concat-stream/-/concat-stream-1.6.1.tgz", "integrity": "sha512-eHE4cQPoj6ngxBZMvVf6Hw7Mh4jMW4U9lpGmS5GBPB9RYxlFg+CHaVN7ErNY4W9XfLIEn20b4VDYaIrbq0q4uA==", - "requires": { + "dependencies": { "@types/node": "*" } }, - "@types/form-data": { + "node_modules/@types/form-data": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-0.0.33.tgz", "integrity": "sha512-8BSvG1kGm83cyJITQMZSulnl6QV8jqAGreJsc5tPu1Jq0vTSOiY/k24Wx82JRpWwZSqrala6sd5rWi6aNXvqcw==", - "requires": { + "dependencies": { "@types/node": "*" } }, - "@types/mocha": { + "node_modules/@types/mocha": { "version": "5.2.7", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-5.2.7.tgz", "integrity": "sha512-NYrtPht0wGzhwe9+/idPaBB+TqkY9AhTvOLMkThm0IoEfLaiVQZwBwyJ5puCkO3AUCWrmcoePjp2mbFocKy4SQ==" }, - "@types/node": { + "node_modules/@types/node": { "version": "20.7.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.7.0.tgz", "integrity": "sha512-zI22/pJW2wUZOVyguFaUL1HABdmSVxpXrzIqkjsHmyUjNhPoWM1CKfvVuXfetHhIok4RY573cqS0mZ1SJEnoTg==" }, - "@types/qs": { + "node_modules/@types/qs": { "version": "6.9.8", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.8.tgz", "integrity": "sha512-u95svzDlTysU5xecFNTgfFG5RUWu1A9P0VzgpcIiGZA9iraHOdSzcxMxQ55DyeRaGCSxQi7LxXDI4rzq/MYfdg==" }, - "asap": { + "node_modules/asap": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" }, - "asn1": { + "node_modules/asn1": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", - "requires": { + "dependencies": { "safer-buffer": "~2.1.0" } }, - "asynckit": { + "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, - "azure-pipelines-task-lib": { + "node_modules/azure-pipelines-task-lib": { "version": "5.0.0-preview.0", "resolved": "https://registry.npmjs.org/azure-pipelines-task-lib/-/azure-pipelines-task-lib-5.0.0-preview.0.tgz", "integrity": "sha512-uQJEv+q3/7RD7kgphFd33uXijaPwA1ePYoNkAgZpUhbZUtc2u4JL4ujTT/JJfgXBNW/QQNAiDQ2OC7vOQG/0tg==", - "requires": { + "dependencies": { "minimatch": "3.0.5", "q": "^1.5.1", "semver": "^5.1.0", "shelljs": "^0.8.5", "sync-request": "6.1.0", "uuid": "^3.0.1" - }, + } + }, + "node_modules/azure-pipelines-task-lib/node_modules/minimatch": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.5.tgz", + "integrity": "sha512-tUpxzX0VAzJHjLu0xUfFv1gwVp9ba3IOuRAVH2EGuRW8a5emA2FlACLqiT/lDVtS1W+TGNwqz3sWaNyLgDJWuw==", "dependencies": { - "minimatch": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.5.tgz", - "integrity": "sha512-tUpxzX0VAzJHjLu0xUfFv1gwVp9ba3IOuRAVH2EGuRW8a5emA2FlACLqiT/lDVtS1W+TGNwqz3sWaNyLgDJWuw==", - "requires": { - "brace-expansion": "^1.1.7" - } - } + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" } }, - "balanced-match": { + "node_modules/balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, - "bcrypt-pbkdf": { + "node_modules/bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", - "requires": { + "dependencies": { "tweetnacl": "^0.14.3" } }, - "brace-expansion": { + "node_modules/brace-expansion": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.8.tgz", "integrity": "sha1-wHshHHyVLsH479Uad+8NHTmQopI=", - "requires": { + "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, - "buffer-from": { + "node_modules/buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" }, - "buildcheck": { + "node_modules/buildcheck": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", - "optional": true + "optional": true, + "engines": { + "node": ">=10.0.0" + } }, - "call-bind": { + "node_modules/call-bind": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", - "requires": { + "dependencies": { "function-bind": "^1.1.1", "get-intrinsic": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, - "caseless": { + "node_modules/caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==" }, - "combined-stream": { + "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "requires": { + "dependencies": { "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" } }, - "concat-map": { + "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, - "concat-stream": { + "node_modules/concat-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", - "requires": { + "engines": [ + "node >= 6.0" + ], + "dependencies": { "buffer-from": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.0.2", "typedarray": "^0.0.6" - }, + } + }, + "node_modules/concat-stream/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", "dependencies": { - "readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - } + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" } }, - "core-util-is": { + "node_modules/core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" }, - "cpu-features": { + "node_modules/cpu-features": { "version": "0.0.9", "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.9.tgz", "integrity": "sha512-AKjgn2rP2yJyfbepsmLfiYcmtNn/2eUvocUyM/09yB0YDiz39HteK/5/T4Onf0pmdYDMgkBoGvRLvEguzyL7wQ==", + "hasInstallScript": true, "optional": true, - "requires": { + "dependencies": { "buildcheck": "~0.0.6", "nan": "^2.17.0" + }, + "engines": { + "node": ">=10.0.0" } }, - "delayed-stream": { + "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } }, - "err-code": { + "node_modules/err-code": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==" }, - "form-data": { + "node_modules/form-data": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", - "requires": { + "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.6", "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 0.12" } }, - "fs.realpath": { + "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, - "function-bind": { + "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, - "get-intrinsic": { + "node_modules/get-intrinsic": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz", "integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==", - "requires": { + "dependencies": { "function-bind": "^1.1.1", "has": "^1.0.3", "has-proto": "^1.0.1", "has-symbols": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, - "get-port": { + "node_modules/get-port": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/get-port/-/get-port-3.2.0.tgz", - "integrity": "sha512-x5UJKlgeUiNT8nyo/AcnwLnZuZNcSjSw0kogRB+Whd1fjjFq4B1hySFxSFWWSn4mIBzg3sRNUDFYc4g5gjPoLg==" + "integrity": "sha512-x5UJKlgeUiNT8nyo/AcnwLnZuZNcSjSw0kogRB+Whd1fjjFq4B1hySFxSFWWSn4mIBzg3sRNUDFYc4g5gjPoLg==", + "engines": { + "node": ">=4" + } }, - "glob": { + "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "requires": { + "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", @@ -232,201 +278,259 @@ "once": "^1.3.0", "path-is-absolute": "^1.0.0" }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dependencies": { - "minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "requires": { - "brace-expansion": "^1.1.7" - } - } + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" } }, - "has": { + "node_modules/has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "requires": { + "dependencies": { "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" } }, - "has-proto": { + "node_modules/has-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", - "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==" + "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, - "has-symbols": { + "node_modules/has-symbols": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, - "http-basic": { + "node_modules/http-basic": { "version": "8.1.3", "resolved": "https://registry.npmjs.org/http-basic/-/http-basic-8.1.3.tgz", "integrity": "sha512-/EcDMwJZh3mABI2NhGfHOGOeOZITqfkEO4p/xK+l3NpyncIHUQBoMvCSF/b5GqvKtySC2srL/GGG3+EtlqlmCw==", - "requires": { + "dependencies": { "caseless": "^0.12.0", "concat-stream": "^1.6.2", "http-response-object": "^3.0.1", "parse-cache-control": "^1.0.1" }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/http-basic/node_modules/concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "engines": [ + "node >= 0.8" + ], "dependencies": { - "concat-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", - "requires": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^2.2.2", - "typedarray": "^0.0.6" - } - } + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" } }, - "http-response-object": { + "node_modules/http-response-object": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/http-response-object/-/http-response-object-3.0.2.tgz", "integrity": "sha512-bqX0XTF6fnXSQcEJ2Iuyr75yVakyjIDCqroJQ/aHfSdlM743Cwqoi2nDYMzLGWUcuTWGWy8AAvOKXTfiv6q9RA==", - "requires": { - "@types/node": "^10.0.3" - }, "dependencies": { - "@types/node": { - "version": "10.17.60", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", - "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==" - } + "@types/node": "^10.0.3" } }, - "inflight": { + "node_modules/http-response-object/node_modules/@types/node": { + "version": "10.17.60", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", + "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==" + }, + "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "requires": { + "dependencies": { "once": "^1.3.0", "wrappy": "1" } }, - "inherits": { + "node_modules/inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, - "interpret": { + "node_modules/interpret": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", - "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==" + "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", + "engines": { + "node": ">= 0.10" + } }, - "is-core-module": { + "node_modules/is-core-module": { "version": "2.13.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.0.tgz", "integrity": "sha512-Z7dk6Qo8pOCp3l4tsX2C5ZVas4V+UxwQodwZhLopL91TX8UyyHEXafPcyoeeWuLrwzHcr3igO78wNLwHJHsMCQ==", - "requires": { + "dependencies": { "has": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, - "isarray": { + "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" }, - "mime-db": { + "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } }, - "mime-types": { + "node_modules/mime-types": { "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "requires": { + "dependencies": { "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" } }, - "minimatch": { + "node_modules/minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha1-UWbihkV/AzBgZL5Ul+jbsMPTIIM=", - "requires": { + "dependencies": { "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" } }, - "nan": { + "node_modules/nan": { "version": "2.18.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz", "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==", "optional": true }, - "object-inspect": { + "node_modules/object-inspect": { "version": "1.12.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", - "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==" + "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, - "once": { + "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "requires": { + "dependencies": { "wrappy": "1" } }, - "parse-cache-control": { + "node_modules/parse-cache-control": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parse-cache-control/-/parse-cache-control-1.0.1.tgz", "integrity": "sha512-60zvsJReQPX5/QP0Kzfd/VrpjScIQ7SHBW6bFCYfEP+fp0Eppr1SHhIO5nd1PjZtvclzSzES9D/p5nFJurwfWg==" }, - "path-is-absolute": { + "node_modules/path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "engines": { + "node": ">=0.10.0" + } }, - "path-parse": { + "node_modules/path-parse": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, - "process-nextick-args": { + "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, - "promise": { + "node_modules/promise": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/promise/-/promise-8.3.0.tgz", "integrity": "sha512-rZPNPKTOYVNEEKFaq1HqTgOwZD+4/YHS5ukLzQCypkj+OkYx7iv0mA91lJlpPPZ8vMau3IIGj5Qlwrx+8iiSmg==", - "requires": { + "dependencies": { "asap": "~2.0.6" } }, - "promise-retry": { + "node_modules/promise-retry": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", - "requires": { + "dependencies": { "err-code": "^2.0.2", "retry": "^0.12.0" + }, + "engines": { + "node": ">=10" } }, - "q": { + "node_modules/q": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", - "integrity": "sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==" + "integrity": "sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==", + "engines": { + "node": ">=0.6.0", + "teleport": ">=0.2.0" + } }, - "qs": { + "node_modules/qs": { "version": "6.11.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz", "integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==", - "requires": { + "dependencies": { "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, - "readable-stream": { + "node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "requires": { + "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", @@ -436,128 +540,168 @@ "util-deprecate": "~1.0.1" } }, - "rechoir": { + "node_modules/rechoir": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==", - "requires": { + "dependencies": { "resolve": "^1.1.6" + }, + "engines": { + "node": ">= 0.10" } }, - "resolve": { + "node_modules/resolve": { "version": "1.22.6", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.6.tgz", "integrity": "sha512-njhxM7mV12JfufShqGy3Rz8j11RPdLy4xi15UurGJeoHLfJpVXKdh3ueuOqbYUcDZnffr6X739JBo5LzyahEsw==", - "requires": { + "dependencies": { "is-core-module": "^2.13.0", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, - "retry": { + "node_modules/retry": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==" + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "engines": { + "node": ">= 4" + } }, - "safe-buffer": { + "node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, - "safer-buffer": { + "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "semver": { + "node_modules/semver": { "version": "5.7.2", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", - "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==" + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", + "bin": { + "semver": "bin/semver" + } }, - "shelljs": { + "node_modules/shelljs": { "version": "0.8.5", "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", - "requires": { + "dependencies": { "glob": "^7.0.0", "interpret": "^1.0.0", "rechoir": "^0.6.2" + }, + "bin": { + "shjs": "bin/shjs" + }, + "engines": { + "node": ">=4" } }, - "side-channel": { + "node_modules/side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", - "requires": { + "dependencies": { "call-bind": "^1.0.0", "get-intrinsic": "^1.0.2", "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, - "ssh2": { + "node_modules/ssh2": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.15.0.tgz", "integrity": "sha512-C0PHgX4h6lBxYx7hcXwu3QWdh4tg6tZZsTfXcdvc5caW/EMxaB4H9dWsl7qk+F7LAW762hp8VbXOX7x4xUYvEw==", - "requires": { + "hasInstallScript": true, + "dependencies": { "asn1": "^0.2.6", - "bcrypt-pbkdf": "^1.0.2", + "bcrypt-pbkdf": "^1.0.2" + }, + "engines": { + "node": ">=10.16.0" + }, + "optionalDependencies": { "cpu-features": "~0.0.9", "nan": "^2.18.0" } }, - "ssh2-sftp-client": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-9.1.0.tgz", - "integrity": "sha512-Hzdr9OE6GxZjcmyM9tgBSIFVyrHAp9c6U2Y4yBkmYOHoQvZ7pIm27dmltvcmRfxcWiIcg8HBvG5iAikDf+ZuzQ==", - "requires": { + "node_modules/ssh2-sftp-client": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-7.2.3.tgz", + "integrity": "sha512-Bmq4Uewu3e0XOwu5bnPbiS5KRQYv+dff5H6+85V4GZrPrt0Fkt1nUH+uXanyAkoNxUpzjnAPEEoLdOaBO9c3xw==", + "dependencies": { "concat-stream": "^2.0.0", "promise-retry": "^2.0.1", - "ssh2": "^1.12.0" + "ssh2": "^1.8.0" + }, + "engines": { + "node": ">=10.24.1" } }, - "string_decoder": { + "node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - }, "dependencies": { - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - } + "safe-buffer": "~5.1.0" } }, - "supports-preserve-symlinks-flag": { + "node_modules/string_decoder/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/supports-preserve-symlinks-flag": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", - "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==" + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, - "sync-request": { + "node_modules/sync-request": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/sync-request/-/sync-request-6.1.0.tgz", "integrity": "sha512-8fjNkrNlNCrVc/av+Jn+xxqfCjYaBoHqCsDz6mt030UMxJGr+GSfCV1dQt2gRtlL63+VPidwDVLr7V2OcTSdRw==", - "requires": { + "dependencies": { "http-response-object": "^3.0.1", "sync-rpc": "^1.2.1", "then-request": "^6.0.0" + }, + "engines": { + "node": ">=8.0.0" } }, - "sync-rpc": { + "node_modules/sync-rpc": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/sync-rpc/-/sync-rpc-1.3.6.tgz", "integrity": "sha512-J8jTXuZzRlvU7HemDgHi3pGnh/rkoqR/OZSjhTyyZrEkkYQbk7Z33AXp37mkPfPpfdOuj7Ex3H/TJM1z48uPQw==", - "requires": { + "dependencies": { "get-port": "^3.1.0" } }, - "then-request": { + "node_modules/then-request": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/then-request/-/then-request-6.0.2.tgz", "integrity": "sha512-3ZBiG7JvP3wbDzA9iNY5zJQcHL4jn/0BWtXIkagfz7QgOL/LqjCEOBQuJNZfu0XYnv5JhKh+cDxCPM4ILrqruA==", - "requires": { + "dependencies": { "@types/concat-stream": "^1.6.0", "@types/form-data": "0.0.33", "@types/node": "^8.0.0", @@ -570,52 +714,67 @@ "promise": "^8.0.0", "qs": "^6.4.0" }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/then-request/node_modules/@types/node": { + "version": "8.10.66", + "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.66.tgz", + "integrity": "sha512-tktOkFUA4kXx2hhhrB8bIFb5TbwzS4uOhKEmwiD+NoiL0qtP2OQ9mFldbgD4dV1djrlBYP6eBuQZiWjuHUpqFw==" + }, + "node_modules/then-request/node_modules/concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "engines": [ + "node >= 0.8" + ], "dependencies": { - "@types/node": { - "version": "8.10.66", - "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.66.tgz", - "integrity": "sha512-tktOkFUA4kXx2hhhrB8bIFb5TbwzS4uOhKEmwiD+NoiL0qtP2OQ9mFldbgD4dV1djrlBYP6eBuQZiWjuHUpqFw==" - }, - "concat-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", - "requires": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^2.2.2", - "typedarray": "^0.0.6" - } - } - } - }, - "tweetnacl": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + } + }, + "node_modules/tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" }, - "typedarray": { + "node_modules/typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==" }, - "typescript": { + "node_modules/typescript": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.6.tgz", "integrity": "sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==", - "dev": true + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } }, - "util-deprecate": { + "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, - "uuid": { + "node_modules/uuid": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "bin": { + "uuid": "bin/uuid" + } }, - "wrappy": { + "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" diff --git a/Tasks/CopyFilesOverSSHV0/_buildConfigs/Node20/package.json b/Tasks/CopyFilesOverSSHV0/_buildConfigs/Node20/package.json index 960ad8d5df3f..c96ccab01154 100644 --- a/Tasks/CopyFilesOverSSHV0/_buildConfigs/Node20/package.json +++ b/Tasks/CopyFilesOverSSHV0/_buildConfigs/Node20/package.json @@ -17,8 +17,8 @@ }, "homepage": "https://github.com/Microsoft.com/vsts-tasks#readme", "dependencies": { - "ssh2": "^1.15.0", - "ssh2-sftp-client": "^9.1.0", + "ssh2": "^1.4.0", + "ssh2-sftp-client": "^7.0.4", "minimatch": "^3.0.4", "azure-pipelines-task-lib": "^5.0.0-preview.0", "@types/mocha": "^5.2.7", diff --git a/Tasks/CopyFilesOverSSHV0/copyfilesoverssh.ts b/Tasks/CopyFilesOverSSHV0/copyfilesoverssh.ts index e33a1d9f6923..5d3e5a6f87b9 100644 --- a/Tasks/CopyFilesOverSSHV0/copyfilesoverssh.ts +++ b/Tasks/CopyFilesOverSSHV0/copyfilesoverssh.ts @@ -4,6 +4,7 @@ import * as tl from 'azure-pipelines-task-lib/task'; import * as minimatch from 'minimatch'; import * as utils from './utils'; import { SshHelper } from './sshhelper'; +import Queue, { QueueEvents } from './queue'; // This method will find the list of matching files for the specified contents // This logic is the same as the one used by CopyFiles task except for allowing dot folders to be copied @@ -99,6 +100,235 @@ function getFilesToCopy(sourceFolder: string, contents: string[]): string[] { return files; } +function prepareFiles(filesToCopy: string[], sourceFolder: string, targetFolder: string, flattenFolders: boolean) { + return filesToCopy.map(x => { + let targetPath = path.posix.join( + targetFolder, + flattenFolders + ? path.basename(x) + : x.substring(sourceFolder.length).replace(/^\\/g, "").replace(/^\//g, "") + ); + + if (!path.isAbsolute(targetPath) && !utils.pathIsUNC(targetPath)) { + targetPath = `./${targetPath}`; + } + + return [ x, utils.unixyPath(targetPath) ]; + }); +} + +function getUniqueFolders(filesToCopy: string[]) { + const foldersSet = new Set(); + + for (const filePath of filesToCopy) { + const folderPath = path.dirname(filePath); + + if (foldersSet.has(folderPath)) { + continue; + } + + foldersSet.add(folderPath); + } + + return Array.from(foldersSet.values()); +} + +async function newRun() { + tl.setResourcePath(path.join(__dirname, 'task.json')); + + // Read SSH endpoint input + const sshEndpoint = tl.getInput('sshEndpoint', true); + const username = tl.getEndpointAuthorizationParameter(sshEndpoint, 'username', false); + // Passphrase is optional + const password = tl.getEndpointAuthorizationParameter(sshEndpoint, 'password', true); + // Private key is optional, password can be used for connecting + const privateKey = process.env['ENDPOINT_DATA_' + sshEndpoint + '_PRIVATEKEY']; + const hostname = tl.getEndpointDataParameter(sshEndpoint, 'host', false); + // Port is optional, will use 22 as default port if not specified + let port = tl.getEndpointDataParameter(sshEndpoint, 'port', true); + + if (!port) { + console.log(tl.loc('UseDefaultPort')); + port = '22'; + } + + const readyTimeout = parseInt(tl.getInput('readyTimeout', true), 10); + const useFastPut = !(process.env['USE_FAST_PUT'] === 'false'); + const concurrentUploads = parseInt(tl.getInput('concurrentUploads')); + + // Set up the SSH connection configuration based on endpoint details + let sshConfig: Object = { + host: hostname, + port: port, + username: username, + readyTimeout: readyTimeout, + useFastPut: useFastPut, + promiseLimit: isNaN(concurrentUploads) ? 10 : concurrentUploads + }; + + if (privateKey) { + tl.debug('Using private key for ssh connection.'); + + sshConfig = { + ...sshConfig, + privateKey, + passphrase: password + } + } else { + // Use password + tl.debug('Using username and password for ssh connection.'); + + sshConfig = { + ...sshConfig, + password, + } + } + + // Contents is a multiline input containing glob patterns + const contents = tl.getDelimitedInput('contents', '\n', true); + const sourceFolder = tl.getPathInput('sourceFolder', true, true); + let targetFolder = tl.getInput('targetFolder'); + + if (!targetFolder) { + targetFolder = "./"; + } else { + // '~/' is unsupported + targetFolder = targetFolder.replace(/^~\//, "./"); + } + + // Read the copy options + const cleanTargetFolder = tl.getBoolInput('cleanTargetFolder', false); + const overwrite = tl.getBoolInput('overwrite', false); + const failOnEmptySource = tl.getBoolInput('failOnEmptySource', false); + const flattenFolders = tl.getBoolInput('flattenFolders', false); + + if (!tl.stats(sourceFolder).isDirectory()) { + tl.setResult(tl.TaskResult.Failed, tl.loc('SourceNotFolder')); + return; + } + + // Initialize the SSH helpers, set up the connection + const sshHelper = new SshHelper(sshConfig); + await sshHelper.setupConnection(); + + if (cleanTargetFolder && await sshHelper.checkRemotePathExists(targetFolder)) { + console.log(tl.loc('CleanTargetFolder', targetFolder)); + const isWindowsOnTarget = tl.getBoolInput('isWindowsOnTarget', false); + const cleanHiddenFilesInTarget = tl.getBoolInput('cleanHiddenFilesInTarget', false); + const cleanTargetFolderCmd = utils.getCleanTargetFolderCmd(targetFolder, isWindowsOnTarget, cleanHiddenFilesInTarget); + + try { + await sshHelper.runCommandOnRemoteMachine(cleanTargetFolderCmd, null); + } catch (error) { + tl.setResult(tl.TaskResult.Failed, tl.loc('CleanTargetFolderFailed', error)); + tl.debug('Closing the client connection'); + await sshHelper.closeConnection(); + return; + } + } + + // If the contents were parsed into an array and the first element was set as default "**", + // then upload the entire directory + if (contents.length === 1 && contents[0] === "**") { + tl.debug("Upload a directory to a remote machine"); + + try { + const completedDirectory = await sshHelper.uploadFolder(sourceFolder, targetFolder); + tl.setResult(tl.TaskResult.Succeeded, tl.loc('CopyDirectoryCompleted', completedDirectory)); + } catch (error) { + tl.setResult(tl.TaskResult.Failed, tl.loc("CopyDirectoryFailed", sourceFolder, error)); + } + + tl.debug('Closing the client connection'); + await sshHelper.closeConnection(); + return; + } + + // Identify the files to copy + const filesToCopy = getFilesToCopy(sourceFolder, contents); + + // Copy files to remote machine + if (filesToCopy.length === 0) { + if (failOnEmptySource) { + tl.setResult(tl.TaskResult.Failed, tl.loc('NothingToCopy')); + return; + } else { + tl.warning(tl.loc('NothingToCopy')); + return; + } + } + + const preparedFiles = prepareFiles(filesToCopy, sourceFolder, targetFolder, flattenFolders); + + tl.debug(`Number of files to copy = ${preparedFiles.length}`); + tl.debug(`filesToCopy = ${preparedFiles}`); + + console.log(tl.loc('CopyingFiles', preparedFiles.length)); + + // Create remote folders structure + const folderStructure = getUniqueFolders(preparedFiles.map(x => x[1]).sort()); + + for (const foldersPath of folderStructure) { + try { + await sshHelper.createRemoteDirectory(foldersPath); + console.log(tl.loc("FolderCreated", foldersPath)); + } catch (error) { + await sshHelper.closeConnection(); + tl.setResult(tl.TaskResult.Failed, tl.loc('TargetNotCreated', foldersPath, error)); + return; + } + } + + console.log(tl.loc("FoldersCreated", folderStructure.length)); + + const delayBetweenUploads = parseInt(tl.getInput('delayBetweenUploads')); + + // Upload files to remote machine + const q = new Queue({ + concurrent: isNaN(concurrentUploads) ? 10 : concurrentUploads, + delay: isNaN(delayBetweenUploads) ? 50 : delayBetweenUploads, + }); + + q.enqueue(preparedFiles.map((pathTuple) => { + const [ filepath, targetPath ] = pathTuple; + + return { + filepath, + job: async () => { + tl.debug(`Filepath = ${filepath}`); + console.log(tl.loc('StartedFileCopy', filepath, targetPath)); + + if (!overwrite && await sshHelper.checkRemotePathExists(targetPath)) { + throw new Error(tl.loc('FileExists', targetPath)); + } + + return await sshHelper.uploadFile(filepath, targetPath); + } + }; + })); + + const errors = []; + let successfullyCopiedFilesCount = 0; + + q.on(QueueEvents.PROCESSED, () => successfullyCopiedFilesCount++); + q.on(QueueEvents.EMPTY, () => tl.debug('Queue is empty')); + q.on(QueueEvents.END, async () => { + tl.debug('End of the queue processing'); + await sshHelper.closeConnection(); + + if (errors.length === 0) { + tl.setResult(tl.TaskResult.Succeeded, tl.loc('CopyCompleted', successfullyCopiedFilesCount)); + } else { + tl.debug(`Errors count ${errors.length}`); + errors.forEach(tl.error); + tl.setResult(tl.TaskResult.Failed, tl.loc('NumberFailed', errors.length)); + } + }); + q.on(QueueEvents.ERROR, (error, filepath) => { + errors.push(tl.loc('FailedOnFile', filepath, error.message)); + }); +} + async function run() { let sshHelper: SshHelper; try { @@ -248,16 +478,20 @@ async function run() { } } -run().then(() => { +if (tl.getBoolFeatureFlag('COPYFILESOVERSSHV0_USE_QUEUE')) { + newRun(); +} else { + run().then(() => { tl.debug('Task successfully accomplished'); }) .catch(err => { tl.debug('Run was unexpectedly failed due to: ' + err); }); +} function getReadyTimeoutVariable(): number { let readyTimeoutString: string = tl.getInput('readyTimeout', true); const readyTimeout: number = parseInt(readyTimeoutString, 10); return readyTimeout; -} +} \ No newline at end of file diff --git a/Tasks/CopyFilesOverSSHV0/package-lock.json b/Tasks/CopyFilesOverSSHV0/package-lock.json index 7bbc70b607cd..3b17266265e2 100644 --- a/Tasks/CopyFilesOverSSHV0/package-lock.json +++ b/Tasks/CopyFilesOverSSHV0/package-lock.json @@ -103,12 +103,6 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" }, - "buildcheck": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", - "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", - "optional": true - }, "call-bind": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", @@ -165,13 +159,12 @@ "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" }, "cpu-features": { - "version": "0.0.9", - "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.9.tgz", - "integrity": "sha512-AKjgn2rP2yJyfbepsmLfiYcmtNn/2eUvocUyM/09yB0YDiz39HteK/5/T4Onf0pmdYDMgkBoGvRLvEguzyL7wQ==", + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.2.tgz", + "integrity": "sha512-/2yieBqvMcRj8McNzkycjW2v3OIUOibBfd2dLEJ0nWts8NobAxwiyw9phVNS6oDL8x8tz9F7uNVFEVpJncQpeA==", "optional": true, "requires": { - "buildcheck": "~0.0.6", - "nan": "^2.17.0" + "nan": "^2.14.1" } }, "delayed-stream": { @@ -347,9 +340,9 @@ } }, "nan": { - "version": "2.18.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz", - "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==", + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.19.0.tgz", + "integrity": "sha512-nO1xXxfh/RWNxfd/XPfbIfFk5vgLsAxUR9y5O0cHMJu/AW9U95JLXqthYHjEp+8gQ5p96K9jUp8nbVOxCdRbtw==", "optional": true }, "object-inspect": { @@ -488,24 +481,24 @@ } }, "ssh2": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.15.0.tgz", - "integrity": "sha512-C0PHgX4h6lBxYx7hcXwu3QWdh4tg6tZZsTfXcdvc5caW/EMxaB4H9dWsl7qk+F7LAW762hp8VbXOX7x4xUYvEw==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.4.0.tgz", + "integrity": "sha512-XvXwcXKvS452DyQvCa6Ct+chpucwc/UyxgliYz+rWXJ3jDHdtBb9xgmxJdMmnIn5bpgGAEV3KaEsH98ZGPHqwg==", "requires": { - "asn1": "^0.2.6", + "asn1": "^0.2.4", "bcrypt-pbkdf": "^1.0.2", - "cpu-features": "~0.0.9", - "nan": "^2.18.0" + "cpu-features": "0.0.2", + "nan": "^2.15.0" } }, "ssh2-sftp-client": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-9.1.0.tgz", - "integrity": "sha512-Hzdr9OE6GxZjcmyM9tgBSIFVyrHAp9c6U2Y4yBkmYOHoQvZ7pIm27dmltvcmRfxcWiIcg8HBvG5iAikDf+ZuzQ==", + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-7.0.4.tgz", + "integrity": "sha512-4fFSTgoYlzcAtGfEjiXN6N41s1jSUmPlI00f7uD7pQOjt9yK9susminINKTRvPp35dkrATrlNZVhUxNCt3z5+w==", "requires": { "concat-stream": "^2.0.0", "promise-retry": "^2.0.1", - "ssh2": "^1.12.0" + "ssh2": "^1.4.0" } }, "string_decoder": { diff --git a/Tasks/CopyFilesOverSSHV0/package.json b/Tasks/CopyFilesOverSSHV0/package.json index 6e94d249d358..668d6a5a2a0a 100644 --- a/Tasks/CopyFilesOverSSHV0/package.json +++ b/Tasks/CopyFilesOverSSHV0/package.json @@ -17,11 +17,11 @@ }, "homepage": "https://github.com/Microsoft.com/vsts-tasks#readme", "dependencies": { - "ssh2": "^1.15.0", - "ssh2-sftp-client": "^9.1.0", - "minimatch": "^3.0.4", + "ssh2": "1.4.0", + "ssh2-sftp-client": "7.0.4", + "@types/mocha": "^5.2.7", "azure-pipelines-task-lib": "^5.0.0-preview.0", - "@types/mocha": "^5.2.7" + "minimatch": "^3.0.4" }, "devDependencies": { "typescript": "4.0.2" diff --git a/Tasks/CopyFilesOverSSHV0/queue.ts b/Tasks/CopyFilesOverSSHV0/queue.ts new file mode 100644 index 000000000000..a9f0e30cd54f --- /dev/null +++ b/Tasks/CopyFilesOverSSHV0/queue.ts @@ -0,0 +1,98 @@ +const EventEmitter = require('events'); + +type QueueOptions = { + concurrent: number; + delay: number; +} + +export enum QueueEvents { + END = 'end', + PROCESSED = 'processed', + ERROR = 'error', + EMPTY = 'empty' +} + +interface IJob { + filepath: string; + job: () => Promise; +} + +export default class Queue extends EventEmitter { + jobs: IJob[] = []; + concurrent = 10; + inProcess = 0; + delay = 50; + + constructor(options: QueueOptions) { + super(); + + this.concurrent = options.concurrent || 10; + this.delay = options.delay || 50; + } + + enqueue(jobs) { + this.jobs.push(...jobs); + this.start(); + } + + dequeue() { + return this.jobs.shift(); + } + + get isConcurrentCapacityReached() { + return this.inProcess >= this.concurrent; + } + + consume() { + this.inProcess--; + + if (this.inProcess === 0) { + this.emit(QueueEvents.EMPTY); + } + } + + waitForComplete() { + if (this.inProcess === 0) { + this.emit(QueueEvents.END); + } else { + setTimeout(() => { + this.waitForComplete(); + }, this.delay); + } + } + + start() { + if (this.jobs.length === 0) { + this.waitForComplete(); + return; + } + + if (this.isConcurrentCapacityReached) { + setTimeout(() => { + this.start(); + }, this.delay); + return; + } + + const { filepath, job } = this.dequeue(); + + if (!job) { + return; + } + + this.inProcess++; + + setTimeout(async () => { + try { + const result = await job(); + this.emit(QueueEvents.PROCESSED, result); + } catch (error) { + this.emit(QueueEvents.ERROR, error, filepath); + } + + this.consume(); + }); + + this.start(); + } +} \ No newline at end of file diff --git a/Tasks/CopyFilesOverSSHV0/sshhelper.ts b/Tasks/CopyFilesOverSSHV0/sshhelper.ts index 0ff955f3db03..ed6b25302af7 100644 --- a/Tasks/CopyFilesOverSSHV0/sshhelper.ts +++ b/Tasks/CopyFilesOverSSHV0/sshhelper.ts @@ -1,8 +1,7 @@ -import Q = require('q'); import tl = require('azure-pipelines-task-lib/task'); -const path = require('path'); var Ssh2Client = require('ssh2').Client; var SftpClient = require('ssh2-sftp-client'); +var path = require('path'); export class RemoteCommandOptions { public failOnStdErr : boolean; @@ -30,27 +29,27 @@ export class SshHelper { } private async setupSshClientConnection() : Promise { - const defer = Q.defer(); - this.sshClient = new Ssh2Client(); - this.sshClient.once('ready', () => { - defer.resolve(); - }).once('error', (err) => { - defer.reject(tl.loc('ConnectionFailed', err)); - }).connect(this.sshConfig); - await defer.promise; + return new Promise((resolve, reject) => { + this.sshClient = new Ssh2Client(); + this.sshClient.once('ready', () => { + resolve(); + }).once('error', (err) => { + reject(tl.loc('ConnectionFailed', err)); + }).connect(this.sshConfig); + }); } private async setupSftpConnection() : Promise { - const defer = Q.defer(); - try { - this.sftpClient = new SftpClient(); - await this.sftpClient.connect(this.sshConfig); - defer.resolve(); - } catch (err) { - this.sftpClient = null; - defer.reject(tl.loc('ConnectionFailed', err)); - } - await defer.promise; + return new Promise(async (resolve, reject) => { + try { + this.sftpClient = new SftpClient(); + await this.sftpClient.connect(this.sshConfig); + resolve(); + } catch (err) { + this.sftpClient = null; + reject(tl.loc('ConnectionFailed', err)); + } + }); } /** @@ -103,31 +102,46 @@ export class SshHelper { async uploadFile(sourceFile: string, dest: string) : Promise { tl.debug('Upload ' + sourceFile + ' to ' + dest + ' on remote machine.'); - var defer = Q.defer(); - if(!this.sftpClient) { - defer.reject(tl.loc('ConnectionNotSetup')); + if (!this.sftpClient) { + return Promise.reject(tl.loc('ConnectionNotSetup')); } const remotePath = path.dirname(dest); - try { - if (!await this.sftpClient.exists(remotePath)) { - await this.sftpClient.mkdir(remotePath, true); + + if (!tl.getBoolFeatureFlag('COPYFILESOVERSSHV0_USE_QUEUE')) { + try { + + if (!await this.sftpClient.exists(remotePath)) { + await this.sftpClient.mkdir(remotePath, true); + } + } catch (error) { + return Promise.reject(tl.loc('TargetNotCreated', remotePath)); } - } catch (error) { - defer.reject(tl.loc('TargetNotCreated', remotePath)); } - try { - if (this.sshConfig.useFastPut) { - await this.sftpClient.fastPut(sourceFile, dest); - } else { - await this.sftpClient.put(sourceFile, dest); - } - defer.resolve(dest); - } catch (err) { - defer.reject(tl.loc('UploadFileFailed', sourceFile, dest, err)); + if (this.sshConfig.useFastPut) { + return this.sftpClient.fastPut(sourceFile, dest); + } else { + return this.sftpClient.put(sourceFile, dest); } - return defer.promise; + } + + async uploadFolder(sourceFolder: string, destFolder: string) : Promise { + tl.debug('Upload ' + sourceFolder + ' to ' + destFolder + ' on remote machine.'); + + return new Promise(async (resolve, reject) => { + if (!this.sftpClient) { + reject(tl.loc('ConnectionNotSetup')); + return; + } + + try { + await this.sftpClient.uploadDir(sourceFolder, destFolder); + return resolve(destFolder); + } catch (err) { + reject(tl.loc('UploadFolderFailed', sourceFolder, destFolder, err)); + } + }); } /** @@ -136,23 +150,30 @@ export class SshHelper { * @returns {Promise} */ async checkRemotePathExists(path: string) : Promise { - var defer = Q.defer(); + return new Promise(async (resolve, reject) => { + tl.debug(tl.loc('CheckingPathExistance', path)); - tl.debug(tl.loc('CheckingPathExistance', path)); - if(!this.sftpClient) { - defer.reject(tl.loc('ConnectionNotSetup')); - } - if (await this.sftpClient.exists(path)) { - //path exists - tl.debug(tl.loc('PathExists', path)); - defer.resolve(true); - } else { - //path does not exist - tl.debug(tl.loc('PathNotExists', path)); - defer.resolve(false); - } + if (!this.sftpClient) { + reject(tl.loc('ConnectionNotSetup')); + return; + } - return defer.promise; + if (await this.sftpClient.exists(path)) { + // path exists + tl.debug(tl.loc('PathExists', path)); + resolve(true); + } else { + // path does not exist + tl.debug(tl.loc('PathNotExists', path)); + resolve(false); + } + }); + } + + async createRemoteDirectory(path: string) { + if (!await this.sftpClient.exists(path)) { + return await this.sftpClient.mkdir(path, true); + } } /** @@ -161,58 +182,64 @@ export class SshHelper { * @param options * @returns {Promise} */ - runCommandOnRemoteMachine(command: string, options: RemoteCommandOptions) : Q.Promise { - var defer = Q.defer(); - var stdErrWritten:boolean = false; + runCommandOnRemoteMachine(command: string, options: RemoteCommandOptions) : Promise { + return new Promise((resolve, reject) => { + let stdErrWritten = false; - if(!this.sshClient) { - defer.reject(tl.loc('ConnectionNotSetup')); - } + if (!this.sshClient) { + reject(tl.loc('ConnectionNotSetup')); + return; + } - if(!options) { - tl.debug('Options not passed to runCommandOnRemoteMachine, setting defaults.'); - var options = new RemoteCommandOptions(); - options.failOnStdErr = true; - } + if (!options) { + tl.debug('Options not passed to runCommandOnRemoteMachine, setting defaults.'); + const options = new RemoteCommandOptions(); + options.failOnStdErr = true; + } - var cmdToRun = command; - if(cmdToRun.indexOf(';') > 0) { - //multiple commands were passed separated by ; - cmdToRun = cmdToRun.replace(/;/g, '\n'); - } - tl.debug('cmdToRun = ' + cmdToRun); + let cmdToRun = command; - this.sshClient.exec(cmdToRun, (err, stream) => { - if(err) { - defer.reject(tl.loc('RemoteCmdExecutionErr', cmdToRun, err)) + if (cmdToRun.indexOf(';') > 0) { + // multiple commands were passed separated by ; + cmdToRun = cmdToRun.replace(/;/g, '\n'); } - stream.on('close', (code, signal) => { - tl.debug('code = ' + code + ', signal = ' + signal); - if(code && code != 0) { - //non zero exit code - fail - defer.reject(tl.loc('RemoteCmdNonZeroExitCode', cmdToRun, code)); - } else { - //no exit code or exit code of 0 - - //based on the options decide whether to fail the build or not if data was written to STDERR - if(stdErrWritten === true && options.failOnStdErr === true) { - //stderr written - fail the build - defer.reject(tl.loc('RemoteCmdExecutionErr', cmdToRun, tl.loc('CheckLogForStdErr'))); + + tl.debug('cmdToRun = ' + cmdToRun); + + this.sshClient.exec(cmdToRun, (err, stream) => { + if (err) { + reject(tl.loc('RemoteCmdExecutionErr', cmdToRun, err)); + return; + } + + stream.on('close', (code, signal) => { + tl.debug('code = ' + code + ', signal = ' + signal); + + if (code && code != 0) { + // non zero exit code - fail + reject(tl.loc('RemoteCmdNonZeroExitCode', cmdToRun, code)); } else { - //success - defer.resolve('0'); + // no exit code or exit code of 0 + + // based on the options decide whether to fail the build or not if data was written to STDERR + if (stdErrWritten === true && options.failOnStdErr === true) { + // stderr written - fail the build + reject(tl.loc('RemoteCmdExecutionErr', cmdToRun, tl.loc('CheckLogForStdErr'))); + } else { + // success + resolve('0'); + } } - } - }).on('data', (data) => { - console.log(data); - }).stderr.on('data', (data) => { + }).on('data', (data) => { + console.log(data.toString()); + }).stderr.on('data', (data) => { stdErrWritten = true; tl.debug('stderr = ' + data); - if(data && data.toString().trim() !== '') { + if (data && data.toString().trim() !== '') { tl.error(data); } }); + }); }); - return defer.promise; } } diff --git a/Tasks/CopyFilesOverSSHV0/task.json b/Tasks/CopyFilesOverSSHV0/task.json index 42d8cd68e699..eb80c4e5b8f2 100644 --- a/Tasks/CopyFilesOverSSHV0/task.json +++ b/Tasks/CopyFilesOverSSHV0/task.json @@ -17,7 +17,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 237, + "Minor": 240, "Patch": 0 }, "demands": [], @@ -130,6 +130,24 @@ "required": false, "helpMarkDown": "Flatten the folder structure and copy all files into the specified target folder on the remote machine.", "groupName": "advanced" + }, + { + "name": "concurrentUploads", + "type": "string", + "label": "Number of concurrent uploads when copying files", + "defaultValue": "10", + "required": false, + "helpMarkDown": "Number of concurrent uploads when copying files. Default is 10.", + "groupName": "advanced" + }, + { + "name": "delayBetweenUploads", + "type": "string", + "label": "Delay between queueing uploads (in milliseconds)", + "defaultValue": "50", + "required": false, + "helpMarkDown": "Delay between queueing uploads (in milliseconds). Default is 50.", + "groupName": "advanced" } ], "execution": { @@ -153,9 +171,13 @@ "ConnectionFailed": "Failed to connect to remote machine. Verify the SSH service connection details. %s.", "ConnectionNotSetup": "SSH service connection is not set up.", "CopyCompleted": "Completed copying %s files to the remote machine.", + "CopyDirectoryCompleted": "Completed copying %s directory to the remote machine.", + "CopyDirectoryFailed": "Failed to copy %s directory to the remote machine. %s", "CopyingFiles": "Found %s files to copy to the remote machine.", "FailedOnFile": "Failed to copy %s. %s", "FileExists": "File %s cannot be copied to the remote machine because it already exists and the 'Overwrite' option is disabled.", + "FolderCreated": "Created folder %s on the remote machine.", + "FoldersCreated": "Created %s folders on the remote machine.", "NothingToCopy": "No files were found matching the patterns specified to copy to the remote machine.", "NumberFailed": "Failed to copy %d files", "RemoteCmdExecutionErr": "Command %s failed with errors on remote machine. %s.", @@ -165,9 +187,10 @@ "StartedFileCopy": "Copying file %s to %s on remote machine.", "UploadFileFailed": "Failed to upload %s to %s on remote machine. %s.", "UseDefaultPort": "Using port 22 which is the default for SSH since no port was specified.", - "TargetNotCreated": "Unable to create target folder %s.", + "TargetNotCreated": "Unable to create target folder %s. %s.", "CheckingPathExistance": "Checking if %s on the remote machine exists.", "PathExists": "%s exists on the remote machine", - "PathNotExists": "%s doesn't exist on the remote machine" + "PathNotExists": "%s doesn't exist on the remote machine", + "UploadFolderFailed": "Failed to upload folder %s to %s on remote machine. %s." } } \ No newline at end of file diff --git a/Tasks/CopyFilesOverSSHV0/task.loc.json b/Tasks/CopyFilesOverSSHV0/task.loc.json index 4f47b77ce724..889ae78de926 100644 --- a/Tasks/CopyFilesOverSSHV0/task.loc.json +++ b/Tasks/CopyFilesOverSSHV0/task.loc.json @@ -17,7 +17,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 237, + "Minor": 240, "Patch": 0 }, "demands": [], @@ -130,6 +130,24 @@ "required": false, "helpMarkDown": "ms-resource:loc.input.help.flattenFolders", "groupName": "advanced" + }, + { + "name": "concurrentUploads", + "type": "string", + "label": "ms-resource:loc.input.label.concurrentUploads", + "defaultValue": "10", + "required": false, + "helpMarkDown": "ms-resource:loc.input.help.concurrentUploads", + "groupName": "advanced" + }, + { + "name": "delayBetweenUploads", + "type": "string", + "label": "ms-resource:loc.input.label.delayBetweenUploads", + "defaultValue": "50", + "required": false, + "helpMarkDown": "ms-resource:loc.input.help.delayBetweenUploads", + "groupName": "advanced" } ], "execution": { @@ -153,9 +171,13 @@ "ConnectionFailed": "ms-resource:loc.messages.ConnectionFailed", "ConnectionNotSetup": "ms-resource:loc.messages.ConnectionNotSetup", "CopyCompleted": "ms-resource:loc.messages.CopyCompleted", + "CopyDirectoryCompleted": "ms-resource:loc.messages.CopyDirectoryCompleted", + "CopyDirectoryFailed": "ms-resource:loc.messages.CopyDirectoryFailed", "CopyingFiles": "ms-resource:loc.messages.CopyingFiles", "FailedOnFile": "ms-resource:loc.messages.FailedOnFile", "FileExists": "ms-resource:loc.messages.FileExists", + "FolderCreated": "ms-resource:loc.messages.FolderCreated", + "FoldersCreated": "ms-resource:loc.messages.FoldersCreated", "NothingToCopy": "ms-resource:loc.messages.NothingToCopy", "NumberFailed": "ms-resource:loc.messages.NumberFailed", "RemoteCmdExecutionErr": "ms-resource:loc.messages.RemoteCmdExecutionErr", @@ -168,6 +190,7 @@ "TargetNotCreated": "ms-resource:loc.messages.TargetNotCreated", "CheckingPathExistance": "ms-resource:loc.messages.CheckingPathExistance", "PathExists": "ms-resource:loc.messages.PathExists", - "PathNotExists": "ms-resource:loc.messages.PathNotExists" + "PathNotExists": "ms-resource:loc.messages.PathNotExists", + "UploadFolderFailed": "ms-resource:loc.messages.UploadFolderFailed" } } \ No newline at end of file diff --git a/Tasks/CopyFilesOverSSHV0/tsconfig.json b/Tasks/CopyFilesOverSSHV0/tsconfig.json index 0438b79f69ac..3dff88706920 100644 --- a/Tasks/CopyFilesOverSSHV0/tsconfig.json +++ b/Tasks/CopyFilesOverSSHV0/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "ES6", + "target": "ES2020", "module": "commonjs" } } \ No newline at end of file diff --git a/_generated/CopyFilesOverSSHV0.versionmap.txt b/_generated/CopyFilesOverSSHV0.versionmap.txt index 2ddd7845acbe..d31a65dfb2a9 100644 --- a/_generated/CopyFilesOverSSHV0.versionmap.txt +++ b/_generated/CopyFilesOverSSHV0.versionmap.txt @@ -1,2 +1,2 @@ -Default|0.237.0 -Node20-225|0.237.1 +Default|0.240.0 +Node20-225|0.240.1 diff --git a/_generated/CopyFilesOverSSHV0/Strings/resources.resjson/en-US/resources.resjson b/_generated/CopyFilesOverSSHV0/Strings/resources.resjson/en-US/resources.resjson index 56323de2ea28..c7755eb1f5c9 100644 --- a/_generated/CopyFilesOverSSHV0/Strings/resources.resjson/en-US/resources.resjson +++ b/_generated/CopyFilesOverSSHV0/Strings/resources.resjson/en-US/resources.resjson @@ -26,15 +26,23 @@ "loc.input.help.failOnEmptySource": "Fail if no matching files to be copied are found under the source folder.", "loc.input.label.flattenFolders": "Flatten folders", "loc.input.help.flattenFolders": "Flatten the folder structure and copy all files into the specified target folder on the remote machine.", + "loc.input.label.concurrentUploads": "Number of concurrent uploads when copying files", + "loc.input.help.concurrentUploads": "Number of concurrent uploads when copying files. Default is 10.", + "loc.input.label.delayBetweenUploads": "Delay between queueing uploads (in milliseconds)", + "loc.input.help.delayBetweenUploads": "Delay between queueing uploads (in milliseconds). Default is 50.", "loc.messages.CheckLogForStdErr": "Check the build log for STDERR from the command.", "loc.messages.CleanTargetFolder": "Cleaning target folder %s on the remote machine", "loc.messages.CleanTargetFolderFailed": "Failed to clean the target folder on the remote machine. %s", "loc.messages.ConnectionFailed": "Failed to connect to remote machine. Verify the SSH service connection details. %s.", "loc.messages.ConnectionNotSetup": "SSH service connection is not set up.", "loc.messages.CopyCompleted": "Completed copying %s files to the remote machine.", + "loc.messages.CopyDirectoryCompleted": "Completed copying %s directory to the remote machine.", + "loc.messages.CopyDirectoryFailed": "Failed to copy %s directory to the remote machine. %s", "loc.messages.CopyingFiles": "Found %s files to copy to the remote machine.", "loc.messages.FailedOnFile": "Failed to copy %s. %s", "loc.messages.FileExists": "File %s cannot be copied to the remote machine because it already exists and the 'Overwrite' option is disabled.", + "loc.messages.FolderCreated": "Created folder %s on the remote machine.", + "loc.messages.FoldersCreated": "Created %s folders on the remote machine.", "loc.messages.NothingToCopy": "No files were found matching the patterns specified to copy to the remote machine.", "loc.messages.NumberFailed": "Failed to copy %d files", "loc.messages.RemoteCmdExecutionErr": "Command %s failed with errors on remote machine. %s.", @@ -44,8 +52,9 @@ "loc.messages.StartedFileCopy": "Copying file %s to %s on remote machine.", "loc.messages.UploadFileFailed": "Failed to upload %s to %s on remote machine. %s.", "loc.messages.UseDefaultPort": "Using port 22 which is the default for SSH since no port was specified.", - "loc.messages.TargetNotCreated": "Unable to create target folder %s.", + "loc.messages.TargetNotCreated": "Unable to create target folder %s. %s.", "loc.messages.CheckingPathExistance": "Checking if %s on the remote machine exists.", "loc.messages.PathExists": "%s exists on the remote machine", - "loc.messages.PathNotExists": "%s doesn't exist on the remote machine" + "loc.messages.PathNotExists": "%s doesn't exist on the remote machine", + "loc.messages.UploadFolderFailed": "Failed to upload folder %s to %s on remote machine. %s." } \ No newline at end of file diff --git a/_generated/CopyFilesOverSSHV0/copyfilesoverssh.ts b/_generated/CopyFilesOverSSHV0/copyfilesoverssh.ts index e33a1d9f6923..5d3e5a6f87b9 100644 --- a/_generated/CopyFilesOverSSHV0/copyfilesoverssh.ts +++ b/_generated/CopyFilesOverSSHV0/copyfilesoverssh.ts @@ -4,6 +4,7 @@ import * as tl from 'azure-pipelines-task-lib/task'; import * as minimatch from 'minimatch'; import * as utils from './utils'; import { SshHelper } from './sshhelper'; +import Queue, { QueueEvents } from './queue'; // This method will find the list of matching files for the specified contents // This logic is the same as the one used by CopyFiles task except for allowing dot folders to be copied @@ -99,6 +100,235 @@ function getFilesToCopy(sourceFolder: string, contents: string[]): string[] { return files; } +function prepareFiles(filesToCopy: string[], sourceFolder: string, targetFolder: string, flattenFolders: boolean) { + return filesToCopy.map(x => { + let targetPath = path.posix.join( + targetFolder, + flattenFolders + ? path.basename(x) + : x.substring(sourceFolder.length).replace(/^\\/g, "").replace(/^\//g, "") + ); + + if (!path.isAbsolute(targetPath) && !utils.pathIsUNC(targetPath)) { + targetPath = `./${targetPath}`; + } + + return [ x, utils.unixyPath(targetPath) ]; + }); +} + +function getUniqueFolders(filesToCopy: string[]) { + const foldersSet = new Set(); + + for (const filePath of filesToCopy) { + const folderPath = path.dirname(filePath); + + if (foldersSet.has(folderPath)) { + continue; + } + + foldersSet.add(folderPath); + } + + return Array.from(foldersSet.values()); +} + +async function newRun() { + tl.setResourcePath(path.join(__dirname, 'task.json')); + + // Read SSH endpoint input + const sshEndpoint = tl.getInput('sshEndpoint', true); + const username = tl.getEndpointAuthorizationParameter(sshEndpoint, 'username', false); + // Passphrase is optional + const password = tl.getEndpointAuthorizationParameter(sshEndpoint, 'password', true); + // Private key is optional, password can be used for connecting + const privateKey = process.env['ENDPOINT_DATA_' + sshEndpoint + '_PRIVATEKEY']; + const hostname = tl.getEndpointDataParameter(sshEndpoint, 'host', false); + // Port is optional, will use 22 as default port if not specified + let port = tl.getEndpointDataParameter(sshEndpoint, 'port', true); + + if (!port) { + console.log(tl.loc('UseDefaultPort')); + port = '22'; + } + + const readyTimeout = parseInt(tl.getInput('readyTimeout', true), 10); + const useFastPut = !(process.env['USE_FAST_PUT'] === 'false'); + const concurrentUploads = parseInt(tl.getInput('concurrentUploads')); + + // Set up the SSH connection configuration based on endpoint details + let sshConfig: Object = { + host: hostname, + port: port, + username: username, + readyTimeout: readyTimeout, + useFastPut: useFastPut, + promiseLimit: isNaN(concurrentUploads) ? 10 : concurrentUploads + }; + + if (privateKey) { + tl.debug('Using private key for ssh connection.'); + + sshConfig = { + ...sshConfig, + privateKey, + passphrase: password + } + } else { + // Use password + tl.debug('Using username and password for ssh connection.'); + + sshConfig = { + ...sshConfig, + password, + } + } + + // Contents is a multiline input containing glob patterns + const contents = tl.getDelimitedInput('contents', '\n', true); + const sourceFolder = tl.getPathInput('sourceFolder', true, true); + let targetFolder = tl.getInput('targetFolder'); + + if (!targetFolder) { + targetFolder = "./"; + } else { + // '~/' is unsupported + targetFolder = targetFolder.replace(/^~\//, "./"); + } + + // Read the copy options + const cleanTargetFolder = tl.getBoolInput('cleanTargetFolder', false); + const overwrite = tl.getBoolInput('overwrite', false); + const failOnEmptySource = tl.getBoolInput('failOnEmptySource', false); + const flattenFolders = tl.getBoolInput('flattenFolders', false); + + if (!tl.stats(sourceFolder).isDirectory()) { + tl.setResult(tl.TaskResult.Failed, tl.loc('SourceNotFolder')); + return; + } + + // Initialize the SSH helpers, set up the connection + const sshHelper = new SshHelper(sshConfig); + await sshHelper.setupConnection(); + + if (cleanTargetFolder && await sshHelper.checkRemotePathExists(targetFolder)) { + console.log(tl.loc('CleanTargetFolder', targetFolder)); + const isWindowsOnTarget = tl.getBoolInput('isWindowsOnTarget', false); + const cleanHiddenFilesInTarget = tl.getBoolInput('cleanHiddenFilesInTarget', false); + const cleanTargetFolderCmd = utils.getCleanTargetFolderCmd(targetFolder, isWindowsOnTarget, cleanHiddenFilesInTarget); + + try { + await sshHelper.runCommandOnRemoteMachine(cleanTargetFolderCmd, null); + } catch (error) { + tl.setResult(tl.TaskResult.Failed, tl.loc('CleanTargetFolderFailed', error)); + tl.debug('Closing the client connection'); + await sshHelper.closeConnection(); + return; + } + } + + // If the contents were parsed into an array and the first element was set as default "**", + // then upload the entire directory + if (contents.length === 1 && contents[0] === "**") { + tl.debug("Upload a directory to a remote machine"); + + try { + const completedDirectory = await sshHelper.uploadFolder(sourceFolder, targetFolder); + tl.setResult(tl.TaskResult.Succeeded, tl.loc('CopyDirectoryCompleted', completedDirectory)); + } catch (error) { + tl.setResult(tl.TaskResult.Failed, tl.loc("CopyDirectoryFailed", sourceFolder, error)); + } + + tl.debug('Closing the client connection'); + await sshHelper.closeConnection(); + return; + } + + // Identify the files to copy + const filesToCopy = getFilesToCopy(sourceFolder, contents); + + // Copy files to remote machine + if (filesToCopy.length === 0) { + if (failOnEmptySource) { + tl.setResult(tl.TaskResult.Failed, tl.loc('NothingToCopy')); + return; + } else { + tl.warning(tl.loc('NothingToCopy')); + return; + } + } + + const preparedFiles = prepareFiles(filesToCopy, sourceFolder, targetFolder, flattenFolders); + + tl.debug(`Number of files to copy = ${preparedFiles.length}`); + tl.debug(`filesToCopy = ${preparedFiles}`); + + console.log(tl.loc('CopyingFiles', preparedFiles.length)); + + // Create remote folders structure + const folderStructure = getUniqueFolders(preparedFiles.map(x => x[1]).sort()); + + for (const foldersPath of folderStructure) { + try { + await sshHelper.createRemoteDirectory(foldersPath); + console.log(tl.loc("FolderCreated", foldersPath)); + } catch (error) { + await sshHelper.closeConnection(); + tl.setResult(tl.TaskResult.Failed, tl.loc('TargetNotCreated', foldersPath, error)); + return; + } + } + + console.log(tl.loc("FoldersCreated", folderStructure.length)); + + const delayBetweenUploads = parseInt(tl.getInput('delayBetweenUploads')); + + // Upload files to remote machine + const q = new Queue({ + concurrent: isNaN(concurrentUploads) ? 10 : concurrentUploads, + delay: isNaN(delayBetweenUploads) ? 50 : delayBetweenUploads, + }); + + q.enqueue(preparedFiles.map((pathTuple) => { + const [ filepath, targetPath ] = pathTuple; + + return { + filepath, + job: async () => { + tl.debug(`Filepath = ${filepath}`); + console.log(tl.loc('StartedFileCopy', filepath, targetPath)); + + if (!overwrite && await sshHelper.checkRemotePathExists(targetPath)) { + throw new Error(tl.loc('FileExists', targetPath)); + } + + return await sshHelper.uploadFile(filepath, targetPath); + } + }; + })); + + const errors = []; + let successfullyCopiedFilesCount = 0; + + q.on(QueueEvents.PROCESSED, () => successfullyCopiedFilesCount++); + q.on(QueueEvents.EMPTY, () => tl.debug('Queue is empty')); + q.on(QueueEvents.END, async () => { + tl.debug('End of the queue processing'); + await sshHelper.closeConnection(); + + if (errors.length === 0) { + tl.setResult(tl.TaskResult.Succeeded, tl.loc('CopyCompleted', successfullyCopiedFilesCount)); + } else { + tl.debug(`Errors count ${errors.length}`); + errors.forEach(tl.error); + tl.setResult(tl.TaskResult.Failed, tl.loc('NumberFailed', errors.length)); + } + }); + q.on(QueueEvents.ERROR, (error, filepath) => { + errors.push(tl.loc('FailedOnFile', filepath, error.message)); + }); +} + async function run() { let sshHelper: SshHelper; try { @@ -248,16 +478,20 @@ async function run() { } } -run().then(() => { +if (tl.getBoolFeatureFlag('COPYFILESOVERSSHV0_USE_QUEUE')) { + newRun(); +} else { + run().then(() => { tl.debug('Task successfully accomplished'); }) .catch(err => { tl.debug('Run was unexpectedly failed due to: ' + err); }); +} function getReadyTimeoutVariable(): number { let readyTimeoutString: string = tl.getInput('readyTimeout', true); const readyTimeout: number = parseInt(readyTimeoutString, 10); return readyTimeout; -} +} \ No newline at end of file diff --git a/_generated/CopyFilesOverSSHV0/package-lock.json b/_generated/CopyFilesOverSSHV0/package-lock.json index 7bbc70b607cd..3b17266265e2 100644 --- a/_generated/CopyFilesOverSSHV0/package-lock.json +++ b/_generated/CopyFilesOverSSHV0/package-lock.json @@ -103,12 +103,6 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" }, - "buildcheck": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", - "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", - "optional": true - }, "call-bind": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", @@ -165,13 +159,12 @@ "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" }, "cpu-features": { - "version": "0.0.9", - "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.9.tgz", - "integrity": "sha512-AKjgn2rP2yJyfbepsmLfiYcmtNn/2eUvocUyM/09yB0YDiz39HteK/5/T4Onf0pmdYDMgkBoGvRLvEguzyL7wQ==", + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.2.tgz", + "integrity": "sha512-/2yieBqvMcRj8McNzkycjW2v3OIUOibBfd2dLEJ0nWts8NobAxwiyw9phVNS6oDL8x8tz9F7uNVFEVpJncQpeA==", "optional": true, "requires": { - "buildcheck": "~0.0.6", - "nan": "^2.17.0" + "nan": "^2.14.1" } }, "delayed-stream": { @@ -347,9 +340,9 @@ } }, "nan": { - "version": "2.18.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz", - "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==", + "version": "2.19.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.19.0.tgz", + "integrity": "sha512-nO1xXxfh/RWNxfd/XPfbIfFk5vgLsAxUR9y5O0cHMJu/AW9U95JLXqthYHjEp+8gQ5p96K9jUp8nbVOxCdRbtw==", "optional": true }, "object-inspect": { @@ -488,24 +481,24 @@ } }, "ssh2": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.15.0.tgz", - "integrity": "sha512-C0PHgX4h6lBxYx7hcXwu3QWdh4tg6tZZsTfXcdvc5caW/EMxaB4H9dWsl7qk+F7LAW762hp8VbXOX7x4xUYvEw==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.4.0.tgz", + "integrity": "sha512-XvXwcXKvS452DyQvCa6Ct+chpucwc/UyxgliYz+rWXJ3jDHdtBb9xgmxJdMmnIn5bpgGAEV3KaEsH98ZGPHqwg==", "requires": { - "asn1": "^0.2.6", + "asn1": "^0.2.4", "bcrypt-pbkdf": "^1.0.2", - "cpu-features": "~0.0.9", - "nan": "^2.18.0" + "cpu-features": "0.0.2", + "nan": "^2.15.0" } }, "ssh2-sftp-client": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-9.1.0.tgz", - "integrity": "sha512-Hzdr9OE6GxZjcmyM9tgBSIFVyrHAp9c6U2Y4yBkmYOHoQvZ7pIm27dmltvcmRfxcWiIcg8HBvG5iAikDf+ZuzQ==", + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-7.0.4.tgz", + "integrity": "sha512-4fFSTgoYlzcAtGfEjiXN6N41s1jSUmPlI00f7uD7pQOjt9yK9susminINKTRvPp35dkrATrlNZVhUxNCt3z5+w==", "requires": { "concat-stream": "^2.0.0", "promise-retry": "^2.0.1", - "ssh2": "^1.12.0" + "ssh2": "^1.4.0" } }, "string_decoder": { diff --git a/_generated/CopyFilesOverSSHV0/package.json b/_generated/CopyFilesOverSSHV0/package.json index 6e94d249d358..668d6a5a2a0a 100644 --- a/_generated/CopyFilesOverSSHV0/package.json +++ b/_generated/CopyFilesOverSSHV0/package.json @@ -17,11 +17,11 @@ }, "homepage": "https://github.com/Microsoft.com/vsts-tasks#readme", "dependencies": { - "ssh2": "^1.15.0", - "ssh2-sftp-client": "^9.1.0", - "minimatch": "^3.0.4", + "ssh2": "1.4.0", + "ssh2-sftp-client": "7.0.4", + "@types/mocha": "^5.2.7", "azure-pipelines-task-lib": "^5.0.0-preview.0", - "@types/mocha": "^5.2.7" + "minimatch": "^3.0.4" }, "devDependencies": { "typescript": "4.0.2" diff --git a/_generated/CopyFilesOverSSHV0/queue.ts b/_generated/CopyFilesOverSSHV0/queue.ts new file mode 100644 index 000000000000..a9f0e30cd54f --- /dev/null +++ b/_generated/CopyFilesOverSSHV0/queue.ts @@ -0,0 +1,98 @@ +const EventEmitter = require('events'); + +type QueueOptions = { + concurrent: number; + delay: number; +} + +export enum QueueEvents { + END = 'end', + PROCESSED = 'processed', + ERROR = 'error', + EMPTY = 'empty' +} + +interface IJob { + filepath: string; + job: () => Promise; +} + +export default class Queue extends EventEmitter { + jobs: IJob[] = []; + concurrent = 10; + inProcess = 0; + delay = 50; + + constructor(options: QueueOptions) { + super(); + + this.concurrent = options.concurrent || 10; + this.delay = options.delay || 50; + } + + enqueue(jobs) { + this.jobs.push(...jobs); + this.start(); + } + + dequeue() { + return this.jobs.shift(); + } + + get isConcurrentCapacityReached() { + return this.inProcess >= this.concurrent; + } + + consume() { + this.inProcess--; + + if (this.inProcess === 0) { + this.emit(QueueEvents.EMPTY); + } + } + + waitForComplete() { + if (this.inProcess === 0) { + this.emit(QueueEvents.END); + } else { + setTimeout(() => { + this.waitForComplete(); + }, this.delay); + } + } + + start() { + if (this.jobs.length === 0) { + this.waitForComplete(); + return; + } + + if (this.isConcurrentCapacityReached) { + setTimeout(() => { + this.start(); + }, this.delay); + return; + } + + const { filepath, job } = this.dequeue(); + + if (!job) { + return; + } + + this.inProcess++; + + setTimeout(async () => { + try { + const result = await job(); + this.emit(QueueEvents.PROCESSED, result); + } catch (error) { + this.emit(QueueEvents.ERROR, error, filepath); + } + + this.consume(); + }); + + this.start(); + } +} \ No newline at end of file diff --git a/_generated/CopyFilesOverSSHV0/sshhelper.ts b/_generated/CopyFilesOverSSHV0/sshhelper.ts index 0ff955f3db03..ed6b25302af7 100644 --- a/_generated/CopyFilesOverSSHV0/sshhelper.ts +++ b/_generated/CopyFilesOverSSHV0/sshhelper.ts @@ -1,8 +1,7 @@ -import Q = require('q'); import tl = require('azure-pipelines-task-lib/task'); -const path = require('path'); var Ssh2Client = require('ssh2').Client; var SftpClient = require('ssh2-sftp-client'); +var path = require('path'); export class RemoteCommandOptions { public failOnStdErr : boolean; @@ -30,27 +29,27 @@ export class SshHelper { } private async setupSshClientConnection() : Promise { - const defer = Q.defer(); - this.sshClient = new Ssh2Client(); - this.sshClient.once('ready', () => { - defer.resolve(); - }).once('error', (err) => { - defer.reject(tl.loc('ConnectionFailed', err)); - }).connect(this.sshConfig); - await defer.promise; + return new Promise((resolve, reject) => { + this.sshClient = new Ssh2Client(); + this.sshClient.once('ready', () => { + resolve(); + }).once('error', (err) => { + reject(tl.loc('ConnectionFailed', err)); + }).connect(this.sshConfig); + }); } private async setupSftpConnection() : Promise { - const defer = Q.defer(); - try { - this.sftpClient = new SftpClient(); - await this.sftpClient.connect(this.sshConfig); - defer.resolve(); - } catch (err) { - this.sftpClient = null; - defer.reject(tl.loc('ConnectionFailed', err)); - } - await defer.promise; + return new Promise(async (resolve, reject) => { + try { + this.sftpClient = new SftpClient(); + await this.sftpClient.connect(this.sshConfig); + resolve(); + } catch (err) { + this.sftpClient = null; + reject(tl.loc('ConnectionFailed', err)); + } + }); } /** @@ -103,31 +102,46 @@ export class SshHelper { async uploadFile(sourceFile: string, dest: string) : Promise { tl.debug('Upload ' + sourceFile + ' to ' + dest + ' on remote machine.'); - var defer = Q.defer(); - if(!this.sftpClient) { - defer.reject(tl.loc('ConnectionNotSetup')); + if (!this.sftpClient) { + return Promise.reject(tl.loc('ConnectionNotSetup')); } const remotePath = path.dirname(dest); - try { - if (!await this.sftpClient.exists(remotePath)) { - await this.sftpClient.mkdir(remotePath, true); + + if (!tl.getBoolFeatureFlag('COPYFILESOVERSSHV0_USE_QUEUE')) { + try { + + if (!await this.sftpClient.exists(remotePath)) { + await this.sftpClient.mkdir(remotePath, true); + } + } catch (error) { + return Promise.reject(tl.loc('TargetNotCreated', remotePath)); } - } catch (error) { - defer.reject(tl.loc('TargetNotCreated', remotePath)); } - try { - if (this.sshConfig.useFastPut) { - await this.sftpClient.fastPut(sourceFile, dest); - } else { - await this.sftpClient.put(sourceFile, dest); - } - defer.resolve(dest); - } catch (err) { - defer.reject(tl.loc('UploadFileFailed', sourceFile, dest, err)); + if (this.sshConfig.useFastPut) { + return this.sftpClient.fastPut(sourceFile, dest); + } else { + return this.sftpClient.put(sourceFile, dest); } - return defer.promise; + } + + async uploadFolder(sourceFolder: string, destFolder: string) : Promise { + tl.debug('Upload ' + sourceFolder + ' to ' + destFolder + ' on remote machine.'); + + return new Promise(async (resolve, reject) => { + if (!this.sftpClient) { + reject(tl.loc('ConnectionNotSetup')); + return; + } + + try { + await this.sftpClient.uploadDir(sourceFolder, destFolder); + return resolve(destFolder); + } catch (err) { + reject(tl.loc('UploadFolderFailed', sourceFolder, destFolder, err)); + } + }); } /** @@ -136,23 +150,30 @@ export class SshHelper { * @returns {Promise} */ async checkRemotePathExists(path: string) : Promise { - var defer = Q.defer(); + return new Promise(async (resolve, reject) => { + tl.debug(tl.loc('CheckingPathExistance', path)); - tl.debug(tl.loc('CheckingPathExistance', path)); - if(!this.sftpClient) { - defer.reject(tl.loc('ConnectionNotSetup')); - } - if (await this.sftpClient.exists(path)) { - //path exists - tl.debug(tl.loc('PathExists', path)); - defer.resolve(true); - } else { - //path does not exist - tl.debug(tl.loc('PathNotExists', path)); - defer.resolve(false); - } + if (!this.sftpClient) { + reject(tl.loc('ConnectionNotSetup')); + return; + } - return defer.promise; + if (await this.sftpClient.exists(path)) { + // path exists + tl.debug(tl.loc('PathExists', path)); + resolve(true); + } else { + // path does not exist + tl.debug(tl.loc('PathNotExists', path)); + resolve(false); + } + }); + } + + async createRemoteDirectory(path: string) { + if (!await this.sftpClient.exists(path)) { + return await this.sftpClient.mkdir(path, true); + } } /** @@ -161,58 +182,64 @@ export class SshHelper { * @param options * @returns {Promise} */ - runCommandOnRemoteMachine(command: string, options: RemoteCommandOptions) : Q.Promise { - var defer = Q.defer(); - var stdErrWritten:boolean = false; + runCommandOnRemoteMachine(command: string, options: RemoteCommandOptions) : Promise { + return new Promise((resolve, reject) => { + let stdErrWritten = false; - if(!this.sshClient) { - defer.reject(tl.loc('ConnectionNotSetup')); - } + if (!this.sshClient) { + reject(tl.loc('ConnectionNotSetup')); + return; + } - if(!options) { - tl.debug('Options not passed to runCommandOnRemoteMachine, setting defaults.'); - var options = new RemoteCommandOptions(); - options.failOnStdErr = true; - } + if (!options) { + tl.debug('Options not passed to runCommandOnRemoteMachine, setting defaults.'); + const options = new RemoteCommandOptions(); + options.failOnStdErr = true; + } - var cmdToRun = command; - if(cmdToRun.indexOf(';') > 0) { - //multiple commands were passed separated by ; - cmdToRun = cmdToRun.replace(/;/g, '\n'); - } - tl.debug('cmdToRun = ' + cmdToRun); + let cmdToRun = command; - this.sshClient.exec(cmdToRun, (err, stream) => { - if(err) { - defer.reject(tl.loc('RemoteCmdExecutionErr', cmdToRun, err)) + if (cmdToRun.indexOf(';') > 0) { + // multiple commands were passed separated by ; + cmdToRun = cmdToRun.replace(/;/g, '\n'); } - stream.on('close', (code, signal) => { - tl.debug('code = ' + code + ', signal = ' + signal); - if(code && code != 0) { - //non zero exit code - fail - defer.reject(tl.loc('RemoteCmdNonZeroExitCode', cmdToRun, code)); - } else { - //no exit code or exit code of 0 - - //based on the options decide whether to fail the build or not if data was written to STDERR - if(stdErrWritten === true && options.failOnStdErr === true) { - //stderr written - fail the build - defer.reject(tl.loc('RemoteCmdExecutionErr', cmdToRun, tl.loc('CheckLogForStdErr'))); + + tl.debug('cmdToRun = ' + cmdToRun); + + this.sshClient.exec(cmdToRun, (err, stream) => { + if (err) { + reject(tl.loc('RemoteCmdExecutionErr', cmdToRun, err)); + return; + } + + stream.on('close', (code, signal) => { + tl.debug('code = ' + code + ', signal = ' + signal); + + if (code && code != 0) { + // non zero exit code - fail + reject(tl.loc('RemoteCmdNonZeroExitCode', cmdToRun, code)); } else { - //success - defer.resolve('0'); + // no exit code or exit code of 0 + + // based on the options decide whether to fail the build or not if data was written to STDERR + if (stdErrWritten === true && options.failOnStdErr === true) { + // stderr written - fail the build + reject(tl.loc('RemoteCmdExecutionErr', cmdToRun, tl.loc('CheckLogForStdErr'))); + } else { + // success + resolve('0'); + } } - } - }).on('data', (data) => { - console.log(data); - }).stderr.on('data', (data) => { + }).on('data', (data) => { + console.log(data.toString()); + }).stderr.on('data', (data) => { stdErrWritten = true; tl.debug('stderr = ' + data); - if(data && data.toString().trim() !== '') { + if (data && data.toString().trim() !== '') { tl.error(data); } }); + }); }); - return defer.promise; } } diff --git a/_generated/CopyFilesOverSSHV0/task.json b/_generated/CopyFilesOverSSHV0/task.json index dba00ad76d52..211e420cc279 100644 --- a/_generated/CopyFilesOverSSHV0/task.json +++ b/_generated/CopyFilesOverSSHV0/task.json @@ -17,7 +17,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 237, + "Minor": 240, "Patch": 0 }, "demands": [], @@ -130,6 +130,24 @@ "required": false, "helpMarkDown": "Flatten the folder structure and copy all files into the specified target folder on the remote machine.", "groupName": "advanced" + }, + { + "name": "concurrentUploads", + "type": "string", + "label": "Number of concurrent uploads when copying files", + "defaultValue": "10", + "required": false, + "helpMarkDown": "Number of concurrent uploads when copying files. Default is 10.", + "groupName": "advanced" + }, + { + "name": "delayBetweenUploads", + "type": "string", + "label": "Delay between queueing uploads (in milliseconds)", + "defaultValue": "50", + "required": false, + "helpMarkDown": "Delay between queueing uploads (in milliseconds). Default is 50.", + "groupName": "advanced" } ], "execution": { @@ -153,9 +171,13 @@ "ConnectionFailed": "Failed to connect to remote machine. Verify the SSH service connection details. %s.", "ConnectionNotSetup": "SSH service connection is not set up.", "CopyCompleted": "Completed copying %s files to the remote machine.", + "CopyDirectoryCompleted": "Completed copying %s directory to the remote machine.", + "CopyDirectoryFailed": "Failed to copy %s directory to the remote machine. %s", "CopyingFiles": "Found %s files to copy to the remote machine.", "FailedOnFile": "Failed to copy %s. %s", "FileExists": "File %s cannot be copied to the remote machine because it already exists and the 'Overwrite' option is disabled.", + "FolderCreated": "Created folder %s on the remote machine.", + "FoldersCreated": "Created %s folders on the remote machine.", "NothingToCopy": "No files were found matching the patterns specified to copy to the remote machine.", "NumberFailed": "Failed to copy %d files", "RemoteCmdExecutionErr": "Command %s failed with errors on remote machine. %s.", @@ -165,13 +187,14 @@ "StartedFileCopy": "Copying file %s to %s on remote machine.", "UploadFileFailed": "Failed to upload %s to %s on remote machine. %s.", "UseDefaultPort": "Using port 22 which is the default for SSH since no port was specified.", - "TargetNotCreated": "Unable to create target folder %s.", + "TargetNotCreated": "Unable to create target folder %s. %s.", "CheckingPathExistance": "Checking if %s on the remote machine exists.", "PathExists": "%s exists on the remote machine", - "PathNotExists": "%s doesn't exist on the remote machine" + "PathNotExists": "%s doesn't exist on the remote machine", + "UploadFolderFailed": "Failed to upload folder %s to %s on remote machine. %s." }, "_buildConfigMapping": { - "Default": "0.237.0", - "Node20-225": "0.237.1" + "Default": "0.240.0", + "Node20-225": "0.240.1" } } \ No newline at end of file diff --git a/_generated/CopyFilesOverSSHV0/task.loc.json b/_generated/CopyFilesOverSSHV0/task.loc.json index 0e58330cf7c2..5329ddc0775d 100644 --- a/_generated/CopyFilesOverSSHV0/task.loc.json +++ b/_generated/CopyFilesOverSSHV0/task.loc.json @@ -17,7 +17,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 237, + "Minor": 240, "Patch": 0 }, "demands": [], @@ -130,6 +130,24 @@ "required": false, "helpMarkDown": "ms-resource:loc.input.help.flattenFolders", "groupName": "advanced" + }, + { + "name": "concurrentUploads", + "type": "string", + "label": "ms-resource:loc.input.label.concurrentUploads", + "defaultValue": "10", + "required": false, + "helpMarkDown": "ms-resource:loc.input.help.concurrentUploads", + "groupName": "advanced" + }, + { + "name": "delayBetweenUploads", + "type": "string", + "label": "ms-resource:loc.input.label.delayBetweenUploads", + "defaultValue": "50", + "required": false, + "helpMarkDown": "ms-resource:loc.input.help.delayBetweenUploads", + "groupName": "advanced" } ], "execution": { @@ -153,9 +171,13 @@ "ConnectionFailed": "ms-resource:loc.messages.ConnectionFailed", "ConnectionNotSetup": "ms-resource:loc.messages.ConnectionNotSetup", "CopyCompleted": "ms-resource:loc.messages.CopyCompleted", + "CopyDirectoryCompleted": "ms-resource:loc.messages.CopyDirectoryCompleted", + "CopyDirectoryFailed": "ms-resource:loc.messages.CopyDirectoryFailed", "CopyingFiles": "ms-resource:loc.messages.CopyingFiles", "FailedOnFile": "ms-resource:loc.messages.FailedOnFile", "FileExists": "ms-resource:loc.messages.FileExists", + "FolderCreated": "ms-resource:loc.messages.FolderCreated", + "FoldersCreated": "ms-resource:loc.messages.FoldersCreated", "NothingToCopy": "ms-resource:loc.messages.NothingToCopy", "NumberFailed": "ms-resource:loc.messages.NumberFailed", "RemoteCmdExecutionErr": "ms-resource:loc.messages.RemoteCmdExecutionErr", @@ -168,10 +190,11 @@ "TargetNotCreated": "ms-resource:loc.messages.TargetNotCreated", "CheckingPathExistance": "ms-resource:loc.messages.CheckingPathExistance", "PathExists": "ms-resource:loc.messages.PathExists", - "PathNotExists": "ms-resource:loc.messages.PathNotExists" + "PathNotExists": "ms-resource:loc.messages.PathNotExists", + "UploadFolderFailed": "ms-resource:loc.messages.UploadFolderFailed" }, "_buildConfigMapping": { - "Default": "0.237.0", - "Node20-225": "0.237.1" + "Default": "0.240.0", + "Node20-225": "0.240.1" } } \ No newline at end of file diff --git a/_generated/CopyFilesOverSSHV0/tsconfig.json b/_generated/CopyFilesOverSSHV0/tsconfig.json index 0438b79f69ac..3dff88706920 100644 --- a/_generated/CopyFilesOverSSHV0/tsconfig.json +++ b/_generated/CopyFilesOverSSHV0/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "ES6", + "target": "ES2020", "module": "commonjs" } } \ No newline at end of file diff --git a/_generated/CopyFilesOverSSHV0_Node20/Strings/resources.resjson/en-US/resources.resjson b/_generated/CopyFilesOverSSHV0_Node20/Strings/resources.resjson/en-US/resources.resjson index 56323de2ea28..c7755eb1f5c9 100644 --- a/_generated/CopyFilesOverSSHV0_Node20/Strings/resources.resjson/en-US/resources.resjson +++ b/_generated/CopyFilesOverSSHV0_Node20/Strings/resources.resjson/en-US/resources.resjson @@ -26,15 +26,23 @@ "loc.input.help.failOnEmptySource": "Fail if no matching files to be copied are found under the source folder.", "loc.input.label.flattenFolders": "Flatten folders", "loc.input.help.flattenFolders": "Flatten the folder structure and copy all files into the specified target folder on the remote machine.", + "loc.input.label.concurrentUploads": "Number of concurrent uploads when copying files", + "loc.input.help.concurrentUploads": "Number of concurrent uploads when copying files. Default is 10.", + "loc.input.label.delayBetweenUploads": "Delay between queueing uploads (in milliseconds)", + "loc.input.help.delayBetweenUploads": "Delay between queueing uploads (in milliseconds). Default is 50.", "loc.messages.CheckLogForStdErr": "Check the build log for STDERR from the command.", "loc.messages.CleanTargetFolder": "Cleaning target folder %s on the remote machine", "loc.messages.CleanTargetFolderFailed": "Failed to clean the target folder on the remote machine. %s", "loc.messages.ConnectionFailed": "Failed to connect to remote machine. Verify the SSH service connection details. %s.", "loc.messages.ConnectionNotSetup": "SSH service connection is not set up.", "loc.messages.CopyCompleted": "Completed copying %s files to the remote machine.", + "loc.messages.CopyDirectoryCompleted": "Completed copying %s directory to the remote machine.", + "loc.messages.CopyDirectoryFailed": "Failed to copy %s directory to the remote machine. %s", "loc.messages.CopyingFiles": "Found %s files to copy to the remote machine.", "loc.messages.FailedOnFile": "Failed to copy %s. %s", "loc.messages.FileExists": "File %s cannot be copied to the remote machine because it already exists and the 'Overwrite' option is disabled.", + "loc.messages.FolderCreated": "Created folder %s on the remote machine.", + "loc.messages.FoldersCreated": "Created %s folders on the remote machine.", "loc.messages.NothingToCopy": "No files were found matching the patterns specified to copy to the remote machine.", "loc.messages.NumberFailed": "Failed to copy %d files", "loc.messages.RemoteCmdExecutionErr": "Command %s failed with errors on remote machine. %s.", @@ -44,8 +52,9 @@ "loc.messages.StartedFileCopy": "Copying file %s to %s on remote machine.", "loc.messages.UploadFileFailed": "Failed to upload %s to %s on remote machine. %s.", "loc.messages.UseDefaultPort": "Using port 22 which is the default for SSH since no port was specified.", - "loc.messages.TargetNotCreated": "Unable to create target folder %s.", + "loc.messages.TargetNotCreated": "Unable to create target folder %s. %s.", "loc.messages.CheckingPathExistance": "Checking if %s on the remote machine exists.", "loc.messages.PathExists": "%s exists on the remote machine", - "loc.messages.PathNotExists": "%s doesn't exist on the remote machine" + "loc.messages.PathNotExists": "%s doesn't exist on the remote machine", + "loc.messages.UploadFolderFailed": "Failed to upload folder %s to %s on remote machine. %s." } \ No newline at end of file diff --git a/_generated/CopyFilesOverSSHV0_Node20/Tests/package-lock.json b/_generated/CopyFilesOverSSHV0_Node20/Tests/package-lock.json index ba6519c9013a..d1b155f0581e 100644 --- a/_generated/CopyFilesOverSSHV0_Node20/Tests/package-lock.json +++ b/_generated/CopyFilesOverSSHV0_Node20/Tests/package-lock.json @@ -1,10 +1,18 @@ { "name": "copy-files-over-ssh-tests", "version": "1.0.0", - "lockfileVersion": 1, + "lockfileVersion": 3, "requires": true, - "dependencies": { - "@types/mocha": { + "packages": { + "": { + "name": "copy-files-over-ssh-tests", + "version": "1.0.0", + "license": "MIT", + "devDependencies": { + "@types/mocha": "^5.2.7" + } + }, + "node_modules/@types/mocha": { "version": "5.2.7", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-5.2.7.tgz", "integrity": "sha512-NYrtPht0wGzhwe9+/idPaBB+TqkY9AhTvOLMkThm0IoEfLaiVQZwBwyJ5puCkO3AUCWrmcoePjp2mbFocKy4SQ==", diff --git a/_generated/CopyFilesOverSSHV0_Node20/copyfilesoverssh.ts b/_generated/CopyFilesOverSSHV0_Node20/copyfilesoverssh.ts index e33a1d9f6923..5d3e5a6f87b9 100644 --- a/_generated/CopyFilesOverSSHV0_Node20/copyfilesoverssh.ts +++ b/_generated/CopyFilesOverSSHV0_Node20/copyfilesoverssh.ts @@ -4,6 +4,7 @@ import * as tl from 'azure-pipelines-task-lib/task'; import * as minimatch from 'minimatch'; import * as utils from './utils'; import { SshHelper } from './sshhelper'; +import Queue, { QueueEvents } from './queue'; // This method will find the list of matching files for the specified contents // This logic is the same as the one used by CopyFiles task except for allowing dot folders to be copied @@ -99,6 +100,235 @@ function getFilesToCopy(sourceFolder: string, contents: string[]): string[] { return files; } +function prepareFiles(filesToCopy: string[], sourceFolder: string, targetFolder: string, flattenFolders: boolean) { + return filesToCopy.map(x => { + let targetPath = path.posix.join( + targetFolder, + flattenFolders + ? path.basename(x) + : x.substring(sourceFolder.length).replace(/^\\/g, "").replace(/^\//g, "") + ); + + if (!path.isAbsolute(targetPath) && !utils.pathIsUNC(targetPath)) { + targetPath = `./${targetPath}`; + } + + return [ x, utils.unixyPath(targetPath) ]; + }); +} + +function getUniqueFolders(filesToCopy: string[]) { + const foldersSet = new Set(); + + for (const filePath of filesToCopy) { + const folderPath = path.dirname(filePath); + + if (foldersSet.has(folderPath)) { + continue; + } + + foldersSet.add(folderPath); + } + + return Array.from(foldersSet.values()); +} + +async function newRun() { + tl.setResourcePath(path.join(__dirname, 'task.json')); + + // Read SSH endpoint input + const sshEndpoint = tl.getInput('sshEndpoint', true); + const username = tl.getEndpointAuthorizationParameter(sshEndpoint, 'username', false); + // Passphrase is optional + const password = tl.getEndpointAuthorizationParameter(sshEndpoint, 'password', true); + // Private key is optional, password can be used for connecting + const privateKey = process.env['ENDPOINT_DATA_' + sshEndpoint + '_PRIVATEKEY']; + const hostname = tl.getEndpointDataParameter(sshEndpoint, 'host', false); + // Port is optional, will use 22 as default port if not specified + let port = tl.getEndpointDataParameter(sshEndpoint, 'port', true); + + if (!port) { + console.log(tl.loc('UseDefaultPort')); + port = '22'; + } + + const readyTimeout = parseInt(tl.getInput('readyTimeout', true), 10); + const useFastPut = !(process.env['USE_FAST_PUT'] === 'false'); + const concurrentUploads = parseInt(tl.getInput('concurrentUploads')); + + // Set up the SSH connection configuration based on endpoint details + let sshConfig: Object = { + host: hostname, + port: port, + username: username, + readyTimeout: readyTimeout, + useFastPut: useFastPut, + promiseLimit: isNaN(concurrentUploads) ? 10 : concurrentUploads + }; + + if (privateKey) { + tl.debug('Using private key for ssh connection.'); + + sshConfig = { + ...sshConfig, + privateKey, + passphrase: password + } + } else { + // Use password + tl.debug('Using username and password for ssh connection.'); + + sshConfig = { + ...sshConfig, + password, + } + } + + // Contents is a multiline input containing glob patterns + const contents = tl.getDelimitedInput('contents', '\n', true); + const sourceFolder = tl.getPathInput('sourceFolder', true, true); + let targetFolder = tl.getInput('targetFolder'); + + if (!targetFolder) { + targetFolder = "./"; + } else { + // '~/' is unsupported + targetFolder = targetFolder.replace(/^~\//, "./"); + } + + // Read the copy options + const cleanTargetFolder = tl.getBoolInput('cleanTargetFolder', false); + const overwrite = tl.getBoolInput('overwrite', false); + const failOnEmptySource = tl.getBoolInput('failOnEmptySource', false); + const flattenFolders = tl.getBoolInput('flattenFolders', false); + + if (!tl.stats(sourceFolder).isDirectory()) { + tl.setResult(tl.TaskResult.Failed, tl.loc('SourceNotFolder')); + return; + } + + // Initialize the SSH helpers, set up the connection + const sshHelper = new SshHelper(sshConfig); + await sshHelper.setupConnection(); + + if (cleanTargetFolder && await sshHelper.checkRemotePathExists(targetFolder)) { + console.log(tl.loc('CleanTargetFolder', targetFolder)); + const isWindowsOnTarget = tl.getBoolInput('isWindowsOnTarget', false); + const cleanHiddenFilesInTarget = tl.getBoolInput('cleanHiddenFilesInTarget', false); + const cleanTargetFolderCmd = utils.getCleanTargetFolderCmd(targetFolder, isWindowsOnTarget, cleanHiddenFilesInTarget); + + try { + await sshHelper.runCommandOnRemoteMachine(cleanTargetFolderCmd, null); + } catch (error) { + tl.setResult(tl.TaskResult.Failed, tl.loc('CleanTargetFolderFailed', error)); + tl.debug('Closing the client connection'); + await sshHelper.closeConnection(); + return; + } + } + + // If the contents were parsed into an array and the first element was set as default "**", + // then upload the entire directory + if (contents.length === 1 && contents[0] === "**") { + tl.debug("Upload a directory to a remote machine"); + + try { + const completedDirectory = await sshHelper.uploadFolder(sourceFolder, targetFolder); + tl.setResult(tl.TaskResult.Succeeded, tl.loc('CopyDirectoryCompleted', completedDirectory)); + } catch (error) { + tl.setResult(tl.TaskResult.Failed, tl.loc("CopyDirectoryFailed", sourceFolder, error)); + } + + tl.debug('Closing the client connection'); + await sshHelper.closeConnection(); + return; + } + + // Identify the files to copy + const filesToCopy = getFilesToCopy(sourceFolder, contents); + + // Copy files to remote machine + if (filesToCopy.length === 0) { + if (failOnEmptySource) { + tl.setResult(tl.TaskResult.Failed, tl.loc('NothingToCopy')); + return; + } else { + tl.warning(tl.loc('NothingToCopy')); + return; + } + } + + const preparedFiles = prepareFiles(filesToCopy, sourceFolder, targetFolder, flattenFolders); + + tl.debug(`Number of files to copy = ${preparedFiles.length}`); + tl.debug(`filesToCopy = ${preparedFiles}`); + + console.log(tl.loc('CopyingFiles', preparedFiles.length)); + + // Create remote folders structure + const folderStructure = getUniqueFolders(preparedFiles.map(x => x[1]).sort()); + + for (const foldersPath of folderStructure) { + try { + await sshHelper.createRemoteDirectory(foldersPath); + console.log(tl.loc("FolderCreated", foldersPath)); + } catch (error) { + await sshHelper.closeConnection(); + tl.setResult(tl.TaskResult.Failed, tl.loc('TargetNotCreated', foldersPath, error)); + return; + } + } + + console.log(tl.loc("FoldersCreated", folderStructure.length)); + + const delayBetweenUploads = parseInt(tl.getInput('delayBetweenUploads')); + + // Upload files to remote machine + const q = new Queue({ + concurrent: isNaN(concurrentUploads) ? 10 : concurrentUploads, + delay: isNaN(delayBetweenUploads) ? 50 : delayBetweenUploads, + }); + + q.enqueue(preparedFiles.map((pathTuple) => { + const [ filepath, targetPath ] = pathTuple; + + return { + filepath, + job: async () => { + tl.debug(`Filepath = ${filepath}`); + console.log(tl.loc('StartedFileCopy', filepath, targetPath)); + + if (!overwrite && await sshHelper.checkRemotePathExists(targetPath)) { + throw new Error(tl.loc('FileExists', targetPath)); + } + + return await sshHelper.uploadFile(filepath, targetPath); + } + }; + })); + + const errors = []; + let successfullyCopiedFilesCount = 0; + + q.on(QueueEvents.PROCESSED, () => successfullyCopiedFilesCount++); + q.on(QueueEvents.EMPTY, () => tl.debug('Queue is empty')); + q.on(QueueEvents.END, async () => { + tl.debug('End of the queue processing'); + await sshHelper.closeConnection(); + + if (errors.length === 0) { + tl.setResult(tl.TaskResult.Succeeded, tl.loc('CopyCompleted', successfullyCopiedFilesCount)); + } else { + tl.debug(`Errors count ${errors.length}`); + errors.forEach(tl.error); + tl.setResult(tl.TaskResult.Failed, tl.loc('NumberFailed', errors.length)); + } + }); + q.on(QueueEvents.ERROR, (error, filepath) => { + errors.push(tl.loc('FailedOnFile', filepath, error.message)); + }); +} + async function run() { let sshHelper: SshHelper; try { @@ -248,16 +478,20 @@ async function run() { } } -run().then(() => { +if (tl.getBoolFeatureFlag('COPYFILESOVERSSHV0_USE_QUEUE')) { + newRun(); +} else { + run().then(() => { tl.debug('Task successfully accomplished'); }) .catch(err => { tl.debug('Run was unexpectedly failed due to: ' + err); }); +} function getReadyTimeoutVariable(): number { let readyTimeoutString: string = tl.getInput('readyTimeout', true); const readyTimeout: number = parseInt(readyTimeoutString, 10); return readyTimeout; -} +} \ No newline at end of file diff --git a/_generated/CopyFilesOverSSHV0_Node20/package-lock.json b/_generated/CopyFilesOverSSHV0_Node20/package-lock.json index 8c721d54b2db..8c3e7035c1f5 100644 --- a/_generated/CopyFilesOverSSHV0_Node20/package-lock.json +++ b/_generated/CopyFilesOverSSHV0_Node20/package-lock.json @@ -1,230 +1,276 @@ { "name": "vsts-copyssh-task", "version": "1.0.0", - "lockfileVersion": 1, + "lockfileVersion": 3, "requires": true, - "dependencies": { - "@types/concat-stream": { + "packages": { + "": { + "name": "vsts-copyssh-task", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "@types/mocha": "^5.2.7", + "@types/node": "^20.3.1", + "azure-pipelines-task-lib": "^5.0.0-preview.0", + "minimatch": "^3.0.4", + "ssh2": "^1.4.0", + "ssh2-sftp-client": "^7.0.4" + }, + "devDependencies": { + "typescript": "5.1.6" + } + }, + "node_modules/@types/concat-stream": { "version": "1.6.1", "resolved": "https://registry.npmjs.org/@types/concat-stream/-/concat-stream-1.6.1.tgz", "integrity": "sha512-eHE4cQPoj6ngxBZMvVf6Hw7Mh4jMW4U9lpGmS5GBPB9RYxlFg+CHaVN7ErNY4W9XfLIEn20b4VDYaIrbq0q4uA==", - "requires": { + "dependencies": { "@types/node": "*" } }, - "@types/form-data": { + "node_modules/@types/form-data": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-0.0.33.tgz", "integrity": "sha512-8BSvG1kGm83cyJITQMZSulnl6QV8jqAGreJsc5tPu1Jq0vTSOiY/k24Wx82JRpWwZSqrala6sd5rWi6aNXvqcw==", - "requires": { + "dependencies": { "@types/node": "*" } }, - "@types/mocha": { + "node_modules/@types/mocha": { "version": "5.2.7", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-5.2.7.tgz", "integrity": "sha512-NYrtPht0wGzhwe9+/idPaBB+TqkY9AhTvOLMkThm0IoEfLaiVQZwBwyJ5puCkO3AUCWrmcoePjp2mbFocKy4SQ==" }, - "@types/node": { + "node_modules/@types/node": { "version": "20.7.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.7.0.tgz", "integrity": "sha512-zI22/pJW2wUZOVyguFaUL1HABdmSVxpXrzIqkjsHmyUjNhPoWM1CKfvVuXfetHhIok4RY573cqS0mZ1SJEnoTg==" }, - "@types/qs": { + "node_modules/@types/qs": { "version": "6.9.8", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.8.tgz", "integrity": "sha512-u95svzDlTysU5xecFNTgfFG5RUWu1A9P0VzgpcIiGZA9iraHOdSzcxMxQ55DyeRaGCSxQi7LxXDI4rzq/MYfdg==" }, - "asap": { + "node_modules/asap": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" }, - "asn1": { + "node_modules/asn1": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", - "requires": { + "dependencies": { "safer-buffer": "~2.1.0" } }, - "asynckit": { + "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, - "azure-pipelines-task-lib": { + "node_modules/azure-pipelines-task-lib": { "version": "5.0.0-preview.0", "resolved": "https://registry.npmjs.org/azure-pipelines-task-lib/-/azure-pipelines-task-lib-5.0.0-preview.0.tgz", "integrity": "sha512-uQJEv+q3/7RD7kgphFd33uXijaPwA1ePYoNkAgZpUhbZUtc2u4JL4ujTT/JJfgXBNW/QQNAiDQ2OC7vOQG/0tg==", - "requires": { + "dependencies": { "minimatch": "3.0.5", "q": "^1.5.1", "semver": "^5.1.0", "shelljs": "^0.8.5", "sync-request": "6.1.0", "uuid": "^3.0.1" - }, + } + }, + "node_modules/azure-pipelines-task-lib/node_modules/minimatch": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.5.tgz", + "integrity": "sha512-tUpxzX0VAzJHjLu0xUfFv1gwVp9ba3IOuRAVH2EGuRW8a5emA2FlACLqiT/lDVtS1W+TGNwqz3sWaNyLgDJWuw==", "dependencies": { - "minimatch": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.5.tgz", - "integrity": "sha512-tUpxzX0VAzJHjLu0xUfFv1gwVp9ba3IOuRAVH2EGuRW8a5emA2FlACLqiT/lDVtS1W+TGNwqz3sWaNyLgDJWuw==", - "requires": { - "brace-expansion": "^1.1.7" - } - } + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" } }, - "balanced-match": { + "node_modules/balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, - "bcrypt-pbkdf": { + "node_modules/bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", - "requires": { + "dependencies": { "tweetnacl": "^0.14.3" } }, - "brace-expansion": { + "node_modules/brace-expansion": { "version": "1.1.8", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.8.tgz", "integrity": "sha1-wHshHHyVLsH479Uad+8NHTmQopI=", - "requires": { + "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, - "buffer-from": { + "node_modules/buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" }, - "buildcheck": { + "node_modules/buildcheck": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", - "optional": true + "optional": true, + "engines": { + "node": ">=10.0.0" + } }, - "call-bind": { + "node_modules/call-bind": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", - "requires": { + "dependencies": { "function-bind": "^1.1.1", "get-intrinsic": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, - "caseless": { + "node_modules/caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==" }, - "combined-stream": { + "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "requires": { + "dependencies": { "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" } }, - "concat-map": { + "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" }, - "concat-stream": { + "node_modules/concat-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", - "requires": { + "engines": [ + "node >= 6.0" + ], + "dependencies": { "buffer-from": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.0.2", "typedarray": "^0.0.6" - }, + } + }, + "node_modules/concat-stream/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", "dependencies": { - "readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - } + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" } }, - "core-util-is": { + "node_modules/core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" }, - "cpu-features": { + "node_modules/cpu-features": { "version": "0.0.9", "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.9.tgz", "integrity": "sha512-AKjgn2rP2yJyfbepsmLfiYcmtNn/2eUvocUyM/09yB0YDiz39HteK/5/T4Onf0pmdYDMgkBoGvRLvEguzyL7wQ==", + "hasInstallScript": true, "optional": true, - "requires": { + "dependencies": { "buildcheck": "~0.0.6", "nan": "^2.17.0" + }, + "engines": { + "node": ">=10.0.0" } }, - "delayed-stream": { + "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } }, - "err-code": { + "node_modules/err-code": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==" }, - "form-data": { + "node_modules/form-data": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", - "requires": { + "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.6", "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 0.12" } }, - "fs.realpath": { + "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, - "function-bind": { + "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, - "get-intrinsic": { + "node_modules/get-intrinsic": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz", "integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==", - "requires": { + "dependencies": { "function-bind": "^1.1.1", "has": "^1.0.3", "has-proto": "^1.0.1", "has-symbols": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, - "get-port": { + "node_modules/get-port": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/get-port/-/get-port-3.2.0.tgz", - "integrity": "sha512-x5UJKlgeUiNT8nyo/AcnwLnZuZNcSjSw0kogRB+Whd1fjjFq4B1hySFxSFWWSn4mIBzg3sRNUDFYc4g5gjPoLg==" + "integrity": "sha512-x5UJKlgeUiNT8nyo/AcnwLnZuZNcSjSw0kogRB+Whd1fjjFq4B1hySFxSFWWSn4mIBzg3sRNUDFYc4g5gjPoLg==", + "engines": { + "node": ">=4" + } }, - "glob": { + "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "requires": { + "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", @@ -232,201 +278,259 @@ "once": "^1.3.0", "path-is-absolute": "^1.0.0" }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dependencies": { - "minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "requires": { - "brace-expansion": "^1.1.7" - } - } + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" } }, - "has": { + "node_modules/has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "requires": { + "dependencies": { "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" } }, - "has-proto": { + "node_modules/has-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", - "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==" + "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, - "has-symbols": { + "node_modules/has-symbols": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, - "http-basic": { + "node_modules/http-basic": { "version": "8.1.3", "resolved": "https://registry.npmjs.org/http-basic/-/http-basic-8.1.3.tgz", "integrity": "sha512-/EcDMwJZh3mABI2NhGfHOGOeOZITqfkEO4p/xK+l3NpyncIHUQBoMvCSF/b5GqvKtySC2srL/GGG3+EtlqlmCw==", - "requires": { + "dependencies": { "caseless": "^0.12.0", "concat-stream": "^1.6.2", "http-response-object": "^3.0.1", "parse-cache-control": "^1.0.1" }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/http-basic/node_modules/concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "engines": [ + "node >= 0.8" + ], "dependencies": { - "concat-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", - "requires": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^2.2.2", - "typedarray": "^0.0.6" - } - } + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" } }, - "http-response-object": { + "node_modules/http-response-object": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/http-response-object/-/http-response-object-3.0.2.tgz", "integrity": "sha512-bqX0XTF6fnXSQcEJ2Iuyr75yVakyjIDCqroJQ/aHfSdlM743Cwqoi2nDYMzLGWUcuTWGWy8AAvOKXTfiv6q9RA==", - "requires": { - "@types/node": "^10.0.3" - }, "dependencies": { - "@types/node": { - "version": "10.17.60", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", - "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==" - } + "@types/node": "^10.0.3" } }, - "inflight": { + "node_modules/http-response-object/node_modules/@types/node": { + "version": "10.17.60", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", + "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==" + }, + "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "requires": { + "dependencies": { "once": "^1.3.0", "wrappy": "1" } }, - "inherits": { + "node_modules/inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, - "interpret": { + "node_modules/interpret": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", - "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==" + "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", + "engines": { + "node": ">= 0.10" + } }, - "is-core-module": { + "node_modules/is-core-module": { "version": "2.13.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.0.tgz", "integrity": "sha512-Z7dk6Qo8pOCp3l4tsX2C5ZVas4V+UxwQodwZhLopL91TX8UyyHEXafPcyoeeWuLrwzHcr3igO78wNLwHJHsMCQ==", - "requires": { + "dependencies": { "has": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, - "isarray": { + "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" }, - "mime-db": { + "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } }, - "mime-types": { + "node_modules/mime-types": { "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "requires": { + "dependencies": { "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" } }, - "minimatch": { + "node_modules/minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha1-UWbihkV/AzBgZL5Ul+jbsMPTIIM=", - "requires": { + "dependencies": { "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" } }, - "nan": { + "node_modules/nan": { "version": "2.18.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz", "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==", "optional": true }, - "object-inspect": { + "node_modules/object-inspect": { "version": "1.12.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", - "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==" + "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, - "once": { + "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "requires": { + "dependencies": { "wrappy": "1" } }, - "parse-cache-control": { + "node_modules/parse-cache-control": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parse-cache-control/-/parse-cache-control-1.0.1.tgz", "integrity": "sha512-60zvsJReQPX5/QP0Kzfd/VrpjScIQ7SHBW6bFCYfEP+fp0Eppr1SHhIO5nd1PjZtvclzSzES9D/p5nFJurwfWg==" }, - "path-is-absolute": { + "node_modules/path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "engines": { + "node": ">=0.10.0" + } }, - "path-parse": { + "node_modules/path-parse": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, - "process-nextick-args": { + "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, - "promise": { + "node_modules/promise": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/promise/-/promise-8.3.0.tgz", "integrity": "sha512-rZPNPKTOYVNEEKFaq1HqTgOwZD+4/YHS5ukLzQCypkj+OkYx7iv0mA91lJlpPPZ8vMau3IIGj5Qlwrx+8iiSmg==", - "requires": { + "dependencies": { "asap": "~2.0.6" } }, - "promise-retry": { + "node_modules/promise-retry": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", - "requires": { + "dependencies": { "err-code": "^2.0.2", "retry": "^0.12.0" + }, + "engines": { + "node": ">=10" } }, - "q": { + "node_modules/q": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", - "integrity": "sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==" + "integrity": "sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==", + "engines": { + "node": ">=0.6.0", + "teleport": ">=0.2.0" + } }, - "qs": { + "node_modules/qs": { "version": "6.11.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz", "integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==", - "requires": { + "dependencies": { "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, - "readable-stream": { + "node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", - "requires": { + "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", @@ -436,128 +540,168 @@ "util-deprecate": "~1.0.1" } }, - "rechoir": { + "node_modules/rechoir": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==", - "requires": { + "dependencies": { "resolve": "^1.1.6" + }, + "engines": { + "node": ">= 0.10" } }, - "resolve": { + "node_modules/resolve": { "version": "1.22.6", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.6.tgz", "integrity": "sha512-njhxM7mV12JfufShqGy3Rz8j11RPdLy4xi15UurGJeoHLfJpVXKdh3ueuOqbYUcDZnffr6X739JBo5LzyahEsw==", - "requires": { + "dependencies": { "is-core-module": "^2.13.0", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, - "retry": { + "node_modules/retry": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==" + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "engines": { + "node": ">= 4" + } }, - "safe-buffer": { + "node_modules/safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, - "safer-buffer": { + "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "semver": { + "node_modules/semver": { "version": "5.7.2", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", - "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==" + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", + "bin": { + "semver": "bin/semver" + } }, - "shelljs": { + "node_modules/shelljs": { "version": "0.8.5", "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", - "requires": { + "dependencies": { "glob": "^7.0.0", "interpret": "^1.0.0", "rechoir": "^0.6.2" + }, + "bin": { + "shjs": "bin/shjs" + }, + "engines": { + "node": ">=4" } }, - "side-channel": { + "node_modules/side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", - "requires": { + "dependencies": { "call-bind": "^1.0.0", "get-intrinsic": "^1.0.2", "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, - "ssh2": { + "node_modules/ssh2": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.15.0.tgz", "integrity": "sha512-C0PHgX4h6lBxYx7hcXwu3QWdh4tg6tZZsTfXcdvc5caW/EMxaB4H9dWsl7qk+F7LAW762hp8VbXOX7x4xUYvEw==", - "requires": { + "hasInstallScript": true, + "dependencies": { "asn1": "^0.2.6", - "bcrypt-pbkdf": "^1.0.2", + "bcrypt-pbkdf": "^1.0.2" + }, + "engines": { + "node": ">=10.16.0" + }, + "optionalDependencies": { "cpu-features": "~0.0.9", "nan": "^2.18.0" } }, - "ssh2-sftp-client": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-9.1.0.tgz", - "integrity": "sha512-Hzdr9OE6GxZjcmyM9tgBSIFVyrHAp9c6U2Y4yBkmYOHoQvZ7pIm27dmltvcmRfxcWiIcg8HBvG5iAikDf+ZuzQ==", - "requires": { + "node_modules/ssh2-sftp-client": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-7.2.3.tgz", + "integrity": "sha512-Bmq4Uewu3e0XOwu5bnPbiS5KRQYv+dff5H6+85V4GZrPrt0Fkt1nUH+uXanyAkoNxUpzjnAPEEoLdOaBO9c3xw==", + "dependencies": { "concat-stream": "^2.0.0", "promise-retry": "^2.0.1", - "ssh2": "^1.12.0" + "ssh2": "^1.8.0" + }, + "engines": { + "node": ">=10.24.1" } }, - "string_decoder": { + "node_modules/string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - }, "dependencies": { - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - } + "safe-buffer": "~5.1.0" } }, - "supports-preserve-symlinks-flag": { + "node_modules/string_decoder/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/supports-preserve-symlinks-flag": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", - "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==" + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, - "sync-request": { + "node_modules/sync-request": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/sync-request/-/sync-request-6.1.0.tgz", "integrity": "sha512-8fjNkrNlNCrVc/av+Jn+xxqfCjYaBoHqCsDz6mt030UMxJGr+GSfCV1dQt2gRtlL63+VPidwDVLr7V2OcTSdRw==", - "requires": { + "dependencies": { "http-response-object": "^3.0.1", "sync-rpc": "^1.2.1", "then-request": "^6.0.0" + }, + "engines": { + "node": ">=8.0.0" } }, - "sync-rpc": { + "node_modules/sync-rpc": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/sync-rpc/-/sync-rpc-1.3.6.tgz", "integrity": "sha512-J8jTXuZzRlvU7HemDgHi3pGnh/rkoqR/OZSjhTyyZrEkkYQbk7Z33AXp37mkPfPpfdOuj7Ex3H/TJM1z48uPQw==", - "requires": { + "dependencies": { "get-port": "^3.1.0" } }, - "then-request": { + "node_modules/then-request": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/then-request/-/then-request-6.0.2.tgz", "integrity": "sha512-3ZBiG7JvP3wbDzA9iNY5zJQcHL4jn/0BWtXIkagfz7QgOL/LqjCEOBQuJNZfu0XYnv5JhKh+cDxCPM4ILrqruA==", - "requires": { + "dependencies": { "@types/concat-stream": "^1.6.0", "@types/form-data": "0.0.33", "@types/node": "^8.0.0", @@ -570,52 +714,67 @@ "promise": "^8.0.0", "qs": "^6.4.0" }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/then-request/node_modules/@types/node": { + "version": "8.10.66", + "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.66.tgz", + "integrity": "sha512-tktOkFUA4kXx2hhhrB8bIFb5TbwzS4uOhKEmwiD+NoiL0qtP2OQ9mFldbgD4dV1djrlBYP6eBuQZiWjuHUpqFw==" + }, + "node_modules/then-request/node_modules/concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "engines": [ + "node >= 0.8" + ], "dependencies": { - "@types/node": { - "version": "8.10.66", - "resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.66.tgz", - "integrity": "sha512-tktOkFUA4kXx2hhhrB8bIFb5TbwzS4uOhKEmwiD+NoiL0qtP2OQ9mFldbgD4dV1djrlBYP6eBuQZiWjuHUpqFw==" - }, - "concat-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", - "requires": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^2.2.2", - "typedarray": "^0.0.6" - } - } - } - }, - "tweetnacl": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + } + }, + "node_modules/tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" }, - "typedarray": { + "node_modules/typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==" }, - "typescript": { + "node_modules/typescript": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.6.tgz", "integrity": "sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==", - "dev": true + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } }, - "util-deprecate": { + "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, - "uuid": { + "node_modules/uuid": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==" + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "bin": { + "uuid": "bin/uuid" + } }, - "wrappy": { + "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" diff --git a/_generated/CopyFilesOverSSHV0_Node20/package.json b/_generated/CopyFilesOverSSHV0_Node20/package.json index 960ad8d5df3f..c96ccab01154 100644 --- a/_generated/CopyFilesOverSSHV0_Node20/package.json +++ b/_generated/CopyFilesOverSSHV0_Node20/package.json @@ -17,8 +17,8 @@ }, "homepage": "https://github.com/Microsoft.com/vsts-tasks#readme", "dependencies": { - "ssh2": "^1.15.0", - "ssh2-sftp-client": "^9.1.0", + "ssh2": "^1.4.0", + "ssh2-sftp-client": "^7.0.4", "minimatch": "^3.0.4", "azure-pipelines-task-lib": "^5.0.0-preview.0", "@types/mocha": "^5.2.7", diff --git a/_generated/CopyFilesOverSSHV0_Node20/queue.ts b/_generated/CopyFilesOverSSHV0_Node20/queue.ts new file mode 100644 index 000000000000..a9f0e30cd54f --- /dev/null +++ b/_generated/CopyFilesOverSSHV0_Node20/queue.ts @@ -0,0 +1,98 @@ +const EventEmitter = require('events'); + +type QueueOptions = { + concurrent: number; + delay: number; +} + +export enum QueueEvents { + END = 'end', + PROCESSED = 'processed', + ERROR = 'error', + EMPTY = 'empty' +} + +interface IJob { + filepath: string; + job: () => Promise; +} + +export default class Queue extends EventEmitter { + jobs: IJob[] = []; + concurrent = 10; + inProcess = 0; + delay = 50; + + constructor(options: QueueOptions) { + super(); + + this.concurrent = options.concurrent || 10; + this.delay = options.delay || 50; + } + + enqueue(jobs) { + this.jobs.push(...jobs); + this.start(); + } + + dequeue() { + return this.jobs.shift(); + } + + get isConcurrentCapacityReached() { + return this.inProcess >= this.concurrent; + } + + consume() { + this.inProcess--; + + if (this.inProcess === 0) { + this.emit(QueueEvents.EMPTY); + } + } + + waitForComplete() { + if (this.inProcess === 0) { + this.emit(QueueEvents.END); + } else { + setTimeout(() => { + this.waitForComplete(); + }, this.delay); + } + } + + start() { + if (this.jobs.length === 0) { + this.waitForComplete(); + return; + } + + if (this.isConcurrentCapacityReached) { + setTimeout(() => { + this.start(); + }, this.delay); + return; + } + + const { filepath, job } = this.dequeue(); + + if (!job) { + return; + } + + this.inProcess++; + + setTimeout(async () => { + try { + const result = await job(); + this.emit(QueueEvents.PROCESSED, result); + } catch (error) { + this.emit(QueueEvents.ERROR, error, filepath); + } + + this.consume(); + }); + + this.start(); + } +} \ No newline at end of file diff --git a/_generated/CopyFilesOverSSHV0_Node20/sshhelper.ts b/_generated/CopyFilesOverSSHV0_Node20/sshhelper.ts index 0ff955f3db03..ed6b25302af7 100644 --- a/_generated/CopyFilesOverSSHV0_Node20/sshhelper.ts +++ b/_generated/CopyFilesOverSSHV0_Node20/sshhelper.ts @@ -1,8 +1,7 @@ -import Q = require('q'); import tl = require('azure-pipelines-task-lib/task'); -const path = require('path'); var Ssh2Client = require('ssh2').Client; var SftpClient = require('ssh2-sftp-client'); +var path = require('path'); export class RemoteCommandOptions { public failOnStdErr : boolean; @@ -30,27 +29,27 @@ export class SshHelper { } private async setupSshClientConnection() : Promise { - const defer = Q.defer(); - this.sshClient = new Ssh2Client(); - this.sshClient.once('ready', () => { - defer.resolve(); - }).once('error', (err) => { - defer.reject(tl.loc('ConnectionFailed', err)); - }).connect(this.sshConfig); - await defer.promise; + return new Promise((resolve, reject) => { + this.sshClient = new Ssh2Client(); + this.sshClient.once('ready', () => { + resolve(); + }).once('error', (err) => { + reject(tl.loc('ConnectionFailed', err)); + }).connect(this.sshConfig); + }); } private async setupSftpConnection() : Promise { - const defer = Q.defer(); - try { - this.sftpClient = new SftpClient(); - await this.sftpClient.connect(this.sshConfig); - defer.resolve(); - } catch (err) { - this.sftpClient = null; - defer.reject(tl.loc('ConnectionFailed', err)); - } - await defer.promise; + return new Promise(async (resolve, reject) => { + try { + this.sftpClient = new SftpClient(); + await this.sftpClient.connect(this.sshConfig); + resolve(); + } catch (err) { + this.sftpClient = null; + reject(tl.loc('ConnectionFailed', err)); + } + }); } /** @@ -103,31 +102,46 @@ export class SshHelper { async uploadFile(sourceFile: string, dest: string) : Promise { tl.debug('Upload ' + sourceFile + ' to ' + dest + ' on remote machine.'); - var defer = Q.defer(); - if(!this.sftpClient) { - defer.reject(tl.loc('ConnectionNotSetup')); + if (!this.sftpClient) { + return Promise.reject(tl.loc('ConnectionNotSetup')); } const remotePath = path.dirname(dest); - try { - if (!await this.sftpClient.exists(remotePath)) { - await this.sftpClient.mkdir(remotePath, true); + + if (!tl.getBoolFeatureFlag('COPYFILESOVERSSHV0_USE_QUEUE')) { + try { + + if (!await this.sftpClient.exists(remotePath)) { + await this.sftpClient.mkdir(remotePath, true); + } + } catch (error) { + return Promise.reject(tl.loc('TargetNotCreated', remotePath)); } - } catch (error) { - defer.reject(tl.loc('TargetNotCreated', remotePath)); } - try { - if (this.sshConfig.useFastPut) { - await this.sftpClient.fastPut(sourceFile, dest); - } else { - await this.sftpClient.put(sourceFile, dest); - } - defer.resolve(dest); - } catch (err) { - defer.reject(tl.loc('UploadFileFailed', sourceFile, dest, err)); + if (this.sshConfig.useFastPut) { + return this.sftpClient.fastPut(sourceFile, dest); + } else { + return this.sftpClient.put(sourceFile, dest); } - return defer.promise; + } + + async uploadFolder(sourceFolder: string, destFolder: string) : Promise { + tl.debug('Upload ' + sourceFolder + ' to ' + destFolder + ' on remote machine.'); + + return new Promise(async (resolve, reject) => { + if (!this.sftpClient) { + reject(tl.loc('ConnectionNotSetup')); + return; + } + + try { + await this.sftpClient.uploadDir(sourceFolder, destFolder); + return resolve(destFolder); + } catch (err) { + reject(tl.loc('UploadFolderFailed', sourceFolder, destFolder, err)); + } + }); } /** @@ -136,23 +150,30 @@ export class SshHelper { * @returns {Promise} */ async checkRemotePathExists(path: string) : Promise { - var defer = Q.defer(); + return new Promise(async (resolve, reject) => { + tl.debug(tl.loc('CheckingPathExistance', path)); - tl.debug(tl.loc('CheckingPathExistance', path)); - if(!this.sftpClient) { - defer.reject(tl.loc('ConnectionNotSetup')); - } - if (await this.sftpClient.exists(path)) { - //path exists - tl.debug(tl.loc('PathExists', path)); - defer.resolve(true); - } else { - //path does not exist - tl.debug(tl.loc('PathNotExists', path)); - defer.resolve(false); - } + if (!this.sftpClient) { + reject(tl.loc('ConnectionNotSetup')); + return; + } - return defer.promise; + if (await this.sftpClient.exists(path)) { + // path exists + tl.debug(tl.loc('PathExists', path)); + resolve(true); + } else { + // path does not exist + tl.debug(tl.loc('PathNotExists', path)); + resolve(false); + } + }); + } + + async createRemoteDirectory(path: string) { + if (!await this.sftpClient.exists(path)) { + return await this.sftpClient.mkdir(path, true); + } } /** @@ -161,58 +182,64 @@ export class SshHelper { * @param options * @returns {Promise} */ - runCommandOnRemoteMachine(command: string, options: RemoteCommandOptions) : Q.Promise { - var defer = Q.defer(); - var stdErrWritten:boolean = false; + runCommandOnRemoteMachine(command: string, options: RemoteCommandOptions) : Promise { + return new Promise((resolve, reject) => { + let stdErrWritten = false; - if(!this.sshClient) { - defer.reject(tl.loc('ConnectionNotSetup')); - } + if (!this.sshClient) { + reject(tl.loc('ConnectionNotSetup')); + return; + } - if(!options) { - tl.debug('Options not passed to runCommandOnRemoteMachine, setting defaults.'); - var options = new RemoteCommandOptions(); - options.failOnStdErr = true; - } + if (!options) { + tl.debug('Options not passed to runCommandOnRemoteMachine, setting defaults.'); + const options = new RemoteCommandOptions(); + options.failOnStdErr = true; + } - var cmdToRun = command; - if(cmdToRun.indexOf(';') > 0) { - //multiple commands were passed separated by ; - cmdToRun = cmdToRun.replace(/;/g, '\n'); - } - tl.debug('cmdToRun = ' + cmdToRun); + let cmdToRun = command; - this.sshClient.exec(cmdToRun, (err, stream) => { - if(err) { - defer.reject(tl.loc('RemoteCmdExecutionErr', cmdToRun, err)) + if (cmdToRun.indexOf(';') > 0) { + // multiple commands were passed separated by ; + cmdToRun = cmdToRun.replace(/;/g, '\n'); } - stream.on('close', (code, signal) => { - tl.debug('code = ' + code + ', signal = ' + signal); - if(code && code != 0) { - //non zero exit code - fail - defer.reject(tl.loc('RemoteCmdNonZeroExitCode', cmdToRun, code)); - } else { - //no exit code or exit code of 0 - - //based on the options decide whether to fail the build or not if data was written to STDERR - if(stdErrWritten === true && options.failOnStdErr === true) { - //stderr written - fail the build - defer.reject(tl.loc('RemoteCmdExecutionErr', cmdToRun, tl.loc('CheckLogForStdErr'))); + + tl.debug('cmdToRun = ' + cmdToRun); + + this.sshClient.exec(cmdToRun, (err, stream) => { + if (err) { + reject(tl.loc('RemoteCmdExecutionErr', cmdToRun, err)); + return; + } + + stream.on('close', (code, signal) => { + tl.debug('code = ' + code + ', signal = ' + signal); + + if (code && code != 0) { + // non zero exit code - fail + reject(tl.loc('RemoteCmdNonZeroExitCode', cmdToRun, code)); } else { - //success - defer.resolve('0'); + // no exit code or exit code of 0 + + // based on the options decide whether to fail the build or not if data was written to STDERR + if (stdErrWritten === true && options.failOnStdErr === true) { + // stderr written - fail the build + reject(tl.loc('RemoteCmdExecutionErr', cmdToRun, tl.loc('CheckLogForStdErr'))); + } else { + // success + resolve('0'); + } } - } - }).on('data', (data) => { - console.log(data); - }).stderr.on('data', (data) => { + }).on('data', (data) => { + console.log(data.toString()); + }).stderr.on('data', (data) => { stdErrWritten = true; tl.debug('stderr = ' + data); - if(data && data.toString().trim() !== '') { + if (data && data.toString().trim() !== '') { tl.error(data); } }); + }); }); - return defer.promise; } } diff --git a/_generated/CopyFilesOverSSHV0_Node20/task.json b/_generated/CopyFilesOverSSHV0_Node20/task.json index fc42f06f04c1..dd17e048b42e 100644 --- a/_generated/CopyFilesOverSSHV0_Node20/task.json +++ b/_generated/CopyFilesOverSSHV0_Node20/task.json @@ -17,7 +17,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 237, + "Minor": 240, "Patch": 1 }, "demands": [], @@ -130,6 +130,24 @@ "required": false, "helpMarkDown": "Flatten the folder structure and copy all files into the specified target folder on the remote machine.", "groupName": "advanced" + }, + { + "name": "concurrentUploads", + "type": "string", + "label": "Number of concurrent uploads when copying files", + "defaultValue": "10", + "required": false, + "helpMarkDown": "Number of concurrent uploads when copying files. Default is 10.", + "groupName": "advanced" + }, + { + "name": "delayBetweenUploads", + "type": "string", + "label": "Delay between queueing uploads (in milliseconds)", + "defaultValue": "50", + "required": false, + "helpMarkDown": "Delay between queueing uploads (in milliseconds). Default is 50.", + "groupName": "advanced" } ], "execution": { @@ -157,9 +175,13 @@ "ConnectionFailed": "Failed to connect to remote machine. Verify the SSH service connection details. %s.", "ConnectionNotSetup": "SSH service connection is not set up.", "CopyCompleted": "Completed copying %s files to the remote machine.", + "CopyDirectoryCompleted": "Completed copying %s directory to the remote machine.", + "CopyDirectoryFailed": "Failed to copy %s directory to the remote machine. %s", "CopyingFiles": "Found %s files to copy to the remote machine.", "FailedOnFile": "Failed to copy %s. %s", "FileExists": "File %s cannot be copied to the remote machine because it already exists and the 'Overwrite' option is disabled.", + "FolderCreated": "Created folder %s on the remote machine.", + "FoldersCreated": "Created %s folders on the remote machine.", "NothingToCopy": "No files were found matching the patterns specified to copy to the remote machine.", "NumberFailed": "Failed to copy %d files", "RemoteCmdExecutionErr": "Command %s failed with errors on remote machine. %s.", @@ -169,13 +191,14 @@ "StartedFileCopy": "Copying file %s to %s on remote machine.", "UploadFileFailed": "Failed to upload %s to %s on remote machine. %s.", "UseDefaultPort": "Using port 22 which is the default for SSH since no port was specified.", - "TargetNotCreated": "Unable to create target folder %s.", + "TargetNotCreated": "Unable to create target folder %s. %s.", "CheckingPathExistance": "Checking if %s on the remote machine exists.", "PathExists": "%s exists on the remote machine", - "PathNotExists": "%s doesn't exist on the remote machine" + "PathNotExists": "%s doesn't exist on the remote machine", + "UploadFolderFailed": "Failed to upload folder %s to %s on remote machine. %s." }, "_buildConfigMapping": { - "Default": "0.237.0", - "Node20-225": "0.237.1" + "Default": "0.240.0", + "Node20-225": "0.240.1" } } \ No newline at end of file diff --git a/_generated/CopyFilesOverSSHV0_Node20/task.loc.json b/_generated/CopyFilesOverSSHV0_Node20/task.loc.json index 7b92827e839c..61257baa31e3 100644 --- a/_generated/CopyFilesOverSSHV0_Node20/task.loc.json +++ b/_generated/CopyFilesOverSSHV0_Node20/task.loc.json @@ -17,7 +17,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 237, + "Minor": 240, "Patch": 1 }, "demands": [], @@ -130,6 +130,24 @@ "required": false, "helpMarkDown": "ms-resource:loc.input.help.flattenFolders", "groupName": "advanced" + }, + { + "name": "concurrentUploads", + "type": "string", + "label": "ms-resource:loc.input.label.concurrentUploads", + "defaultValue": "10", + "required": false, + "helpMarkDown": "ms-resource:loc.input.help.concurrentUploads", + "groupName": "advanced" + }, + { + "name": "delayBetweenUploads", + "type": "string", + "label": "ms-resource:loc.input.label.delayBetweenUploads", + "defaultValue": "50", + "required": false, + "helpMarkDown": "ms-resource:loc.input.help.delayBetweenUploads", + "groupName": "advanced" } ], "execution": { @@ -157,9 +175,13 @@ "ConnectionFailed": "ms-resource:loc.messages.ConnectionFailed", "ConnectionNotSetup": "ms-resource:loc.messages.ConnectionNotSetup", "CopyCompleted": "ms-resource:loc.messages.CopyCompleted", + "CopyDirectoryCompleted": "ms-resource:loc.messages.CopyDirectoryCompleted", + "CopyDirectoryFailed": "ms-resource:loc.messages.CopyDirectoryFailed", "CopyingFiles": "ms-resource:loc.messages.CopyingFiles", "FailedOnFile": "ms-resource:loc.messages.FailedOnFile", "FileExists": "ms-resource:loc.messages.FileExists", + "FolderCreated": "ms-resource:loc.messages.FolderCreated", + "FoldersCreated": "ms-resource:loc.messages.FoldersCreated", "NothingToCopy": "ms-resource:loc.messages.NothingToCopy", "NumberFailed": "ms-resource:loc.messages.NumberFailed", "RemoteCmdExecutionErr": "ms-resource:loc.messages.RemoteCmdExecutionErr", @@ -172,10 +194,11 @@ "TargetNotCreated": "ms-resource:loc.messages.TargetNotCreated", "CheckingPathExistance": "ms-resource:loc.messages.CheckingPathExistance", "PathExists": "ms-resource:loc.messages.PathExists", - "PathNotExists": "ms-resource:loc.messages.PathNotExists" + "PathNotExists": "ms-resource:loc.messages.PathNotExists", + "UploadFolderFailed": "ms-resource:loc.messages.UploadFolderFailed" }, "_buildConfigMapping": { - "Default": "0.237.0", - "Node20-225": "0.237.1" + "Default": "0.240.0", + "Node20-225": "0.240.1" } } \ No newline at end of file diff --git a/_generated/CopyFilesOverSSHV0_Node20/tsconfig.json b/_generated/CopyFilesOverSSHV0_Node20/tsconfig.json index 0438b79f69ac..3dff88706920 100644 --- a/_generated/CopyFilesOverSSHV0_Node20/tsconfig.json +++ b/_generated/CopyFilesOverSSHV0_Node20/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "ES6", + "target": "ES2020", "module": "commonjs" } } \ No newline at end of file