fix: stateless s3 storage for verdaccio/verdaccio/issues/1595

favoyang · Nov 27, 2019 · 46d1394 · 46d1394
1 parent e309b93
commit 46d1394
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 41 deletions.
diff --git a/plugins/aws-s3-storage/package.json b/plugins/aws-s3-storage/package.json
@@ -30,7 +30,8 @@
   "dependencies": {
     "@verdaccio/commons-api": "^0.1.2",
     "@verdaccio/streams": "^2.0.0",
-    "aws-sdk": "2.567.0"
+    "aws-sdk": "2.567.0",
+    "s3-ls": "^3.0.0"
   },
   "devDependencies": {
     "@verdaccio/babel-preset": "^8.4.2",

diff --git a/plugins/aws-s3-storage/src/index.ts b/plugins/aws-s3-storage/src/index.ts
@@ -10,6 +10,7 @@ import {
 } from '@verdaccio/types';
 import { getInternalError, VerdaccioError, getServiceUnavailable } from '@verdaccio/commons-api';
 import { S3 } from 'aws-sdk';
+import s3ls from 's3-ls';
 
 import { S3Config } from './config';
 import S3PackageManager from './s3PackageManager';
@@ -58,26 +59,14 @@ export default class S3Database implements IPluginStorage<S3Config> {
 
   public add(name: string, callback: Callback): void {
     this.logger.debug({ name }, 's3: [add] private package @{name}');
-    this._getData().then(async data => {
-      if (data.list.indexOf(name) === -1) {
-        data.list.push(name);
-        this.logger.trace({ name }, 's3: [add] @{name} has been added');
-        try {
-          await this._sync();
-          callback(null);
-        } catch (err) {
-          callback(err);
-        }
-      } else {
-        callback(null);
-      }
-    });
+    // s3 backend doesn't need explicitly add a package
+    callback(null);
   }
 
   public async search(onPackage: Function, onEnd: Function): Promise<void> {
     this.logger.debug('s3: [search]');
-    const storage = await this._getData();
-    const storageInfoMap = storage.list.map(this._fetchPackageInfo.bind(this, onPackage));
+    const packageList = await this._getPackageList();
+    const storageInfoMap = packageList.map(this._fetchPackageInfo.bind(this, onPackage));
     this.logger.debug({ l: storageInfoMap.length }, 's3: [search] storageInfoMap length is @{l}');
     await Promise.all(storageInfoMap);
     onEnd();
@@ -118,34 +107,13 @@ export default class S3Database implements IPluginStorage<S3Config> {
 
   public remove(name: string, callback: Callback): void {
     this.logger.debug({ name }, 's3: [remove] @{name}');
-    this.get(async (err, data) => {
-      if (err) {
-        this.logger.error({ err }, 's3: [remove] error: @{err}');
-        callback(getInternalError('something went wrong on remove a package'));
-      }
-
-      const pkgName = data.indexOf(name);
-      if (pkgName !== -1) {
-        const data = await this._getData();
-        data.list.splice(pkgName, 1);
-        this.logger.debug({ pkgName }, 's3: [remove] sucessfully removed @{pkgName}');
-      }
-
-      try {
-        this.logger.trace('s3: [remove] starting sync');
-        await this._sync();
-        this.logger.trace('s3: [remove] finish sync');
-        callback(null);
-      } catch (err) {
-        this.logger.error({ err }, 's3: [remove] sync error: @{err}');
-        callback(err);
-      }
-    });
+    // s3 backend doesn't need explicitly remove a package
+    callback(null);
   }
 
   public get(callback: Callback): void {
     this.logger.debug('s3: [get]');
-    this._getData().then(data => callback(null, data.list));
+    this._getPackageList().then(list => callback(null, list));
   }
 
   // Create/write database file to s3
@@ -217,6 +185,19 @@ export default class S3Database implements IPluginStorage<S3Config> {
     return this._localData as LocalStorage;
   }
 
+  private async _getPackageList(): Promise<any> {
+    const { bucket, keyPrefix } = this.config;
+    const listPath = keyPrefix.endsWith('/') ? keyPrefix : '/';
+    this.logger.debug({ keyPrefix, bucket }, 's3: [_getPackageList] bucket: @{bucket} prefix: @{keyPrefix} listPath: @{listPath}');
+    const lister = s3ls({ bucket, s3: this.s3 });
+    const { files, folders } = await lister.ls(listPath);
+    // folders: ['prefix/package-a/', 'prefix/package-b/', ...]
+    if (!folders) return [];
+    else return folders.map(function(folder) {
+      return folder.split('/').slice(-2)[0];
+    })
+  }
+
   public saveToken(token: Token): Promise<void> {
     this.logger.warn({ token }, 'save token has not been implemented yet @{token}');