diff --git a/README.md b/README.md index 8e7f641..0b126fb 100644 --- a/README.md +++ b/README.md @@ -12,4 +12,12 @@ The source code is released under [AGPL v3](http://www.gnu.org/licenses/agpl-3.0 ## Credits -Please visit [this page](http://uku.im/contributors) for an updated list of our contributors. +Please visit [this page](http://uku.im/contributors) for an up-to-date list of our contributors. + +## Developmeent + +Update URL and server configs under the folder `src/configs/`. + +Run `npm ci` to install dependencies exactly as they are listed in the package-lock.json file. + +Run `npm run test` to create a zip file for uploading and testing. diff --git a/babel.config.json b/babel.config.json new file mode 100644 index 0000000..7521eb0 --- /dev/null +++ b/babel.config.json @@ -0,0 +1,7 @@ +{ + "env": { + "test": { + "plugins": ["@babel/plugin-transform-modules-commonjs"] + } + } +} diff --git a/e2e_tests/chrome_extention.test.js b/e2e_tests/chrome_extention.test.js index af583aa..53f9bd6 100644 --- a/e2e_tests/chrome_extention.test.js +++ b/e2e_tests/chrome_extention.test.js @@ -1,4 +1,4 @@ -jest.setTimeout(60000); // in milliseconds +jest.setTimeout(90000); // in milliseconds const EXTENSION_INIT_WAIT_TIME = 3000; const HEADER_TEST_URL = 'https://httpbin.org/headers'; diff --git a/jest.config.js b/jest.config.js index 2762fa4..96dfa19 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,3 +1,9 @@ module.exports = { - preset: 'jest-puppeteer', + 'preset': 'jest-puppeteer', + 'testRegex': '(/__tests__/.*|(\\.|/)(test|spec))\\.(mjs?|jsx?|js?|tsx?|ts?)$', + 'transform': { + '^.+\\.jsx?$': 'babel-jest', + '^.+\\.mjs$': 'babel-jest', + }, + 'moduleFileExtensions': ['js', 'jsx', 'mjs'], }; diff --git a/package-lock.json b/package-lock.json index b8130ee..9131f13 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,7 @@ "name": "unblock-youku-dev-cli", "license": "AGPL-3.0", "devDependencies": { + "@babel/plugin-transform-modules-commonjs": "^7.18.6", "archiver": "^5.3.1", "eslint": "^8.18.0", "eslint-config-google": "^0.14.0", @@ -552,6 +553,24 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/plugin-transform-modules-commonjs": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.18.6.tgz", + "integrity": "sha512-Qfv2ZOWikpvmedXQJDSbxNqy7Xr/j2Y8/KfijM0iJyKkBTmWuvCA1yeH1yDM7NJhBW/2aXxeucLj6i80/LAJ/Q==", + "dev": true, + "dependencies": { + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-simple-access": "^7.18.6", + "babel-plugin-dynamic-import-node": "^2.3.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, "node_modules/@babel/template": { "version": "7.18.6", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.18.6.tgz", @@ -1560,6 +1579,15 @@ "@babel/core": "^7.8.0" } }, + "node_modules/babel-plugin-dynamic-import-node": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz", + "integrity": "sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ==", + "dev": true, + "dependencies": { + "object.assign": "^4.1.0" + } + }, "node_modules/babel-plugin-istanbul": { "version": "6.1.1", "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", @@ -6697,6 +6725,18 @@ "@babel/helper-plugin-utils": "^7.18.6" } }, + "@babel/plugin-transform-modules-commonjs": { + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-modules-commonjs/-/plugin-transform-modules-commonjs-7.18.6.tgz", + "integrity": "sha512-Qfv2ZOWikpvmedXQJDSbxNqy7Xr/j2Y8/KfijM0iJyKkBTmWuvCA1yeH1yDM7NJhBW/2aXxeucLj6i80/LAJ/Q==", + "dev": true, + "requires": { + "@babel/helper-module-transforms": "^7.18.6", + "@babel/helper-plugin-utils": "^7.18.6", + "@babel/helper-simple-access": "^7.18.6", + "babel-plugin-dynamic-import-node": "^2.3.3" + } + }, "@babel/template": { "version": "7.18.6", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.18.6.tgz", @@ -7529,6 +7569,15 @@ "slash": "^3.0.0" } }, + "babel-plugin-dynamic-import-node": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz", + "integrity": "sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ==", + "dev": true, + "requires": { + "object.assign": "^4.1.0" + } + }, "babel-plugin-istanbul": { "version": "6.1.1", "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", diff --git a/package.json b/package.json index f12bfb1..2c3349a 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "test": "npm run clean && node tools/create_zip.js && unzip -q dist/upload_to_chrome_store.zip -d dist/unzipped_chrome_extension && jest --verbose" }, "devDependencies": { + "@babel/plugin-transform-modules-commonjs": "^7.18.6", "archiver": "^5.3.1", "eslint": "^8.18.0", "eslint-config-google": "^0.14.0", diff --git a/src/configs/servers.test.mjs b/src/configs/servers.test.mjs new file mode 100644 index 0000000..44aade7 --- /dev/null +++ b/src/configs/servers.test.mjs @@ -0,0 +1,15 @@ +import { + DEFAULT_PROXY_ADDRESS, BACKUP_PROXY_ADDRESS, + DEFAULT_PROXY_PROTOCOL, BACKUP_PROXY_PROTOCOL} from './servers.mjs'; + +test('The proxy addresses must not be localhost', () => { + expect(DEFAULT_PROXY_ADDRESS).not.toMatch(/localhost/i); + expect(DEFAULT_PROXY_ADDRESS).not.toMatch(/127\.0\.0\.1/i); + expect(BACKUP_PROXY_ADDRESS).not.toMatch(/localhost/i); + expect(BACKUP_PROXY_ADDRESS).not.toMatch(/127\.0\.0\.1/i); +}); + +test('Production must use HTTPS proxy', () => { + expect(DEFAULT_PROXY_PROTOCOL).toBe('HTTPS'); + expect(BACKUP_PROXY_PROTOCOL).toBe('HTTPS'); +}); diff --git a/src/configs/urls.test.mjs b/src/configs/urls.test.mjs new file mode 100644 index 0000000..dcfe7d6 --- /dev/null +++ b/src/configs/urls.test.mjs @@ -0,0 +1,39 @@ +import {HEADER_URLS, PROXY_BYPASS_URLS, PROXY_URLS} from './urls.mjs'; + + +test('The proxy URL list must contain some domains', () => { + expect(PROXY_URLS.filter((url) => url.includes('.qq.com'))).not.toHaveLength(0); + expect(PROXY_URLS.filter( + (url) => url.includes('flask-test-iwauxcyxjb.cn-hangzhou.fcapp.run/'))).not.toHaveLength(0); +}); + +test('Must not contain the all-url rules', () => { + [HEADER_URLS, PROXY_BYPASS_URLS, PROXY_URLS].forEach((urlList) => { + expect(urlList.filter((url) => url.startsWith('http://*/'))).toHaveLength(0); + expect(urlList.filter((url) => url.startsWith('https://*/'))).toHaveLength(0); + }); +}); + +test('All URLs must start with http:// or https://', () => { + const regex = /^(http|https):\/\//i; + [HEADER_URLS, PROXY_BYPASS_URLS, PROXY_URLS].forEach((urlList) => { + expect(urlList.filter((url) => !regex.test(url))).toHaveLength(0); + }); +}); + +test('All https URLs must have path that is * or empty', () => { + /* + * For example: + * - Wrong: https://example.com/abc/* + * - Right: https://example.com/* + * - Right: https://example.com/ + */ + [HEADER_URLS, PROXY_BYPASS_URLS, PROXY_URLS].forEach((urlList) => { + for (const url of urlList) { + if (url.startsWith('https://')) { + const domainRemoved = url.slice('https://'.length).split('/')[1]; + expect(['*', '']).toContain(domainRemoved); + } + } + }); +}); diff --git a/src/modules/_url_utils.test.mjs b/src/modules/_url_utils.test.mjs new file mode 100644 index 0000000..df2c93c --- /dev/null +++ b/src/modules/_url_utils.test.mjs @@ -0,0 +1,132 @@ +import {urls2pac} from './_url_utils.mjs'; + +// urlWhitelist, urlList, +// proxyProtocol1, proxyAddress1, +// proxyProtocol2, proxyAddress2) { + +const TEST_PROXY_PROTOCOL_1 = 'HTTPS'; +const TEST_PROXY_ADDRESS_1 = 'proxy.example.com'; +const TEST_PROXY_PROTOCOL_2 = 'https'; +const TEST_PROXY_ADDRESS_2 = '1.2.3.4'; + + +const TEST_BYPASS_URL_LIST = [ + 'http://bangumi.bilibili.com/index/ding-count.json', +]; + +const TEST_URL_LIST = [ + 'http://*/*', + 'https://*/*', + 'http://*.video.qq.com/*', + 'https://*.video.qq.com/*', + 'http://vd.l.qq.com/*', + 'https://vd.l.qq.com/*', + 'http://example.com', + 'https://example.com', + 'http://example.com/', + 'https://example.com/', + 'http://*.example.com/*', + 'https://*.example.com/*', + 'http://*.example.com/path.json?aaa=bbb', + 'https://*.example.com/path.json?aaa=bbb', + 'http://*.example.com/path.json?aaa=bbb*', + 'https://*.example.com/path.json?aaa=bbb*', + 'http://122.72.82.31/*', +]; + +const EXPECTED_PAC_CONTENT = [ + 'var _http_map = {', + ' \'white\': {', + ' \'any\': [],', + ' \'bangumi.bilibili.com\': [', + ' /^\\/index\\/ding\\-count\\.json$/i', + ' ]', + ' },', + ' \'proxy\': {', + ' \'any\': [', + ' /^[^/]*\\//i,', + ' /^[^/]*\\.video\\.qq\\.com\\//i,', + ' /^[^/]*\\.example\\.com\\//i,', + ' /^[^/]*\\.example\\.com\\/path\\.json\\?aaa=bbb$/i,', + ' /^[^/]*\\.example\\.com\\/path\\.json\\?aaa=bbb/i', + ' ],', + ' \'vd.l.qq.com\': [', + ' /^\\//i', + ' ],', + ' \'example.com\': [', + ' /^\\/$/i,', + ' /^\\/$/i', + ' ],', + ' \'122.72.82.31\': [', + ' /^\\//i', + ' ]', + ' }', + '};', + 'var _https_map = {', + ' \'white\': {', + ' \'any\': []', + ' },', + ' \'proxy\': {', + ' \'any\': [', + ' /^[^/]*\\//i,', + ' /^[^/]*\\.video\\.qq\\.com\\//i,', + ' /^[^/]*\\.example\\.com\\//i,', + ' /^[^/]*\\.example\\.com\\/path\\.json\\?aaa=bbb$/i,', + ' /^[^/]*\\.example\\.com\\/path\\.json\\?aaa=bbb/i', + ' ],', + ' \'vd.l.qq.com\': [', + ' /^\\//i', + ' ],', + ' \'example.com\': [', + ' /^\\/$/i,', + ' /^\\/$/i', + ' ]', + ' }', + '};', + 'var _proxy_str = \'HTTPS proxy.example.com; HTTPS 1.2.3.4; DIRECT;\';', + '', + 'function _check_regex_list(regex_list, str) {', + ' if (str.slice(0, 4) === \':80/\')', + ' str = str.slice(3);', + ' for (var i = 0; i < regex_list.length; i++)', + ' if (regex_list[i].test(str))', + ' return true;', + ' return false;', + '}', + '', + 'function _check_patterns(patterns, hostname, full_url, prot_len) {', + ' if (patterns.hasOwnProperty(hostname))', + ' if (_check_regex_list(patterns[hostname],', + ' full_url.slice(prot_len + hostname.length)))', + ' return true;', + ' if (_check_regex_list(patterns.any,', + ' full_url.slice(prot_len)))', + ' return true;', + ' return false;', + '}', + '', + 'function _find_proxy(url_map, host, url, prot_len) {', + ' if (_check_patterns(url_map.white, host, url, prot_len))', + ' return \'DIRECT\';', + ' if (_check_patterns(url_map.proxy, host, url, prot_len))', + ' return _proxy_str;', + ' return \'DIRECT\';', + '}', + '', + 'function FindProxyForURL(url, host) {', + ' var prot = url.slice(0, 6);', + ' if (prot === \'http:/\')', + ' return _find_proxy(_http_map, host, url, 7);', + ' else if (prot === \'https:\')', + ' return _find_proxy(_https_map, host, url, 8);', + ' return \'DIRECT\';', + '}', +].join('\n') + '\n'; + + +test('Should produce the expected PAC content', () => { + expect(urls2pac( + TEST_BYPASS_URL_LIST, TEST_URL_LIST, + TEST_PROXY_PROTOCOL_1, TEST_PROXY_ADDRESS_1, + TEST_PROXY_PROTOCOL_2, TEST_PROXY_ADDRESS_2)).toMatch(EXPECTED_PAC_CONTENT); +}); diff --git a/tools/_regex_utils.mjs b/tools/_regex_utils.mjs new file mode 100644 index 0000000..67394c3 --- /dev/null +++ b/tools/_regex_utils.mjs @@ -0,0 +1,41 @@ +function urls2regexs(urlList) { + const regexList = []; + + for (let str of urlList) { + // Escape all possibly problematic symbols + // http://stackoverflow.com/a/6969486/1766096 + str = str.replace(/[\-\[\]\/\{\}\(\)\+\?\.\\\^\$\|]/g, '\\$&'); + str = str.replace(/\*/g, '.*'); + + // make the first * matches only domain names or ip addresses + // just as http://developer.chrome.com/extensions/match_patterns.html + str = str.replace(/^http:\\\/\\\/\.\*/i, 'http:\\/\\/[^\/]*'); + str = str.replace(/^https:\\\/\\\/\.\*/i, 'https:\\/\\/[^\/]*'); + + regexList.push(new RegExp('^' + str + '$', 'i')); + } + + // console.log(regex_list); + return regexList; +} + + +export function produceSquidRegexList(urlList) { + const regexList = urls2regexs(urlList); + const regexToExtractHttpsDomain = /^\^https:\\\/\\\/([^:]+)\\\//i; + + let str; + const result = []; + for (const regex of regexList) { + str = regex.toString(); + str = str.substring(1, str.length - 2); + + if (str.match(regexToExtractHttpsDomain)) { + str = '^' + str.match(regexToExtractHttpsDomain)[1] + ':443'; + } + + result.push(str); + } + + return result; +} diff --git a/tools/_regex_utils.test.mjs b/tools/_regex_utils.test.mjs new file mode 100644 index 0000000..fa9fbb3 --- /dev/null +++ b/tools/_regex_utils.test.mjs @@ -0,0 +1,47 @@ +import {produceSquidRegexList} from './_regex_utils.mjs'; + + +const TEST_URL_LIST = [ + 'http://*/*', + 'https://*/*', + 'http://*.video.qq.com/*', + 'https://*.video.qq.com/*', + 'http://vd.l.qq.com/*', + 'https://vd.l.qq.com/*', + 'http://example.com', + 'https://example.com', + 'http://example.com/', + 'https://example.com/', + 'http://*.example.com/*', + 'https://*.example.com/*', + 'http://*.example.com/path.json?aaa=bbb', + 'https://*.example.com/path.json?aaa=bbb', + 'http://*.example.com/path.json?aaa=bbb*', + 'https://*.example.com/path.json?aaa=bbb*', + 'http://122.72.82.31/*', +]; + +const EXPECTED_REGEX_LIST = [ + '^http:\\/\\/[^/]*\\/.*$', + '^[^/]*:443', + '^http:\\/\\/[^/]*\\.video\\.qq\\.com\\/.*$', + '^[^/]*\\.video\\.qq\\.com:443', + '^http:\\/\\/vd\\.l\\.qq\\.com\\/.*$', + '^vd\\.l\\.qq\\.com:443', + '^http:\\/\\/example\\.com$', + '^https:\\/\\/example\\.com$', + '^http:\\/\\/example\\.com\\/$', + '^example\\.com:443', + '^http:\\/\\/[^/]*\\.example\\.com\\/.*$', + '^[^/]*\\.example\\.com:443', + '^http:\\/\\/[^/]*\\.example\\.com\\/path\\.json\\?aaa=bbb$', + '^[^/]*\\.example\\.com:443', + '^http:\\/\\/[^/]*\\.example\\.com\\/path\\.json\\?aaa=bbb.*$', + '^[^/]*\\.example\\.com:443', + '^http:\\/\\/122\\.72\\.82\\.31\\/.*$', +]; + + +test('Should produce the expected regex list', () => { + expect(produceSquidRegexList(TEST_URL_LIST).sort()).toEqual(EXPECTED_REGEX_LIST.sort()); +}); diff --git a/tools/create_zip.js b/tools/create_zip.js index 69dc6b8..4a4f2df 100644 --- a/tools/create_zip.js +++ b/tools/create_zip.js @@ -21,19 +21,19 @@ const FILES_TO_BE_ZIPPED = [ ]; const EXCLUDED_FILE_PATTERNS = [ // Using regex - /.*_test\.js/i, - /.*\.test\.js/i, - /.*tests.*/i, - /.*\.zip/i, - /.*\.tar/i, - /.*\.gz/i, - /.*node_modules.*/i, - /.*\.DS_Store/i, - /.*~/i, - /.*\.swp/i, - /.*\.pyc/i, - /.*\.bak/i, - /.*\.log/i, + /_test\./i, + /\.test\./i, + /tests/i, + /node_modules/i, + /\.DS_Store/i, + /\.zip$/i, + /\.tar$/i, + /\.gz$/i, + /~$/i, + /\.swp$/i, + /\.pyc$/i, + /\.bak$/i, + /\.log$/i, ]; diff --git a/tools/produce_regex.mjs b/tools/produce_regex.mjs index 20203e6..4cf5ba8 100644 --- a/tools/produce_regex.mjs +++ b/tools/produce_regex.mjs @@ -1,48 +1,26 @@ import {PROXY_URLS} from '../src/configs/urls.mjs'; - - -function urls2regexs(urlList) { - const regexList = []; - - for (let str of urlList) { - // Escape all possibly problematic symbols - // http://stackoverflow.com/a/6969486/1766096 - str = str.replace(/[\-\[\]\/\{\}\(\)\+\?\.\\\^\$\|]/g, '\\$&'); - str = str.replace(/\*/g, '.*'); - - // make the first * matches only domain names or ip addresses - // just as http://developer.chrome.com/extensions/match_patterns.html - str = str.replace(/^http:\\\/\\\/\.\*/i, 'http:\\/\\/[^\/]*'); - str = str.replace(/^https:\\\/\\\/\.\*/i, 'https:\\/\\/[^\/]*'); - - regexList.push(new RegExp('^' + str + '$', 'i')); - } - - // console.log(regex_list); - return regexList; -} - - -function produceSquidRegexList() { - const regexList = urls2regexs(PROXY_URLS); - const regexToExtractHttpsDomain = /^\^https:\\\/\\\/([^:]+)\\\//i; - - let str; - const result = []; - for (const regex of regexList) { - str = regex.toString(); - str = str.substring(1, str.length - 2); - - if (str.match(regexToExtractHttpsDomain)) { - str = '^' + str.match(regexToExtractHttpsDomain)[1] + ':443'; - } - - result.push(str); - } - - return result; -} - - -console.log(produceSquidRegexList().join('\n') + '\n'); +import {produceSquidRegexList} from './_regex_utils.mjs'; + +const TEST_URL_LIST = [ + 'http://*/*', + 'https://*/*', + 'http://*.video.qq.com/*', + 'https://*.video.qq.com/*', + 'http://vd.l.qq.com/*', + 'https://vd.l.qq.com/*', + 'http://example.com', + 'https://example.com', + 'http://example.com/', + 'https://example.com/', + 'http://*.example.com/*', + 'https://*.example.com/*', + 'http://*.example.com/path.json?aaa=bbb', + 'https://*.example.com/path.json?aaa=bbb', + 'http://*.example.com/path.json?aaa=bbb*', + 'https://*.example.com/path.json?aaa=bbb*', + 'http://122.72.82.31/*', +]; + +// console.log(produceSquidRegexList(PROXY_URLS).join('\n') + '\n'); +console.log(produceSquidRegexList(TEST_URL_LIST).join('\n') + '\n'); process.exit(0);