From 97e3321a3ed53795e839fc15a46891641eedd5a6 Mon Sep 17 00:00:00 2001 From: Bo Zhu Date: Mon, 7 Jul 2014 23:01:42 -0400 Subject: [PATCH] start to remove sogou code from the entire project --- Makefile | 2 +- Procfile | 2 +- README.md | 9 +- newrelic.js | 24 -- package.json | 28 +- server/server.js | 333 +++--------------- server/utils.js | 202 ++--------- shared/sogou.js | 146 -------- test/run-all-tests.py | 11 +- test/test-163-proxy.js | 17 - test/{test-163-redirect.js => test-163.js} | 0 test/test-google-proxy.js | 12 - ...test-google-redirect.js => test-google.js} | 0 test/test-https-google-proxy.js | 13 - test/test-https-httpbin-proxy.js.bak | 23 -- ...ocalhost-redirect.js => test-localhost.js} | 0 test/test-sohu-proxy.js | 17 - test/{test-sohu-redirect.js => test-sohu.js} | 0 test/test-status-page.js | 18 - test/test-youku-proxy.js | 17 - .../{test-youku-redirect.js => test-youku.js} | 0 21 files changed, 90 insertions(+), 784 deletions(-) delete mode 100644 newrelic.js delete mode 100644 shared/sogou.js delete mode 100644 test/test-163-proxy.js rename test/{test-163-redirect.js => test-163.js} (100%) delete mode 100644 test/test-google-proxy.js rename test/{test-google-redirect.js => test-google.js} (100%) delete mode 100644 test/test-https-google-proxy.js delete mode 100644 test/test-https-httpbin-proxy.js.bak rename test/{test-localhost-redirect.js => test-localhost.js} (100%) delete mode 100644 test/test-sohu-proxy.js rename test/{test-sohu-redirect.js => test-sohu.js} (100%) delete mode 100644 test/test-status-page.js delete mode 100644 test/test-youku-proxy.js rename test/{test-youku-redirect.js => test-youku.js} (100%) diff --git a/Makefile b/Makefile index b5cd0be3..926180bd 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ZIP_FILENAME = crx.zip +ZIP_FILENAME = ../ubuku_crx.zip INCLUDED_FILES = manifest.json chrome shared _locales COPYING.txt EXCLUDED_FILES = _locales/backup* diff --git a/Procfile b/Procfile index 9cf142c9..91a2fef4 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ -web: node server/server.js --production +web: node server/server.js --nolog --proxy=http://proxy.uku.im:8888 diff --git a/README.md b/README.md index cf33fc71..56bb26cb 100644 --- a/README.md +++ b/README.md @@ -6,17 +6,14 @@ This project contains four functionalities. You can find this extension on Chrome Web Store at [http://uku.im/chrome](http://uku.im/chrome) -2. _A local proxy server_ based on the non-blocking [Node.js](http://nodejs.org) for bypassing the mainland restriction _for other browsers and mobile devices_ (but not guaranteed). +2. _A backend server_ to support the redirect mode of the Chrome extension. - For more information, please see [the instructions](http://uku.im/proxy) (in Chinese). + [![Build Status](https://secure.travis-ci.org/zhuzhuor/Unblock-Youku.png?branch=master)](https://travis-ci.org/zhuzhuor/Unblock-Youku) [![Dependency Status](https://gemnasium.com/zhuzhuor/Unblock-Youku.png)](https://gemnasium.com/zhuzhuor/Unblock-Youku) -3. _A DNS server_ combined with the local proxy server. +3. _A local DNS server_ for devices that cannot set up proxies freely. For more information, please read [its README file](https://github.com/zhuzhuor/Unblock-Youku/tree/master/dns-reverse-proxy). -4. _A backend server_ to support the redirection mode of the Chrome extension. - - [![Build Status](https://secure.travis-ci.org/zhuzhuor/Unblock-Youku.png?branch=master)](https://travis-ci.org/zhuzhuor/Unblock-Youku) [![Dependency Status](https://gemnasium.com/zhuzhuor/Unblock-Youku.png)](https://gemnasium.com/zhuzhuor/Unblock-Youku) ## Disclaimer diff --git a/newrelic.js b/newrelic.js deleted file mode 100644 index 2d7b52ff..00000000 --- a/newrelic.js +++ /dev/null @@ -1,24 +0,0 @@ -/** - * New Relic agent configuration. - * - * See lib/config.defaults.js in the agent distribution for a more complete - * description of configuration variables and their potential values. - */ -exports.config = { - /** - * Array of application names. - */ - app_name : ['ub.uku.js'], - /** - * Your New Relic license key. - */ - // license_key : 'license key here', // should be overridden by ENV setting - logging : { - /** - * Level at which to log. 'trace' is most useful to New Relic when diagnosing - * issues with the agent, 'info' and higher will impose the least overhead on - * production applications. - */ - level : 'info' - } -}; diff --git a/package.json b/package.json index 60289a53..7d852989 100644 --- a/package.json +++ b/package.json @@ -1,24 +1,11 @@ { "name": "ub.uku.js", - "preferGlobal": "true", - "version": "1.8.10", - "author": "zhuzhuor ", - "description": "A Backend Server for Unblock-Youku", - "keywords": ["Unblock Youku", "proxy", "server"], - "bin": { - "ub.uku.js": "server/server.js" - }, - "repository": { - "type": "git", - "url": "https://github.com/zhuzhuor/Unblock-Youku.git" - }, "dependencies": { "colors": "*", "optimist": "0.6.x", - "validator": "3.13.x", "uglify-js": "2.4.x", "raven": "0.6.x", - "newrelic": "1.6.x" + "request": "2.37.x" }, "scripts": { "start": "node server/server.js", @@ -26,13 +13,10 @@ }, "engines": { "node": "0.10.x", - "npm": "1.2.x" + "npm": "1.4.x" }, - "bugs": { - "url": "https://github.com/zhuzhuor/Unblock-Youku/issues" - }, - "licenses": [{ - "type": "AGPL v3", - "url": "http://www.gnu.org/licenses/agpl-3.0.html" - }] + "repository": { + "type": "git", + "url": "https://github.com/zhuzhuor/Unblock-Youku.git" + } } diff --git a/server/server.js b/server/server.js index 7b619abe..51e28bef 100755 --- a/server/server.js +++ b/server/server.js @@ -1,7 +1,6 @@ #!/usr/bin/env node /* - * Allow you smoothly surf on many websites blocking non-mainland visitors. * Copyright (C) 2012 - 2014 Bo Zhu http://zhuzhu.org * * This program is free software: you can redistribute it and/or modify @@ -19,54 +18,27 @@ */ -var new_relic_working = false; -if (process.env.NEW_RELIC_LICENSE_KEY) { - require('newrelic'); - new_relic_working = true; -} +var net = require('net'); +var url = require('url'); +var util = require('util'); +var http = require('http'); +var cluster = require('cluster'); +http.globalAgent.maxSockets = Infinity; var argv = require('optimist') - .default('ip', '0.0.0.0') // listen to all interfaces - .default('port', '8888') - .boolean('local_only') // force --ip=127.0.0.1 - .boolean('mitm_proxy') // for debug use; no access control - .boolean('nolog') // do not show network logs - .boolean('production') // pre-set configs for production server + .string('proxy') // remote proxy address, such as http://123.45.67.89:8888 + .boolean('nolog') // do not show network logs .argv ; var colors = require('colors'); +var request = require('request'); -// check input parameters -var validator = require('validator'); -function validate_input(str, fun, msg) { - if (!fun(str.toString())) { - console.error(msg.red); - process.exit(1); - } -} - -validate_input(argv.ip, validator.isIP, 'Invalid input for IP address.'); -validate_input(argv.port, validator.isNumeric, 'Invalid input for port number.'); -if (argv.ext_ip) { // custom IP/domain address in the PAC file, in case the proxy server is behind a router or firewall - if (argv.ext_ip !== validator.escape(argv.ext_ip)) { - console.error('Invalid input for external IP/domain address.'.red); - process.exit(1); - } -} -if (argv.ext_port) { // custom port number - validate_input(argv.ext_port, validator.isNumeric, 'Invalid input for external port number.'); -} +var shared_tools = require('../shared/tools'); +var server_utils = require('./utils'); -var net = require('net'); -var url = require('url'); -var util = require('util'); -var http = require('http'); -var cluster = require('cluster'); -http.globalAgent.maxSockets = Infinity; - var raven = null; var raven_client = null; if (process.env.SENTRY_ADDRESS) { @@ -76,112 +48,28 @@ if (process.env.SENTRY_ADDRESS) { raven_client.captureMessage('Sentry is running...'); } -var sogou = require('../shared/sogou'); -var shared_tools = require('../shared/tools'); -var server_utils = require('./utils'); - -var local_addr, local_port, proxy_addr, proxy_port, status_text; -if (!argv.production) { - if (argv.local_only) { - local_addr = '127.0.0.1'; - proxy_addr = '127.0.0.1'; - } else { - local_addr = argv.ip; - if (argv.ext_ip) { - proxy_addr = argv.ext_ip; - } else if (local_addr === '0.0.0.0') { - proxy_addr = server_utils.get_first_external_ip(); - } else { - proxy_addr = local_addr; - } - } - - local_port = argv.port; - if (argv.ext_port) { - proxy_port = argv.ext_port; - } else { - proxy_port = local_port; - } -} else { - local_addr = '0.0.0.0'; - local_port = process.env.PORT || 8888; - proxy_addr = 'proxy.uku.im'; - proxy_port = '8888'; - - argv.local_only = false; - argv.mitm_proxy = false; - argv.nolog = true; +local_port = process.env.PORT || 8888; +var pac_file_content = null; +if (argv.proxy) { + // TODO parse proxy argv + pac_file_content = server_utils.generate_pac_file(proxy_addr + ':' + proxy_port); } -var pac_file_content = server_utils.generate_pac_file(proxy_addr + ':' + proxy_port); // console.log(pac_file_content); -var sogou_server_addr; -var reset_count = 0, MAX_RESET_COUNT = 1; -var refuse_count = 0, MAX_REFUSE_COUNT = 2; -var timeout_count = 0, MAX_TIMEOUT_COUNT = 4; -var in_changing_server = false, last_error_code = null; -function change_sogou_server(error_code) { - if (true === in_changing_server) { - return; // should already be in the process of changing new server - } - in_changing_server = true; - - if ('string' === typeof error_code) { - last_error_code = error_code; - } else { - last_error_code = null; - } - server_utils.renew_sogou_server(function(new_addr) { - sogou_server_addr = new_addr; - if (null !== last_error_code) { - util.error('[ub.uku.js] on ' + last_error_code + ' error, changed server to ' + new_addr); - } - reset_count = 0; - refuse_count = 0; - timeout_count = 0; - in_changing_server = false; - }); -} -function count_sogou_server_errors(err) { - if ('ECONNRESET' === err.code) { - reset_count++; - util.log('[ub.uku.js] ' + sogou_server_addr + ' reset_count: ' + reset_count); - if (reset_count >= MAX_RESET_COUNT) { - change_sogou_server('ECONNRESET'); - } - } else if ('ECONNREFUSED' === err.code) { - refuse_count++; - util.log('[ub.uku.js] ' + sogou_server_addr + ' refuse_count: ' + refuse_count); - if (refuse_count >= MAX_REFUSE_COUNT) { - change_sogou_server('ECONNREFUSED'); - } - } else if ('ETIMEDOUT' === err.code) { - timeout_count++; - util.log('[ub.uku.js] ' + sogou_server_addr + ' timeout_count: ' + timeout_count); - if (timeout_count >= MAX_TIMEOUT_COUNT) { - change_sogou_server('ETIMEOUT'); - } - } -} - - function http_req_handler(client_request, client_response) { if (!argv.nolog) { - console.log('[ub.uku.js] ' + client_request.connection.remoteAddress + ': ' + client_request.method + ' ' + client_request.url.underline); + console.log( + '[ub.uku.js] ' + + client_request.connection.remoteAddress + ': ' + + client_request.method + ' ' + client_request.url.underline); } if (client_request.url === '/proxy.pac' || (!shared_tools.string_starts_with(client_request.url, '/proxy') && !shared_tools.string_starts_with(client_request.url, 'http'))) { - server_utils.static_responses(client_request, client_response, argv.production, pac_file_content); - } - - // cannot forward cookie settings for other domains in redirect mode - var forward_cookies = false; - if (shared_tools.string_starts_with(client_request.url, 'http')) { - forward_cookies = true; + server_utils.static_responses(client_request, client_response, pac_file_content); } var target = server_utils.get_real_target(client_request.url); @@ -195,162 +83,41 @@ function http_req_handler(client_request, client_response) { // access control var proxy_request_options; - var to_use_proxy = server_utils.is_valid_url(target.href); - if (to_use_proxy) { - var proxy_request_headers = server_utils.filtered_request_headers( - client_request.headers, - forward_cookies - ); - server_utils.add_sogou_headers(proxy_request_headers, target.hostname); - proxy_request_headers.Host = target.host; - - proxy_request_options = { - hostname: sogou_server_addr, - host: sogou_server_addr, - port: 80, - path: target.href, - method: client_request.method, - headers: proxy_request_headers - }; - } else if (argv.mitm_proxy) { - // serve as a normal proxy server - client_request.headers.host = target.host; - proxy_request_options = { - host: target.host, - hostname: target.hostname, - port: +target.port, - path: target.path, - method: client_request.method, - headers: server_utils.filtered_request_headers(client_request.headers, forward_cookies) - }; - } else { + if (!server_utils.is_valid_url(target.href)) { client_response.writeHead(403, { 'Cache-Control': 'public, max-age=14400' }); client_response.end(); return; } + var proxy_request_headers = server_utils.filtered_request_headers(client_request.headers); + proxy_request_headers.Host = target.host; - // connect to the target server - var proxy_request = http.request(proxy_request_options, function(proxy_response) { - proxy_response.on('error', function(err) { - util.error('[ub.uku.js] proxy_response error: (' + err.code + ') ' + err.message, err.stack); - client_response.statusCode = 599; - client_response.end(); - proxy_request.end(); - }); - - proxy_response.pipe(client_response); - - client_response.writeHead( - proxy_response.statusCode, - server_utils.filtered_response_headers(proxy_response.headers, forward_cookies) - ); - }); - proxy_request.on('error', function(err) { - util.error('[ub.uku.js] proxy_request error: (' + err.code + ') ' + err.message, err.stack); - - if (to_use_proxy) { - // do not need to count the errors if the url is not proxied - count_sogou_server_errors(err); - } + var proxy_request_options = { + url: target.href, + method: client_request.method, + headers: proxy_request_headers + }; + if (argv.proxy) { + proxy_request_options.proxy = argv.proxy; + } + var proxy_request = request(proxy_request_options); + client_request.pipe(proxy_request); + proxy_request.pipe(client_response); - // should we explicitly end client_response when error occurs? - client_response.statusCode = 599; - client_response.end(); - proxy_request.end(); - // should we also destroy the proxy_request object? - }); + /* client_request.on('error', function(err) { util.error('[ub.uku.js] client_request error: (' + err.code + ') ' + err.message, err.stack); proxy_request.end(); // is this correct? }); - client_response.on('error', function(err) { // does this work? + client_response.on('error', function(err) { util.error('[ub.uku.js] client_response error: (' + err.code + ') ' + err.message, err.stack); proxy_request.end(); }); - - client_request.pipe(proxy_request); + */ } -function connect_req_handler(client_request, client_socket, client_head) { - // this should only be used for proxy server, not redirect server - - client_request.on('error', function(err) { - util.error('[ub.uku.js] CONNECT client_request error: (' + err.code + ') ' + err.message, err.stack); - }); - - if (!argv.nolog) { - console.log('[ub.uku.js] ' + client_request.connection.remoteAddress + ': CONNECT ' + client_request.url.underline); - } - - // if (true) { - if (server_utils.is_valid_https_domain(url.parse('https://' + client_request.url).hostname)) { - var proxy_request_headers = client_request.headers; - server_utils.add_sogou_headers(proxy_request_headers, client_request.headers.host); - - var proxy_request_options = { - hostname: sogou_server_addr, - host: client_request.url, - port: 80, - path: client_request.url, - method: 'CONNECT', - headers: proxy_request_headers - }; - var proxy_request = http.request(proxy_request_options); - proxy_request.on('connect', function(proxy_response, proxy_socket) { - proxy_response.on('error', function(err) { - // do we need to listen to the error of proxy_response? - util.error('[ub.uku.js] CONNECT proxy_response error: (' + err.code + ') ' + err.message, err.stack); - }); - proxy_socket.on('error', function(err) { - util.error('[ub.uku.js] CONNECT proxy_socket error: (' + err.code + ') ' + err.message, err.stack); - count_sogou_server_errors(err); - client_socket.end(); // what's the difference with emit('end')? - }); - client_socket.on('error', function(err) { - util.error('[ub.uku.js] CONNECT client_socket error: (' + err.code + ') ' + err.message, err.stack); - proxy_socket.end(); - }); - - client_socket.write('HTTP/1.1 200 Connection Established\r\n\r\n'); - proxy_socket.write(client_head); - proxy_socket.pipe(client_socket); - client_socket.pipe(proxy_socket); - }); - proxy_request.on('error', function(err) { - util.error('[ub.uku.js] CONNECT proxy_request error: (' + err.code + ') ' + err.message, err.stack); - count_sogou_server_errors(err); - client_socket.end(); - }); - - proxy_request.end(); - - } else if (argv.mitm_proxy) { - // serve as a normal proxy server - var target = url.parse('https://' + client_request.url); - var proxy_socket = net.connect(443, target.hostname, function() { - client_socket.write('HTTP/1.1 200 Connection Established\r\n\r\n'); - proxy_socket.write(client_head); - proxy_socket.pipe(client_socket); - client_socket.pipe(proxy_socket); - }); - proxy_socket.on('error', function(err) { - util.error('[ub.uku.js] CONNECT proxy_socket error: (' + err.code + ') ' + err.message, err.stack); - client_socket.end(); - }); - client_socket.on('error', function(err) { - util.error('[ub.uku.js] CONNECT client_socket error: (' + err.code + ') ' + err.message, err.stack); - proxy_socket.end(); - }); - - } else { - client_socket.end('HTTP/1.1 403 Forbidden\r\n\r\n'); - } -} - - if (cluster.isMaster) { var num_CPUs = require('os').cpus().length; // num_CPUs = 1; @@ -381,32 +148,18 @@ if (cluster.isMaster) { } }); - if (argv.production) { - console.log('Starting in production mode...'.yellow); + console.log('The redirect server is running...'.green); + if (argv.proxy) { + console.log(('Using the remote proxy: ' + argv.proxy).green); } else { - var srv = 'http://' + proxy_addr + ':' + proxy_port + '/proxy.pac\n'; - var msg = 'The local proxy server is running...\nPlease use this PAC file: ' + srv.underline; - console.log(msg.green); - } - - if (new_relic_working) { - console.log('\nNew Relic is working...\n'.cyan); + console.log('Running locally without remote proxies.'.green); } } else if (cluster.isWorker) { - sogou_server_addr = sogou.new_sogou_proxy_addr(); - // console.log('default server: ' + sogou_server_addr); - change_sogou_server(); - var change_server_timer = setInterval(change_sogou_server, 10 * 60 * 1000); // every 10 mins - if ('function' === typeof change_server_timer.unref) { - change_server_timer.unref(); // doesn't exist in nodejs v0.8 - } - var ubuku_server = http.createServer(); ubuku_server.on('request', http_req_handler); - ubuku_server.on('connect', connect_req_handler); - ubuku_server.listen(local_port, local_addr).on('error', function(err) { + ubuku_server.listen(local_port, '0.0.0.0').on('error', function(err) { if (err.code === 'EADDRINUSE') { util.error('[ub.uku.js] Port number is already in use! Exiting now...'); process.exit(); diff --git a/server/utils.js b/server/utils.js index e7825c8e..8f89bfbe 100644 --- a/server/utils.js +++ b/server/utils.js @@ -24,59 +24,29 @@ var querystring = require('querystring'); var uglify = require('uglify-js'); var shared_urls = require('../shared/urls'); -var sogou = require('../shared/sogou'); var shared_tools = require('../shared/tools'); var string_starts_with = shared_tools.string_starts_with; var to_title_case = shared_tools.to_title_case; -function get_first_external_ip() { - try { - // only return the first external ip, which should be fine for usual cases - var interfaces = require('os').networkInterfaces(); - var i, j; - for (i in interfaces) { - if (interfaces.hasOwnProperty(i)) { - for (j = 0; j < interfaces[i].length; j++) { - var addr = interfaces[i][j]; - if (addr.family === 'IPv4' && !addr.internal) { - return addr.address; - } - } - } - } - } catch (err) { - return '127.0.0.1'; - } - - return '127.0.0.1'; // no external ip, so bind internal ip -} - - function get_real_target(req_path) { var real_target = {}; - // the 'path' in proxy requests should always start with http - if (string_starts_with(req_path, 'http')) { - real_target = url.parse(req_path); - } else { - var real_url = querystring.parse(url.parse(req_path).query).url; - if (real_url) { - // to use urlsafe_b64encode - real_url = real_url.replace('-', '+').replace('_', '/'); - - // fix possible padding errors - // real_url += (new Array((4 - real_url.length % 4) % 4 + 1)).join('='); - var i; - for (i = 0; i < (4 - real_url.length % 4) % 4; i++) { - real_url += '='; - } + var real_url = querystring.parse(url.parse(req_path).query).url; + if (real_url) { + // to use urlsafe_b64encode + real_url = real_url.replace('-', '+').replace('_', '/'); - var buf = new Buffer(real_url, 'base64'); - real_url = buf.toString(); - real_target = url.parse(real_url); + // fix possible padding errors + // real_url += (new Array((4 - real_url.length % 4) % 4 + 1)).join('='); + for (var i = 0; i < (4 - real_url.length % 4) % 4; i++) { + real_url += '='; } + + var buf = new Buffer(real_url, 'base64'); + real_url = buf.toString(); + real_target = url.parse(real_url); } if (!real_target.port) { real_target.port = 80; @@ -86,13 +56,12 @@ function get_real_target(req_path) { function is_valid_url(target_url) { - var i; - for (i = 0; i < shared_urls.url_regex_whitelist.length; i++) { - if (shared_urls.url_regex_whitelist[i].test(target_url)) { - return false; - } + for (var i = 0; i < shared_urls.url_regex_whitelist.length; i++) { + if (shared_urls.url_regex_whitelist[i].test(target_url)) { + return false; + } } - for (i = 0; i < shared_urls.url_regex_list.length; i++) { + for (var i = 0; i < shared_urls.url_regex_list.length; i++) { if (shared_urls.url_regex_list[i].test(target_url)) { return true; } @@ -105,82 +74,7 @@ function is_valid_url(target_url) { } -var utils_global = utils_global || {}; - -(function() { - utils_global.https_domains = []; - var i; - for (i = 0; i < shared_urls.url_list.length; i++) { - if (string_starts_with(shared_urls.url_list[i], 'https://')) { - var parsed_url = url.parse(shared_urls.url_list[i]); - if (parsed_url.hostname) { - utils_global.https_domains.push(parsed_url.hostname); - } - } - } -}()); - - -function is_valid_https_domain(domain_name) { - if (domain_name && utils_global.https_domains.indexOf(domain_name) >= 0) { - return true; - } - return false; -} - - -function renew_sogou_server(callback, depth) { - var new_addr = sogou.new_sogou_proxy_addr(); - // new_addr = 'h8.dxt.bj.ie.sogou.com'; - - if (typeof depth === 'undefined') { - depth = 0; - } else if (depth >= 10) { - // } else if (depth >= 3) { - callback(new_addr); - return; - } - - // console.log(new_addr + ' depth ' + depth); - - var options = { - host: new_addr, - headers: { - "Accept-Language": "en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4,zh-TW;q=0.2", - "Accept-Encoding": "deflate", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11", - "Accept-Charset": "gb18030,utf-8;q=0.7,*;q=0.3" - } - }; - - var req = http.request(options, function(res) { - if (400 === res.statusCode) { - callback(new_addr); - } else { - util.error('[ub.uku.js] statusCode for ' + new_addr + ' is unexpected: ' + res.statusCode); - renew_sogou_server(callback, depth + 1); - } - }); - - // http://goo.gl/G2CoU - req.on('socket', function(socket) { - socket.setTimeout(10 * 1000, function() { // 10s - req.abort(); - util.error('[ub.uku.js] Timeout for ' + new_addr + '. Aborted.'); - }); - }); - - req.on('error', function(err) { - util.error('[ub.uku.js] Error when testing ' + new_addr + ': ' + err); - renew_sogou_server(callback, depth + 1); - }); - - req.end(); -} - - -function filtered_request_headers(headers, forward_cookie) { +function filtered_request_headers(headerse) { var ret_headers = {}; var field; @@ -190,10 +84,6 @@ function filtered_request_headers(headers, forward_cookie) { if (field === 'proxy-connection') { ret_headers.Connection = headers['proxy-connection']; } - } else if (field === 'cookie') { - if (forward_cookie) { - ret_headers.Cookie = headers.cookie; - } } else if (field === 'user-agent') { if (headers['user-agent'].indexOf('CloudFront') !== -1 || headers['user-agent'].indexOf('CloudFlare') !== -1) { @@ -202,7 +92,9 @@ function filtered_request_headers(headers, forward_cookie) { } else { ret_headers['User-Agent'] = headers['user-agent']; } - } else if (field !== 'via' && (!string_starts_with(field, 'x-'))) { + } else if (field !== 'cookie' + && field !== 'via' + && (!string_starts_with(field, 'x-'))) { // in case some servers do not recognize lower-case headers, such as hacker news ret_headers[to_title_case(field)] = headers[field]; } @@ -212,7 +104,8 @@ function filtered_request_headers(headers, forward_cookie) { return ret_headers; } -function filtered_response_headers(headers, forward_cookie) { + +function filtered_response_headers(headers) { var res_headers = {}; var field; @@ -222,14 +115,14 @@ function filtered_response_headers(headers, forward_cookie) { if (field === 'proxy-connection') { res_headers.Connection = headers['proxy-connection']; } - } else if (field === 'set-cookie') { - // cannot set cookies for another domain in redirect mode - if (forward_cookie) { - res_headers['Set-Cookie'] = headers['set-cookie']; - } - } else if (field !== 'cache-control' && field !== 'expires' && // to improve caching - field !== 'pragma' && field !== 'age' && field !== 'via' && - field !== 'server' && (!string_starts_with(field, 'x-'))) { + } else if (field !== 'set-cookie' + && field !== 'cache-control' + && field !== 'expires' + && field !== 'pragma' + && field !== 'age' + && field !== 'via' + && field !== 'server' + && (!string_starts_with(field, 'x-'))) { res_headers[field] = headers[field]; } } @@ -241,7 +134,7 @@ function filtered_response_headers(headers, forward_cookie) { } -function static_responses(client_request, client_response, in_production, pac_file_content) { +function static_responses(client_request, client_response, pac_file_content) { if (client_request.url === '/crossdomain.xml') { client_response.writeHead(200, { 'Content-Type': 'text/xml', @@ -253,21 +146,6 @@ function static_responses(client_request, client_response, in_production, pac_fi return; } - if (client_request.url === '/status') { - var status_text = 'OK'; - if (in_production) { - status_text = 'Production OK'; - } - - client_response.writeHead(200, { - 'Content-Type': 'text/plain', - 'Content-Length': status_text.length.toString(), - 'Cache-Control': 'public, max-age=3600' - }); - client_response.end(status_text); - return; - } - if (client_request.url === '/proxy.pac') { client_response.writeHead(200, { 'Content-Type': 'application/x-ns-proxy-autoconfig', @@ -317,25 +195,9 @@ function generate_pac_file(proxy_addr_port) { } -function add_sogou_headers(req_headers, hostname) { - var sogou_auth = sogou.new_sogou_auth_str(); - var timestamp = Math.round(Date.now() / 1000).toString(16); - var sogou_tag = sogou.compute_sogou_tag(timestamp, hostname); - - req_headers['X-Sogou-Auth'] = sogou_auth; - req_headers['X-Sogou-Timestamp'] = timestamp; - req_headers['X-Sogou-Tag'] = sogou_tag; - req_headers['X-Forwarded-For'] = shared_tools.new_random_ip(); -} - - -exports.get_first_external_ip = get_first_external_ip; exports.get_real_target = get_real_target; exports.is_valid_url = is_valid_url; -exports.is_valid_https_domain = is_valid_https_domain; -exports.renew_sogou_server = renew_sogou_server; exports.filtered_request_headers = filtered_request_headers; exports.filtered_response_headers = filtered_response_headers; exports.static_responses = static_responses; exports.generate_pac_file = generate_pac_file; -exports.add_sogou_headers = add_sogou_headers; diff --git a/shared/sogou.js b/shared/sogou.js deleted file mode 100644 index d3e1137d..00000000 --- a/shared/sogou.js +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Allow you smoothly surf on many websites blocking non-mainland visitors. - * Copyright (C) 2012 - 2014 Bo Zhu http://zhuzhu.org - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - - -if (typeof string_starts_with !== 'function') { - // should only reach here in node.js runtime - var string_starts_with = require('./tools').string_starts_with; -} - - -function new_sogou_auth_str() { - "use strict"; - var auth_str = '/30/853edc6d49ba4e27'; - var i, tmp_str; - for (i = 0; i < 8; i++) { - tmp_str = ('0000' + Math.floor(Math.random() * 65536).toString(16)).slice(-4); - auth_str = tmp_str.toUpperCase() + auth_str; - } - return auth_str; -} - - -function new_sogou_proxy_addr() { - "use strict"; - var other_ip_addrs = [ - '220.181.118.128', - ]; - - // var random_num = Math.floor(Math.random() * (16 + 16 + other_ip_addrs.length)); // 0 ~ 15 edu, 0 ~ 15 dxt - // var random_num = Math.floor(Math.random() * 16); // only 0 ~ 15 edu - var random_num = Math.floor(Math.random() * 4); // only 0 ~ 3 edu - var proxy_addr; - - if (random_num < 16) { - if (6 === random_num || 7 === random_num) { - return new_sogou_proxy_addr(); // just retry - } - proxy_addr = 'h' + random_num + '.edu.bj.ie.sogou.com'; // 0 ~ 15 - } else if (random_num < 16 + 16) { - random_num -= 16; - proxy_addr = 'h' + random_num + '.dxt.bj.ie.sogou.com'; // (16 ~ 31) - 16 - } else { - random_num -= 16 + 16; - proxy_addr = other_ip_addrs[random_num]; - } - - return proxy_addr; -} - - -// based on http://goo.gl/th215 -function compute_sogou_tag(timestamp, target_link) { - "use strict"; - var hostname; - if (string_starts_with(target_link, 'http://')) { - hostname = target_link.match(/^http:\/\/(.[^:\/]+)/)[1]; - } else { - hostname = target_link; - } - var s = timestamp + hostname + 'SogouExplorerProxy'; - var total_len = s.length; - var numb_iter = Math.floor(total_len / 4); - var numb_left = total_len % 4; - - var hash = total_len; // output hash tag - - var i, low, high; - for (i = 0; i < numb_iter; i++) { - low = s.charCodeAt(4 * i + 1) * 256 + s.charCodeAt(4 * i); // right most 16 bits in little-endian - high = s.charCodeAt(4 * i + 3) * 256 + s.charCodeAt(4 * i + 2); // left most - - hash += low; - hash %= 0x100000000; - hash ^= hash << 16; - - hash ^= high << 11; - hash += hash >>> 11; - hash %= 0x100000000; - } - - switch (numb_left) { - case 3: - hash += (s.charCodeAt(total_len - 2) << 8) + s.charCodeAt(total_len - 3); - hash %= 0x100000000; - hash ^= hash << 16; - hash ^= s.charCodeAt(total_len - 1) << 18; - hash += hash >>> 11; - hash %= 0x100000000; - break; - case 2: - hash += (s.charCodeAt(total_len - 1) << 8) + s.charCodeAt(total_len - 2); - hash %= 0x100000000; - hash ^= hash << 11; - hash += hash >>> 17; - hash %= 0x100000000; - break; - case 1: - hash += s.charCodeAt(total_len - 1); - hash %= 0x100000000; - hash ^= hash << 10; - hash += hash >>> 1; - hash %= 0x100000000; - break; - default: - break; - } - - hash ^= hash << 3; - hash += hash >>> 5; - hash %= 0x100000000; - - hash ^= hash << 4; - hash += hash >>> 17; - hash %= 0x100000000; - - hash ^= hash << 25; - hash += hash >>> 6; - hash %= 0x100000000; - - // learnt from http://goo.gl/oRJ0o - hash = hash >>> 0; - - return ('00000000' + hash.toString(16)).slice(-8); -} - - -// export as a node.js module -var exports = exports || {}; -exports.new_sogou_auth_str = new_sogou_auth_str; -exports.compute_sogou_tag = compute_sogou_tag; -exports.new_sogou_proxy_addr = new_sogou_proxy_addr; diff --git a/test/run-all-tests.py b/test/run-all-tests.py index 1dc183e4..4137c78f 100755 --- a/test/run-all-tests.py +++ b/test/run-all-tests.py @@ -45,11 +45,11 @@ def start_server(): global server_process - print 'To start the server, and wait for 21 seconds to set up...' + print 'To start the server, and wait for 3 seconds to set up...' sys.stdout.flush() server_process = subprocess.Popen( - ['node', '../server/server.js', '--production', '--port=8888']) - time.sleep(21) + ['node', '../server/server.js', '--proxy=http://proxy.uku.im:8888']) + time.sleep(3) def stop_server(): @@ -76,10 +76,7 @@ def run_all_tests(): num_passed = 0 for file_name in os.listdir('.'): if file_name.startswith('test-') and file_name.endswith('.js'): - if file_name.endswith('-proxy.js'): - command = ['phantomjs', '--proxy=127.0.0.1:8888', file_name] - else: - command = ['phantomjs', file_name] + command = ['phantomjs', file_name] print print ' '.join(command) sys.stdout.flush() diff --git a/test/test-163-proxy.js b/test/test-163-proxy.js deleted file mode 100644 index d00892da..00000000 --- a/test/test-163-proxy.js +++ /dev/null @@ -1,17 +0,0 @@ -var page = require('webpage').create(); - -var test_url = 'http://ipservice.163.com/isFromMainland'; - -page.open(test_url, function(status) { - if (status !== 'success') { - console.error('Failed in opening page...'); - phantom.exit(1); - } else { - if (page.content.indexOf('true') === -1) { - console.error("Proxy didn't work well..."); - phantom.exit(2); - } else { - phantom.exit(0); - } - } -}); diff --git a/test/test-163-redirect.js b/test/test-163.js similarity index 100% rename from test/test-163-redirect.js rename to test/test-163.js diff --git a/test/test-google-proxy.js b/test/test-google-proxy.js deleted file mode 100644 index fe99bbfc..00000000 --- a/test/test-google-proxy.js +++ /dev/null @@ -1,12 +0,0 @@ -var page = require('webpage').create(); - -var test_url = 'http://google.com'; - -page.open(test_url, function(status) { - if (status !== 'success') { - phantom.exit(0); - } else { - console.error('Access control went wrong!'); - phantom.exit(1); - } -}); diff --git a/test/test-google-redirect.js b/test/test-google.js similarity index 100% rename from test/test-google-redirect.js rename to test/test-google.js diff --git a/test/test-https-google-proxy.js b/test/test-https-google-proxy.js deleted file mode 100644 index 8a1dc2d6..00000000 --- a/test/test-https-google-proxy.js +++ /dev/null @@ -1,13 +0,0 @@ -var page = require('webpage').create(); - -var test_url = 'https://www.google.com/'; - - -page.open(test_url, function(status) { - if (status === 'success') { - console.error('Access control failed in HTTPS proxy...'); - phantom.exit(1); - } else { - phantom.exit(0); - } -}); diff --git a/test/test-https-httpbin-proxy.js.bak b/test/test-https-httpbin-proxy.js.bak deleted file mode 100644 index ddeafcee..00000000 --- a/test/test-https-httpbin-proxy.js.bak +++ /dev/null @@ -1,23 +0,0 @@ -var page = require('webpage').create(); - -// var test_url = 'https://openapi.youku.com/'; -var test_url = 'https://httpbin.org/get'; -// may find a url from https://openapi.youku.com for testing - -page.onResourceReceived = function(res) { - if (res.stage === 'end') { - if (200 !== res.status) { - console.error("Proxy didn't work well..."); - phantom.exit(2); - } else { - phantom.exit(0); - } - } -}; - -page.open(test_url, function(status) { - if (status !== 'success') { - console.error('Failed in opening page...'); - phantom.exit(1); - } -}); diff --git a/test/test-localhost-redirect.js b/test/test-localhost.js similarity index 100% rename from test/test-localhost-redirect.js rename to test/test-localhost.js diff --git a/test/test-sohu-proxy.js b/test/test-sohu-proxy.js deleted file mode 100644 index e3c159cd..00000000 --- a/test/test-sohu-proxy.js +++ /dev/null @@ -1,17 +0,0 @@ -var page = require('webpage').create(); - -var test_url = 'http://hot.vrs.sohu.com/vrs_flash.action?vid=517980&af=1&g=0&referer=http%3A//tv.sohu.com/20111223/n330048227.shtml&t=0.6285748523660004'; - -page.open(test_url, function(status) { - if (status !== 'success') { - console.error('Failed in opening page...'); - phantom.exit(1); - } else { - if (page.content.indexOf('"data": null') !== -1 || page.content.indexOf('ipLimit') === -1) { - console.error("Proxy didn't work well..."); - phantom.exit(2); - } else { - phantom.exit(0); - } - } -}); diff --git a/test/test-sohu-redirect.js b/test/test-sohu.js similarity index 100% rename from test/test-sohu-redirect.js rename to test/test-sohu.js diff --git a/test/test-status-page.js b/test/test-status-page.js deleted file mode 100644 index 48477692..00000000 --- a/test/test-status-page.js +++ /dev/null @@ -1,18 +0,0 @@ -var page = require('webpage').create(); - -var test_url = 'http://127.0.0.1:8888/status'; - -page.open(test_url, function(status) { - if (status !== 'success') { - console.error('Failed in opening page...'); - phantom.exit(1); - } else { - if (page.content.indexOf('OK') === -1 - || page.content.indexOf('Production') === -1) { - console.error("Status page didn't work well..."); - phantom.exit(2); - } else { - phantom.exit(0); - } - } -}); diff --git a/test/test-youku-proxy.js b/test/test-youku-proxy.js deleted file mode 100644 index b0e6fc83..00000000 --- a/test/test-youku-proxy.js +++ /dev/null @@ -1,17 +0,0 @@ -var page = require('webpage').create(); - -var test_url = 'http://v.youku.com/player/getPlayList/VideoIDS/XMzkzMzkzNDYw/timezone/+08/version/5/'; - -page.open(test_url, function(status) { - if (status !== 'success') { - console.error('Failed in opening page...'); - phantom.exit(1); - } else { - if (page.content.indexOf('error') !== -1) { - console.error("Proxy didn't work well..."); - phantom.exit(2); - } else { - phantom.exit(0); - } - } -}); diff --git a/test/test-youku-redirect.js b/test/test-youku.js similarity index 100% rename from test/test-youku-redirect.js rename to test/test-youku.js