From 9e91ec6ccb155be15d0f213d6f1a57e71a6d7ffb Mon Sep 17 00:00:00 2001 From: Nikita Savchenko Date: Wed, 3 Jun 2020 12:53:13 +0300 Subject: [PATCH] Fix robots crawling the service; readme update --- readme.md | 8 +++++--- src/api.js | 9 +++++++-- src/modules/proxy.js | 6 +++--- {test-static => static-test}/index.html | 0 4 files changed, 15 insertions(+), 8 deletions(-) rename {test-static => static-test}/index.html (100%) diff --git a/readme.md b/readme.md index 329f5d2..35a7cac 100644 --- a/readme.md +++ b/readme.md @@ -39,7 +39,7 @@ run this proxy server on your end and figure out how to combine it with your app Technically, NodeJS proxy API works as follows: -1. Request to `/` returns sample application (see [src/test-static/index.html](src/test-static/index.html)) if enabled (see config). +1. Request to `/` returns sample application (see [src/static-test/index.html](src/static-test/index.html)) if enabled (see config). 2. Request to `/domain-name-or-masked-name/*` proxies requests to `domain-name-or-masked-name` with path `*`. 3. You can run the application using `npm install && npm run start` and request [http://localhost/www.googletagmanager.com/gtag/js?id=GTM-1234567](http://localhost/www.googletagmanager.com/gtag/js?id=GTM-1234567) (replace `GTM-1234567` with your GTM tag). That's it! @@ -54,7 +54,7 @@ In order to enable analytics proxying, you have to perform some DevOps in your i 3. **Modify your initial Google Tag Manager / Google Analytics script to request the proxied file** 1. Replace `https://www.googletagmanager.com/gtag/js?id=UA-123456-7` there to use `https://your-domain.com/gtm-proxy/www.googletagmanager.com/gtag/js?id=UA-123456-7` (or whatever path you've set up). Also, mask the URL by running `npm run mask ` in this repository so that ad-blockers won't block it right away. 2. For instance, if you run `npm run mask www.google-analytics.com/analytics.js`, you get this masked URL: `*(d3d3Lmdvb2dsZS1hbmFseXRpY3MuY29t)*/*(YW5hbHl0aWNzLmpz)*`. Use it in your script tag now: ``. - 3. The [example](src/test-static/index.html) in this repository uses unmasked `/www.googletagmanager.com/gtm.js` (which is equivalent of `http://localhost/www.googletagmanager.com/gtm.js`). + 3. The [example](src/static-test/index.html) in this repository uses unmasked `/www.googletagmanager.com/gtm.js` (which is equivalent of `http://localhost/www.googletagmanager.com/gtm.js`). 4. Test the thing! **This to consider before implementing the solution**: @@ -95,6 +95,8 @@ APP__STRIPPED_PATH=/gtm-proxy # reaching analytics-saviour so that next front end requests land to the same prefixed path # on your domain e.g. example.com/gtm-proxy/*(d3d3Lmdvb2dsZS1hbmFseXRpY3MuY29t)*/collect?.. # Because of this, she path you strip must be explicitly provided. +APP__ENV_NAME=local +# APP__ENV_NAME=local or APP__ENV_NAME=test (default) will display static content from `static-test`. ``` ### NodeJS Application @@ -113,7 +115,7 @@ Proxied: www.google-analytics.com/analytics.js Proxied: www.google-analytics.com/collect?v=1&_v=j73&a=531530768&t=pageview&_s=1&dl=http%3A%2F%2Flocalhost%2F&ul=ru&de=UTF-8&dt=Test&sd=24-bit&sr=1500x1000&vp=744x880&je=0&_u=AACAAEAB~&jid=&gjid=&cid=2E31579F-EE30-482F-9888-554A248A9495&tid=UA-98253329-1&_gid=1276054211.1554658225&z=1680756830&uip=1 ``` -Check the [test-static/index.html](test-static/index.html) file's code to see how to bind the proxied analytics to your front end. +Check the [static-test/index.html](static-test/index.html) file's code to see how to bind the proxied analytics to your front end. ### Proxy in Front of the Proxy diff --git a/src/api.js b/src/api.js index c444595..3808d52 100644 --- a/src/api.js +++ b/src/api.js @@ -28,14 +28,19 @@ export async function init () { app = express(); app.disable("x-powered-by"); + app.use("/robots.txt", (_, res) => res.status(200).set("Content-Type", "text/plain").send( + 'User-agent: *\nDisallow: /' + )); enableDefaultProxy(app); if (config.isLocalEnv) { - app.use("/", express.static(`${ __dirname }/../test-static`)); + app.use("/", express.static(`${ __dirname }/../static-test`)); } else { app.use("/", (_, res) => res.status(200).set("Content-Type", "text/html").send( - "Proxy APIIt works! Try requesting something like www.google-analytics.com/analytics.js." + 'Mirror' + + 'It works! Try requesting something like ' + + 'www.google-analytics.com/analytics.js.' )); } diff --git a/src/modules/proxy.js b/src/modules/proxy.js index 54f9ff8..8aef67d 100644 --- a/src/modules/proxy.js +++ b/src/modules/proxy.js @@ -85,9 +85,9 @@ export function createDefaultProxy (targetDomain, proxyOptionsOverride = {}) { ) { const parsedUrl = url.parse(unmasked); - const pverwrittenIp = req.headers["x-forwarded-for"] || req.headers["x-real-ip"]; // TODO: || req.connection.remoteAddress; // in case no proxy is used (dedicated domain) - const clientIp = pverwrittenIp - ? pverwrittenIp.split(/,\s?/g)[0] + const overwrittenIp = req.headers["x-forwarded-for"] || req.headers["x-real-ip"]; + const clientIp = overwrittenIp + ? overwrittenIp.split(/,\s?/g)[0] : req.connection.remoteAddress.split(":").pop(); const encodedIp = encodeURIComponent(clientIp); diff --git a/test-static/index.html b/static-test/index.html similarity index 100% rename from test-static/index.html rename to static-test/index.html