Add dynamic routes from crawler in generate mode

.eslintrc.js

@@ -5,4 +5,12 @@ module.exports = {
     // Allow sparse arrays
     'no-sparse-arrays': 'off',
   },
+  overrides: [
+    {
+      files: ['test/fixture/pages/**/*.vue'],
+      rules: {
+        'vue/singleline-html-element-content-newline': 'off',
+      },
+    },
+  ],
 }

README.md

@@ -224,9 +224,9 @@ You can combine sitemap and sitemap index configurations.
 
 ### `routes` (optional) - array | function
 
-- Default: `[]` or [`generate.routes`](https://nuxtjs.org/api/configuration-generate#routes) value from your `nuxt.config.js`
+- Default: `[]` or [`generate.routes`](https://nuxtjs.org/docs/2.x/configuration-glossary/configuration-generate#routes) value from your `nuxt.config.js`
 
-The `routes` parameter follows the same way than the `generate` [configuration](https://nuxtjs.org/api/configuration-generate#routes).
+The `routes` parameter follows the same way than the `generate` [configuration](https://nuxtjs.org/docs/2.x/configuration-glossary/configuration-generate#routes).
 
 See as well the [routes declaration](#routes-declaration) examples below.
 
@@ -240,7 +240,7 @@ The URL path of the generated sitemap.
 
 - Default:
   1. `sitemap.hostname` value from your `nuxt.config.js`
-  2. [`build.publicPath`](https://nuxtjs.org/api/configuration-build/#publicpath) value from your `nuxt.config.js` (⚠️ **deprecated**)
+  2. [`build.publicPath`](https://nuxtjs.org/docs/2.x/configuration-glossary/configuration-build#publicpath) value from your `nuxt.config.js` (⚠️ **deprecated**)
   3. [`os.hostname()`](https://nodejs.org/api/os.html#os_os_hostname) in **generate** or **spa** mode, or dynamically based on request URL (`headers.host`) in **ssr** mode
 
 This value is **mandatory** for generation sitemap file, and you should explicitly provide it in **generate** or **spa** mode.
@@ -257,17 +257,17 @@ Setting a negative value will disable the cache.
 
 Please note that after each invalidation, `routes` will be evaluated again (see [routes declaration](#routes-declaration) section).
 
-This option is only available in **ssr** mode.
+> **notice:** This option is only available in **ssr** mode.
 
 ### `etag` (optional) - object
 
-- Default: [`render.etag`](https://nuxtjs.org/api/configuration-render#etag) value from your `nuxt.config.js`
+- Default: [`render.etag`](https://nuxtjs.org/docs/2.x/configuration-glossary/configuration-render#etag) value from your `nuxt.config.js`
 
-Enable the etag cache header on sitemap (see [etag](https://nuxtjs.org/api/configuration-render#etag) docs for possible options).
+Enable the etag cache header on sitemap (see [etag](https://nuxtjs.org/docs/2.x/configuration-glossary/configuration-render#etag) docs for possible options).
 
 To disable etag for sitemap set `etag: false`
 
-This option is only available in **ssr** mode.
+> **notice:** This option is only available in **ssr** mode.
 
 ### `exclude` (optional) - string array
 
@@ -405,6 +405,16 @@ Example:
   </url>
 ```
 
+### `crawler` (optional) - boolean
+
+- Default: [`generate.crawler`](https://nuxtjs.org/docs/2.x/configuration-glossary/configuration-generate#crawler) value from your `nuxt.config.js` (by default `true` on Nuxt >= v2.13)
+
+Enable the addition of dynamic routes detected by the [Nuxt Crawler](https://nuxtjs.org/docs/2.x/configuration-glossary/configuration-generate#crawler) to each sitemap.
+
+To disable this feature set `false` on each sitemap configuration.
+
+> **notice:** This option is only available in **generate** mode and on **Nuxt >= v2.13**
+
 ### `defaults` (optional) - object
 
 - Default: `{}`
@@ -507,13 +517,13 @@ In addition, the `lastmod` can be defined for each linked sitemap.
 
 ### `etag` (optional) - object
 
-- Default: [`render.etag`](https://nuxtjs.org/api/configuration-render#etag) value from your `nuxt.config.js`
+- Default: [`render.etag`](https://nuxtjs.org/docs/2.x/configuration-glossary/configuration-render#etag) value from your `nuxt.config.js`
 
-Enable the etag cache header on sitemap index (See [etag](https://nuxtjs.org/api/configuration-render#etag) docs for possible options).
+Enable the etag cache header on sitemap index (See [etag](https://nuxtjs.org/docs/2.x/configuration-glossary/configuration-render#etag) docs for possible options).
 
 To disable etag for sitemap index set `etag: false`
 
-This option is only available in **ssr** mode.
+> **notice:** This option is only available in **ssr** mode.
 
 ### `gzip` (optional) - boolean
 
@@ -592,17 +602,16 @@ const axios = require('axios')
 }
 ```
 
-
 ## Hooks
 
-Hooks are listeners to Nuxt events. [Learn more](https://nuxtjs.org/api/configuration-hooks)
+Hooks are listeners to Nuxt events. [Learn more](https://nuxtjs.org/docs/2.x/configuration-glossary/configuration-hooks)
 
 You can register hooks on certain life cycle events.
 
-| Hook  | Arguments  | When  | 
-|---|---|---|
-| sitemap:generate:before  | (nuxt, sitemapOptions)  | Hook on before site generation  |
-| sitemap:generate:done  |  (nuxt) | Hook on sitemap generation finished |
+| Hook                    | Arguments              | When                                |
+| ----------------------- | ---------------------- | ----------------------------------- |
+| sitemap:generate:before | (nuxt, sitemapOptions) | Hook on before site generation      |
+| sitemap:generate:done   | (nuxt)                 | Hook on sitemap generation finished |
 
 ## License
 

lib/generator.js

@@ -47,7 +47,13 @@ async function generateSitemap(options, globalCache, nuxtInstance, depth = 0) {
 
   // Init cache
   const cache = {}
-  cache.staticRoutes = () => excludeRoutes(options.exclude, globalCache.staticRoutes)
+  cache.staticRoutes = () =>
+    excludeRoutes(
+      options.exclude,
+      options.crawler && globalCache.crawledRoutes
+        ? [...globalCache.staticRoutes, ...globalCache.crawledRoutes]
+        : globalCache.staticRoutes
+    )
   cache.routes = createRoutesCache(cache, options)
 
   // Generate sitemap.xml

lib/module.js

@@ -5,7 +5,7 @@ const fs = require('fs-extra')
 const { generateSitemaps } = require('./generator')
 const logger = require('./logger')
 const { registerSitemaps } = require('./middleware')
-const { getStaticRoutes } = require('./routes')
+const { getRoutes } = require('./routes')
 
 module.exports = async function module(moduleOptions) {
   const nuxtInstance = this
@@ -18,17 +18,20 @@ module.exports = async function module(moduleOptions) {
   }
 
   // Init cache
-  // a file "sitemap-routes.json" is written to "dist" dir on "build" mode
+  // On "build" mode, a "sitemap-routes.json" file is written to "dist" dir in order to init cache on "start" mode
   const jsonStaticRoutesPath = !nuxtInstance.options.dev
     ? path.resolve(nuxtInstance.options.buildDir, path.join('dist', 'sitemap-routes.json'))
     : null
-  const staticRoutes = fs.readJsonSync(jsonStaticRoutesPath, { throws: false })
-  const globalCache = { staticRoutes }
+  const globalCache = {
+    staticRoutes: fs.readJsonSync(jsonStaticRoutesPath, { throws: false }),
+    crawledRoutes: null,
+  }
 
   // Init static routes
   nuxtInstance.extendRoutes((routes) => {
-    // Create a cache for static routes
-    globalCache.staticRoutes = getStaticRoutes(routes)
+    // Set cache for static routes
+    const { staticRoutes } = getRoutes(routes)
+    globalCache.staticRoutes = staticRoutes
 
     // On run cmd "build"
     if (!nuxtInstance.options.dev) {
@@ -38,9 +41,17 @@ module.exports = async function module(moduleOptions) {
   })
 
   // On "generate" mode, generate static files for each sitemap or sitemapindex
-  nuxtInstance.nuxt.hook('generate:done', async () => {
+  nuxtInstance.nuxt.hook('generate:done', async ({ generatedRoutes }) => {
     await nuxtInstance.nuxt.callHook('sitemap:generate:before', nuxtInstance, options)
     logger.info('Generating sitemaps')
+    // On "generate.crawler" option enabled
+    if (nuxtInstance.options.generate.crawler) {
+      // Extract crawled routes from generated routes
+      const staticURLs = globalCache.staticRoutes.map(({ url }) => url)
+      const crawledURLs = Array.from(generatedRoutes).filter((route) => !staticURLs.includes(route))
+      // Set cache for crawled routes
+      globalCache.crawledRoutes = crawledURLs.map((url) => ({ url }))
+    }
     await Promise.all(options.map((options) => generateSitemaps(options, globalCache, nuxtInstance)))
     await nuxtInstance.nuxt.callHook('sitemap:generate:done', nuxtInstance)
   })

lib/options.js

@@ -31,6 +31,7 @@ function setDefaultSitemapOptions(options, nuxtInstance, isLinkedToSitemapIndex
     trailingSlash: false,
     lastmod: undefined,
     i18n: undefined,
+    crawler: nuxtInstance.options.generate.crawler,
     defaults: {},
   }
 

lib/routes.js

@@ -17,39 +17,48 @@ function excludeRoutes(patterns, routes) {
 }
 
 /**
- * Get static routes from Nuxt router and ignore dynamic routes
+ * Get all routes from Nuxt router
  *
  * @param   {Object} router
- * @returns {Array}
+ * @returns {Object}
  */
-function getStaticRoutes(router) {
-  return flattenStaticRoutes(router)
+function getRoutes(router) {
+  const flattenedRoutes = flattenRoutes(router)
+  // dissociate static and dynamic routes
+  const routes = {
+    staticRoutes: [],
+    dynamicRoutes: [],
+  }
+  flattenedRoutes.forEach((route) => {
+    const isStatic = ![':', '*'].some((c) => route.url.includes(c))
+    if (isStatic) {
+      routes.staticRoutes.push(route)
+    } else {
+      routes.dynamicRoutes.push(route)
+    }
+  })
+  return routes
 }
 
 /**
- * Recursively flatten all static routes and their nested routes
+ * Recursively flatten all routes and their nested routes
  *
  * @param   {Object} router
  * @param   {string} path
  * @param   {Array}  routes
  * @returns {Array}
  */
-function flattenStaticRoutes(router, path = '', routes = []) {
+function flattenRoutes(router, path = '', routes = []) {
   router.forEach((route) => {
-    // Skip dynamic routes
-    if ([':', '*'].some((c) => route.path.includes(c))) {
-      return
-    }
     // Nested routes
     if (route.children) {
-      return flattenStaticRoutes(route.children, path + route.path + '/', routes)
+      return flattenRoutes(route.children, path + route.path + '/', routes)
     }
     // Normalize url (without trailing slash)
     route.url = path.length && !route.path.length ? path.slice(0, -1) : path + route.path
-
     routes.push(route)
   })
   return routes
 }
 
-module.exports = { excludeRoutes, getStaticRoutes }
+module.exports = { excludeRoutes, getRoutes }

package.json

@@ -29,7 +29,7 @@
     "build": "nuxt build test/fixture",
     "dev": "nuxt test/fixture",
     "format": "yarn lint --fix",
-    "generate": "nuxt generate test/fixture",
+    "generate": "nuxt generate test/fixture --target static",
     "lint": "eslint . --ext js,vue",
     "preview": "standard-version --dry-run",
     "release": "standard-version && git push --follow-tags && npm publish",