fix(module): split big sql queries into two (#2917)

src/module.ts

@@ -282,7 +282,7 @@ async function processCollectionItems(nuxt: Nuxt, collections: ResolvedCollectio
 
       filesCount += _keys.length
 
-      const list: Array<Array<string>> = []
+      const list: Array<[string, Array<string>]> = []
       for await (const chunk of chunks(_keys, 25)) {
         await Promise.all(chunk.map(async (key) => {
           key = key.substring(fixed.length)
@@ -313,14 +313,14 @@ async function processCollectionItems(nuxt: Nuxt, collections: ResolvedCollectio
       }
       // Sort by file name to ensure consistent order
       list.sort((a, b) => String(a[0]).localeCompare(String(b[0])))
-      collectionDump[collection.name]!.push(...list.map(([, sql]) => sql!))
+      collectionDump[collection.name]!.push(...list.flatMap(([, sql]) => sql!))
 
       collectionChecksum[collection.name] = hash(collectionDump[collection.name])
 
       collectionDump[collection.name]!.push(
         generateCollectionTableDefinition(infoCollection, { drop: false }),
-        `DELETE FROM ${infoCollection.tableName} WHERE id = 'checksum_${collection.name}'`,
-        generateCollectionInsert(infoCollection, { id: `checksum_${collection.name}`, version: collectionChecksum[collection.name] }),
+        `DELETE FROM ${infoCollection.tableName} WHERE id = 'checksum_${collection.name}';`,
+        ...generateCollectionInsert(infoCollection, { id: `checksum_${collection.name}`, version: collectionChecksum[collection.name] }),
       )
     }
   }

src/runtime/internal/studio/collection.ts

@@ -81,11 +81,11 @@ export function generateRecordUpdate(collection: CollectionInfo, stem: string, d
 }
 
 export function generateRecordDeletion(collection: CollectionInfo, stem: string) {
-  return `DELETE FROM ${collection.tableName} WHERE stem = '${stem}'`
+  return `DELETE FROM ${collection.tableName} WHERE stem = '${stem}';`
 }
 
 export function generateRecordSelectByColumn(collection: CollectionInfo, column: string, value: string) {
-  return `SELECT * FROM ${collection.tableName} WHERE ${column} = '${value}'`
+  return `SELECT * FROM ${collection.tableName} WHERE ${column} = '${value}';`
 }
 
 function computeValuesBasedOnCollectionSchema(collection: CollectionInfo, data: Record<string, unknown>) {

src/utils/collection.ts

@@ -98,7 +98,7 @@ function resolveSource(source: string | CollectionSource | CollectionSource[] |
 }
 
 // Convert collection data to SQL insert statement
-export function generateCollectionInsert(collection: ResolvedCollection, data: Record<string, unknown>) {
+export function generateCollectionInsert(collection: ResolvedCollection, data: Record<string, unknown>): string[] {
   const fields: string[] = []
   const values: Array<string | number | boolean> = []
   const sortedKeys = getOrderedSchemaKeys((collection.extendedSchema).shape)
@@ -141,9 +141,40 @@ export function generateCollectionInsert(collection: ResolvedCollection, data: R
   })
 
   let index = 0
-
-  return `INSERT INTO ${collection.tableName} VALUES (${'?, '.repeat(values.length).slice(0, -2)})`
+  const sql = `INSERT INTO ${collection.tableName} VALUES (${'?, '.repeat(values.length).slice(0, -2)});`
     .replace(/\?/g, () => values[index++] as string)
+
+  if (sql.length < 100000) {
+    return [sql]
+  }
+
+  // Split the SQL into multiple statements
+  const bigColumn = [...values].sort((a, b) => String(b).length - String(a).length)[0]
+  const bigColumnIndex = values.indexOf(bigColumn)
+  const bigColumnName = fields[bigColumnIndex]
+
+  if (typeof bigColumn === 'string') {
+    let splitIndex = Math.floor(bigColumn.length / 2)
+    while (['\'', '"', '\\'].includes(bigColumn[splitIndex])) {
+      splitIndex -= 1
+    }
+
+    const part1 = bigColumn.slice(0, splitIndex) + '\''
+    const part2 = '\'' + bigColumn.slice(splitIndex)
+
+    values[bigColumnIndex] = part1
+    index = 0
+
+    return [
+      `INSERT INTO ${collection.tableName} VALUES (${'?, '.repeat(values.length).slice(0, -2)});`
+        .replace(/\?/g, () => values[index++] as string),
+      `UPDATE ${collection.tableName} SET ${bigColumnName} = ${part2} WHERE id = ${values[0]};`,
+    ]
+  }
+
+  return [
+    sql,
+  ]
 }
 
 // Convert a collection with Zod schema to SQL table definition

src/utils/dev.ts

@@ -41,11 +41,11 @@ export async function startSocketServer(nuxt: Nuxt, options: ModuleOptions, mani
     })
   }
 
-  async function broadcast(collection: ResolvedCollection, key: string, insertQuery?: string) {
-    const removeQuery = `DELETE FROM ${collection.tableName} WHERE id = '${key}'`
+  async function broadcast(collection: ResolvedCollection, key: string, insertQuery?: string[]) {
+    const removeQuery = `DELETE FROM ${collection.tableName} WHERE id = '${key}';`
     await db.exec(removeQuery)
     if (insertQuery) {
-      await db.exec(insertQuery)
+      await Promise.all(insertQuery.map(query => db.exec(query)))
     }
 
     const collectionDump = manifest.dump[collection.name]
@@ -54,7 +54,7 @@ export async function startSocketServer(nuxt: Nuxt, options: ModuleOptions, mani
     const itemsToRemove = keyIndex === -1 ? 0 : 1
 
     if (insertQuery) {
-      collectionDump?.splice(indexToUpdate, itemsToRemove, insertQuery)
+      collectionDump?.splice(indexToUpdate, itemsToRemove, ...insertQuery)
     }
     else {
       collectionDump?.splice(indexToUpdate, itemsToRemove)
@@ -128,7 +128,8 @@ export async function watchContents(nuxt: Nuxt, options: ModuleOptions, manifest
 
       if (localCache && localCache.checksum === checksum) {
         db.exec(`DELETE FROM ${collection.tableName} WHERE id = '${keyInCollection}'`)
-        db.exec(generateCollectionInsert(collection, JSON.parse(localCache.parsedContent)))
+        const insertQuery = generateCollectionInsert(collection, JSON.parse(localCache.parsedContent))
+        await Promise.all(insertQuery.map(query => db.exec(query)))
         return
       }
 

test/unit/generateCollectionInsert.test.ts

@@ -21,10 +21,10 @@ describe('generateCollectionInsert', () => {
       meta: {},
     })
 
-    expect(sql).toBe([
+    expect(sql[0]).toBe([
       `INSERT INTO ${getTableName('content')}`,
       ' VALUES',
-      ' (\'foo.md\', 13, \'2022-01-01T00:00:00.000Z\', \'md\', \'{}\', \'untitled\', true, \'foo\')',
+      ' (\'foo.md\', 13, \'2022-01-01T00:00:00.000Z\', \'md\', \'{}\', \'untitled\', true, \'foo\');',
     ].join(''))
   })
 
@@ -50,10 +50,40 @@ describe('generateCollectionInsert', () => {
       date: new Date('2022-01-02'),
     })
 
-    expect(sql).toBe([
+    expect(sql[0]).toBe([
       `INSERT INTO ${getTableName('content')}`,
       ' VALUES',
-      ' (\'foo.md\', 42, \'2022-01-02T00:00:00.000Z\', \'md\', \'{}\', \'foo\', false, \'foo\')',
+      ' (\'foo.md\', 42, \'2022-01-02T00:00:00.000Z\', \'md\', \'{}\', \'foo\', false, \'foo\');',
+    ].join(''))
+  })
+
+  test('Split long values', () => {
+    const collection = resolveCollection('content', defineCollection({
+      type: 'data',
+      source: '**',
+      schema: z.object({
+        content: z.string().max(10000),
+      }),
+    }))!
+
+    const sql = generateCollectionInsert(collection, {
+      id: 'foo.md',
+      stem: 'foo',
+      extension: 'md',
+      meta: {},
+      content: 'a' + 'b'.repeat(50000) + 'c'.repeat(50000),
+    })
+
+    expect(sql[0]).toBe([
+      `INSERT INTO ${getTableName('content')}`,
+      ' VALUES',
+      ' (\'foo.md\', \'a' + 'b'.repeat(50000 - 1) + '\', \'md\', \'{}\', \'foo\');',
+    ].join(''))
+    expect(sql[1]).toBe([
+      `UPDATE ${getTableName('content')}`,
+      ' SET',
+      ' content = \'b' + 'c'.repeat(50000) + '\'',
+      ' WHERE id = \'foo.md\';',
     ].join(''))
   })
 })