Remove emitting null value fields during data transformation for Sche…

…maConformingTransformer (#14351) * Remove emitting null value fields during data transformation. * Fix lint issues. * Revise based on comments
apache · Nov 12, 2024 · 447c518 · 447c518
1 parent 0180a11
commit 447c518
Show file tree

Hide file tree

Showing 2 changed files with 94 additions and 128 deletions.
diff --git a/.../java/org/apache/pinot/segment/local/recordtransformer/SchemaConformingTransformerV2.java b/.../java/org/apache/pinot/segment/local/recordtransformer/SchemaConformingTransformerV2.java
@@ -92,7 +92,6 @@
  * <pre>
  * {
  *   "a": 1,
- *   "c": null,
  *   "c.d": 3,
  *   "json_data": {
  *     "b": "2",
@@ -400,6 +399,9 @@ private ExtraFieldsContainer processField(SchemaTreeNode parentNode, Deque<Strin
     SchemaTreeNode currentNode = parentNode == null ? null : parentNode.getChild(key);
     String unindexableFieldSuffix = _transformerConfig.getUnindexableFieldSuffix();
     isIndexable = isIndexable && (null == unindexableFieldSuffix || !key.endsWith(unindexableFieldSuffix));
+    if (value == null) {
+      return extraFieldsContainer;
+    }
     if (!(value instanceof Map)) {
       // leaf node
       if (!isIndexable) {
@@ -413,7 +415,8 @@ private ExtraFieldsContainer processField(SchemaTreeNode parentNode, Deque<Strin
           }
           mergedTextIndexMap.put(keyJsonPath, value);
         } else {
-          // Out of schema
+          // The field is not mapped to one of the dedicated columns in the Pinot table schema. Thus it will be put
+          // into the extraField column of the table.
           if (storeIndexableExtras) {
             extraFieldsContainer.addIndexableEntry(key, value);
             mergedTextIndexMap.put(keyJsonPath, value);