Range Query Optimization (For sequential Vindex types)

go/vt/key/key.go

@@ -90,6 +90,11 @@ func Empty(id []byte) bool {
 // KeyRange helper methods
 //
 
+// Make a Key Range
+func NewKeyRange(start []byte, end []byte) *topodatapb.KeyRange {
+	return &topodatapb.KeyRange{Start: start, End: end}
+}
+
 // KeyRangeAdd adds two adjacent KeyRange values (in any order) into a single value. If the values are not adjacent,
 // it returns false.
 func KeyRangeAdd(a, b *topodatapb.KeyRange) (*topodatapb.KeyRange, bool) {

go/vt/vtgate/engine/routing.go

@@ -51,6 +51,9 @@ const (
 	// IN is for routing a statement to a multi shard.
 	// Requires: A Vindex, and a multi Values.
 	IN
+	// Between is for routing a statement to a multi shard
+	// Requires: A Vindex, and start and end Value.
+	Between
 	// MultiEqual is used for routing queries with IN with tuple clause
 	// Requires: A Vindex, and a multi Tuple Values.
 	MultiEqual
@@ -78,6 +81,7 @@ var opName = map[Opcode]string{
 	EqualUnique:   "EqualUnique",
 	Equal:         "Equal",
 	IN:            "IN",
+	Between:       "Between",
 	MultiEqual:    "MultiEqual",
 	Scatter:       "Scatter",
 	DBA:           "DBA",
@@ -157,6 +161,14 @@ func (rp *RoutingParameters) findRoute(ctx context.Context, vcursor VCursor, bin
 		default:
 			return rp.in(ctx, vcursor, bindVars)
 		}
+	case Between:
+		switch rp.Vindex.(type) {
+		case vindexes.SingleColumn:
+			return rp.between(ctx, vcursor, bindVars)
+		default:
+			// Only SingleColumn vindex supported.
+			return nil, nil, vterrors.VT13001("between supported on SingleColumn vindex only")
+		}
 	case MultiEqual:
 		switch rp.Vindex.(type) {
 		case vindexes.MultiColumn:
@@ -396,6 +408,19 @@ func (rp *RoutingParameters) inMultiCol(ctx context.Context, vcursor VCursor, bi
 	return rss, shardVarsMultiCol(bindVars, mapVals, isSingleVal), nil
 }
 
+func (rp *RoutingParameters) between(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) ([]*srvtopo.ResolvedShard, []map[string]*querypb.BindVariable, error) {
+	env := evalengine.NewExpressionEnv(ctx, bindVars, vcursor)
+	value, err := env.Evaluate(rp.Values[0])
+	if err != nil {
+		return nil, nil, err
+	}
+	rss, values, err := resolveShardsBetween(ctx, vcursor, rp.Vindex.(vindexes.Sequential), rp.Keyspace, value.TupleValues())
+	if err != nil {
+		return nil, nil, err
+	}
+	return rss, shardVars(bindVars, values), nil
+}
+
 func (rp *RoutingParameters) multiEqual(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) ([]*srvtopo.ResolvedShard, []map[string]*querypb.BindVariable, error) {
 	env := evalengine.NewExpressionEnv(ctx, bindVars, vcursor)
 	value, err := env.Evaluate(rp.Values[0])
@@ -520,6 +545,24 @@ func buildMultiColumnVindexValues(shardsValues [][][]sqltypes.Value) [][][]*quer
 	return shardsIds
 }
 
+func resolveShardsBetween(ctx context.Context, vcursor VCursor, vindex vindexes.Sequential, keyspace *vindexes.Keyspace, vindexKeys []sqltypes.Value) ([]*srvtopo.ResolvedShard, [][]*querypb.Value, error) {
+	// Convert vindexKeys to []*querypb.Value
+	ids := make([]*querypb.Value, len(vindexKeys))
+	for i, vik := range vindexKeys {
+		ids[i] = sqltypes.ValueToProto(vik)
+	}
+
+	// RangeMap using the Vindex
+	destinations, err := vindex.RangeMap(ctx, vcursor, vindexKeys[0], vindexKeys[1])
+	if err != nil {
+		return nil, nil, err
+
+	}
+
+	// And use the Resolver to map to ResolvedShards.
+	return vcursor.ResolveDestinations(ctx, keyspace.Name, ids, destinations)
+}
+
 func shardVars(bv map[string]*querypb.BindVariable, mapVals [][]*querypb.Value) []map[string]*querypb.BindVariable {
 	shardVars := make([]map[string]*querypb.BindVariable, len(mapVals))
 	for i, vals := range mapVals {

go/vt/vtgate/planbuilder/operators/sharded_routing.go

@@ -223,6 +223,9 @@ func (tr *ShardedRouting) resetRoutingLogic(ctx *plancontext.PlanningContext) Ro
 func (tr *ShardedRouting) searchForNewVindexes(ctx *plancontext.PlanningContext, predicate sqlparser.Expr) (Routing, bool) {
 	newVindexFound := false
 	switch node := predicate.(type) {
+	case *sqlparser.BetweenExpr:
+		return tr.planBetweenOp(ctx, node)
+
 	case *sqlparser.ComparisonExpr:
 		return tr.planComparison(ctx, node)
 
@@ -234,6 +237,35 @@ func (tr *ShardedRouting) searchForNewVindexes(ctx *plancontext.PlanningContext,
 	return nil, newVindexFound
 }
 
+func (tr *ShardedRouting) planBetweenOp(ctx *plancontext.PlanningContext, node *sqlparser.BetweenExpr) (routing Routing, foundNew bool) {
+	column, ok := node.Left.(*sqlparser.ColName)
+	if !ok {
+		return nil, false
+	}
+	var vdValue sqlparser.ValTuple = sqlparser.ValTuple([]sqlparser.Expr{node.From, node.To})
+
+	opcode := func(vindex *vindexes.ColumnVindex) engine.Opcode {
+		if _, ok := vindex.Vindex.(vindexes.Sequential); ok {
+			return engine.Between
+		}
+		return engine.Scatter
+	}
+
+	sequentialVdx := func(vindex *vindexes.ColumnVindex) vindexes.Vindex {
+		if _, ok := vindex.Vindex.(vindexes.Sequential); ok {
+			return vindex.Vindex
+		}
+		// if vindex is not of type Sequential, we can't use this vindex at all
+		return nil
+	}
+
+	val := makeEvalEngineExpr(ctx, vdValue)
+	if val == nil {
+		return nil, false
+	}
+	return nil, tr.haveMatchingVindex(ctx, node, vdValue, column, val, opcode, sequentialVdx)
+}
+
 func (tr *ShardedRouting) planComparison(ctx *plancontext.PlanningContext, cmp *sqlparser.ComparisonExpr) (routing Routing, foundNew bool) {
 	switch cmp.Operator {
 	case sqlparser.EqualOp:

go/vt/vtgate/planbuilder/plan_test.go

@@ -84,6 +84,8 @@ func (s *planTestSuite) TestPlan() {
 	s.addPKsProvided(vschema, "user", []string{"user_extra"}, []string{"id", "user_id"})
 	s.addPKsProvided(vschema, "ordering", []string{"order"}, []string{"oid", "region_id"})
 	s.addPKsProvided(vschema, "ordering", []string{"order_event"}, []string{"oid", "ename"})
+	s.addPKsProvided(vschema, "sequential", []string{"seq_tbl"}, []string{"id"})
+	s.addPKsProvided(vschema, "sequential", []string{"seq_multicol_tbl"}, []string{"cola", "colb"})
 
 	// You will notice that some tests expect user.Id instead of user.id.
 	// This is because we now pre-create vindex columns in the symbol

go/vt/vtgate/planbuilder/testdata/filter_cases.json

@@ -4820,5 +4820,100 @@
         "user.authoritative"
       ]
     }
+  },
+  {
+    "comment": "Between clause on primary indexed id column (binary vindex on id)",
+    "query": "select id from seq_tbl where id between 1 and 5",
+    "plan": {
+      "QueryType": "SELECT",
+      "Original": "select id from seq_tbl where id between 1 and 5",
+      "Instructions": {
+        "OperatorType": "Route",
+        "Variant": "Between",
+        "Keyspace": {
+          "Name": "sequential",
+          "Sharded": true
+        },
+        "FieldQuery": "select id from seq_tbl where 1 != 1",
+        "Query": "select id from seq_tbl where id between 1 and 5",
+        "Table": "seq_tbl",
+        "Values": [
+          "(1, 5)"
+        ],
+        "Vindex": "binary"
+      },
+      "TablesUsed": [
+        "sequential.seq_tbl"
+      ]
+    }
+  },
+{
+    "comment": "Between clause on group_id column (xxhash vindex on group_id)",
+    "query": "select id, group_id from seq_tbl where group_id between 1 and 5",
+    "plan": {
+      "QueryType": "SELECT",
+      "Original": "select id, group_id from seq_tbl where group_id between 1 and 5",
+      "Instructions": {
+        "OperatorType": "Route",
+        "Variant": "Scatter",
+        "Keyspace": {
+          "Name": "sequential",
+          "Sharded": true
+        },
+        "FieldQuery": "select id, group_id from seq_tbl where 1 != 1",
+        "Query": "select id, group_id from seq_tbl where group_id between 1 and 5",
+        "Table": "seq_tbl"
+      },
+      "TablesUsed": [
+        "sequential.seq_tbl"
+      ]
+    }
+  },
+{
+    "comment": "Between clause on col2 column (there is no vindex on this column)",
+    "query": "select id, col2 from seq_tbl where col2 between 10 and 50",
+    "plan": {
+      "QueryType": "SELECT",
+      "Original": "select id, col2 from seq_tbl where col2 between 10 and 50",
+      "Instructions": {
+        "OperatorType": "Route",
+        "Variant": "Scatter",
+        "Keyspace": {
+          "Name": "sequential",
+          "Sharded": true
+        },
+        "FieldQuery": "select id, col2 from seq_tbl where 1 != 1",
+        "Query": "select id, col2 from seq_tbl where col2 between 10 and 50",
+        "Table": "seq_tbl"
+      },
+      "TablesUsed": [
+        "sequential.seq_tbl"
+      ]
+    }
+},
+{
+  "comment": "Between clause on multicolumn vindex (cola,colb)",
+  "query": "select cola,colb,col1 from seq_multicol_tbl where cola between 1 and 5",
+  "plan": {
+    "QueryType": "SELECT",
+    "Original": "select cola,colb,col1 from seq_multicol_tbl where cola between 1 and 5",
+    "Instructions": {
+      "OperatorType": "Route",
+      "Variant": "Scatter",
+      "Keyspace": {
+        "Name": "sequential",
+        "Sharded": true
+      },
+      "FieldQuery": "select cola, colb, col1 from seq_multicol_tbl where 1 != 1",
+      "Query": "select cola, colb, col1 from seq_multicol_tbl where cola between 1 and 5",
+      "Table": "seq_multicol_tbl",
+      "Values": [
+        "(1, 5)"
+      ]
+    },
+    "TablesUsed": [
+      "sequential.seq_multicol_tbl"
+    ]
   }
+}
 ]

go/vt/vtgate/planbuilder/testdata/vschemas/schema.json

@@ -948,6 +948,61 @@
           ]
         }
       }
+    },
+    "sequential": {
+      "sharded": true,
+      "vindexes": {
+        "xxhash": {
+          "type": "xxhash"
+        },
+        "binary": {
+          "type": "binary"
+        },
+        "multicolIdx": {
+          "type": "multiCol_test"
+        }
+      },
+      "tables": {
+        "seq_tbl": {
+          "column_vindexes": [
+            {
+              "column": "id",
+              "name": "binary"
+            },
+            {
+              "column": "group_id",
+              "name": "xxhash"
+            }
+          ],
+          "columns": [
+            {
+              "name": "col1",
+              "type": "VARCHAR"
+            },
+            {
+              "name": "col2",
+              "type": "INT16"
+            }
+          ]
+        },
+        "seq_multicol_tbl": {
+          "column_vindexes": [
+            {
+              "columns": [
+                "cola",
+                "colb"
+              ],
+              "name": "multicolIdx"
+            }
+          ],
+          "columns" : [
+            {
+              "name": "col1",
+              "type": "INT16"
+            }
+          ]
+          }
+      }
     }
   }
 }

go/vt/vtgate/vindexes/binary.go

@@ -30,6 +30,7 @@ var (
 	_ Reversible      = (*Binary)(nil)
 	_ Hashing         = (*Binary)(nil)
 	_ ParamValidating = (*Binary)(nil)
+	_ Sequential      = (*Binary)(nil)
 )
 
 // Binary is a vindex that converts binary bits to a keyspace id.
@@ -108,6 +109,20 @@ func (*Binary) ReverseMap(_ VCursor, ksids [][]byte) ([]sqltypes.Value, error) {
 	return reverseIds, nil
 }
 
+// RangeMap can map ids to key.Destination objects.
+func (vind *Binary) RangeMap(ctx context.Context, vcursor VCursor, startId sqltypes.Value, endId sqltypes.Value) ([]key.Destination, error) {
+	startKsId, err := vind.Hash(startId)
+	if err != nil {
+		return nil, err
+	}
+	endKsId, err := vind.Hash(endId)
+	if err != nil {
+		return nil, err
+	}
+	out := []key.Destination{&key.DestinationKeyRange{KeyRange: key.NewKeyRange(startKsId, endKsId)}}
+	return out, nil
+}
+
 // UnknownParams implements the ParamValidating interface.
 func (vind *Binary) UnknownParams() []string {
 	return vind.unknownParams

go/vt/vtgate/vindexes/binary_test.go

@@ -24,6 +24,7 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"vitess.io/vitess/go/sqltypes"
@@ -145,3 +146,20 @@ func TestBinaryReverseMap(t *testing.T) {
 		t.Errorf("ReverseMap(): %v, want %s", err, wantErr)
 	}
 }
+
+// TestBinaryRangeMap takes start and env values,
+// and checks against a destination keyrange.
+func TestBinaryRangeMap(t *testing.T) {
+
+	startInterval := "0x01"
+	endInterval := "0x10"
+
+	got, err := binOnlyVindex.(Sequential).RangeMap(context.Background(), nil, sqltypes.NewHexNum([]byte(startInterval)),
+		sqltypes.NewHexNum([]byte(endInterval)))
+	require.NoError(t, err)
+	want := "DestinationKeyRange(01-10)"
+	if !assert.Equal(t, got[0].String(), want) {
+		t.Errorf("RangeMap(): %+v, want %+v", got, want)
+	}
+
+}

go/vt/vtgate/vindexes/numeric.go

@@ -31,6 +31,7 @@ var (
 	_ Reversible      = (*Numeric)(nil)
 	_ Hashing         = (*Numeric)(nil)
 	_ ParamValidating = (*Numeric)(nil)
+	_ Sequential      = (*Numeric)(nil)
 )
 
 // Numeric defines a bit-pattern mapping of a uint64 to the KeyspaceId.
@@ -108,6 +109,20 @@ func (*Numeric) ReverseMap(_ VCursor, ksids [][]byte) ([]sqltypes.Value, error)
 	return reverseIds, nil
 }
 
+// RangeMap implements Between.
+func (vind *Numeric) RangeMap(ctx context.Context, vcursor VCursor, startId sqltypes.Value, endId sqltypes.Value) ([]key.Destination, error) {
+	startKsId, err := vind.Hash(startId)
+	if err != nil {
+		return nil, err
+	}
+	endKsId, err := vind.Hash(endId)
+	if err != nil {
+		return nil, err
+	}
+	out := []key.Destination{&key.DestinationKeyRange{KeyRange: key.NewKeyRange(startKsId, endKsId)}}
+	return out, nil
+}
+
 // UnknownParams implements the ParamValidating interface.
 func (vind *Numeric) UnknownParams() []string {
 	return vind.unknownParams