Merge pull request #5869 from neighbourhoodie/faster-norm-fields

Faster selector execution by pre-parsing fields

Author: janl

src/mango/src/mango_cursor.erl

@@ -358,7 +358,7 @@ explain(#cursor{} = Cursor) ->
             {dbname, DbName},
             {index, JSON},
             {partitioned, Partitioned},
-            {selector, Selector},
+            {selector, mango_util:join_keys(Selector)},
             {opts, {Opts}},
             {limit, Limit},
             {skip, Skip},

src/mango/src/mango_cursor_view.erl

@@ -1016,7 +1016,7 @@ required_fields_disjoint_fields_test() ->
     ?assertEqual([<<"field1">>, <<"field2">>, <<"field3">>], required_fields(Cursor1)),
     Fields2 = [<<"field1">>, <<"field2">>],
     Selector2 = to_selector(#{<<"field3">> => undefined, <<"field4">> => undefined}),
-    Cursor2 = #cursor{fields = Fields2, selector = to_selector(Selector2)},
+    Cursor2 = #cursor{fields = Fields2, selector = Selector2},
     ?assertEqual(
         [<<"field1">>, <<"field2">>, <<"field3">>, <<"field4">>], required_fields(Cursor2)
     ).

src/mango/src/mango_idx.erl

@@ -154,7 +154,7 @@ add(DDoc, Idx) ->
     {ok, NewDDoc1} = Mod:add(DDoc, Idx),
     NewDDoc2 = set_ddoc_partitioned(NewDDoc1, Idx),
     % Round trip through JSON for normalization
-    Body = ?JSON_DECODE(?JSON_ENCODE(NewDDoc2#doc.body)),
+    Body = ?JSON_DECODE(?JSON_ENCODE(mango_util:join_keys(NewDDoc2#doc.body))),
     {ok, NewDDoc2#doc{body = Body}}.
 
 remove(DDoc, Idx) ->

src/mango/src/mango_idx_view.erl

@@ -233,7 +233,7 @@ opts() ->
 make_view(Idx) ->
     View =
         {[
-            {<<"map">>, Idx#idx.def},
+            {<<"map">>, mango_util:join_keys(Idx#idx.def)},
             {<<"reduce">>, <<"_count">>},
             {<<"options">>, {Idx#idx.opts}}
         ]},
@@ -271,23 +271,26 @@ validate_ddoc(VProps) ->
 % the equivalent of a multi-query. But that's for another
 % day.
 
+indexable_fields(Selector) ->
+    [mango_util:join_field(F) || F <- indexable_paths(Selector)].
+
 % We can see through '$and' trivially
-indexable_fields({[{<<"$and">>, Args}]}) ->
-    lists:usort(lists:flatten([indexable_fields(A) || A <- Args]));
+indexable_paths({[{<<"$and">>, Args}]}) ->
+    lists:usort(lists:flatmap(fun(A) -> indexable_paths(A) end, Args));
 % So far we can't see through any other operator
-indexable_fields({[{<<"$", _/binary>>, _}]}) ->
+indexable_paths({[{<<"$", _/binary>>, _}]}) ->
     [];
 % If we have a field with a terminator that is locatable
 % using an index then the field is a possible index
-indexable_fields({[{Field, Cond}]}) ->
+indexable_paths({[{Field, Cond}]}) ->
     case indexable(Cond) of
         true ->
             [Field];
         false ->
             []
     end;
 % An empty selector
-indexable_fields({[]}) ->
+indexable_paths({[]}) ->
     [].
 
 % Check if a condition is indexable. The logical
@@ -320,8 +323,8 @@ indexable({[{<<"$", _/binary>>, _}]}) ->
 
 % For each field, return {Field, Range}
 field_ranges(Selector) ->
-    Fields = indexable_fields(Selector),
-    field_ranges(Selector, Fields).
+    Fields = indexable_paths(Selector),
+    [{mango_util:join_field(F), R} || {F, R} <- field_ranges(Selector, Fields)].
 
 field_ranges(Selector, Fields) ->
     field_ranges(Selector, Fields, []).

src/mango/src/mango_selector.erl

@@ -38,7 +38,7 @@ normalize(Selector) ->
     ],
     {NProps} = lists:foldl(fun(Step, Sel) -> Step(Sel) end, Selector, Steps),
     FieldNames = [Name || {Name, _} <- NProps],
-    case lists:member(<<>>, FieldNames) of
+    case lists:member([], FieldNames) of
         true ->
             ?MANGO_ERROR({invalid_selector, missing_field_name});
         false ->
@@ -210,7 +210,7 @@ norm_ops(Value) ->
 norm_fields({[]}) ->
     {[]};
 norm_fields(Selector) ->
-    norm_fields(Selector, <<>>).
+    norm_fields(Selector, []).
 
 % Operators where we can push the field names further
 % down the operator tree
@@ -237,7 +237,7 @@ norm_fields({[{<<"$keyMapMatch">>, Arg}]}, Path) ->
 % $default field. This also asserts that the $default
 % field is at the root as well as that it only has
 % a $text operator applied.
-norm_fields({[{<<"$default">>, {[{<<"$text">>, _Arg}]}}]} = Sel, <<>>) ->
+norm_fields({[{<<"$default">>, {[{<<"$text">>, _Arg}]}}]} = Sel, []) ->
     Sel;
 norm_fields({[{<<"$default">>, _}]} = Selector, _) ->
     ?MANGO_ERROR({bad_field, Selector});
@@ -249,12 +249,11 @@ norm_fields({[{<<"$", _/binary>>, _}]} = Cond, Path) ->
 % We've found a field name. Append it to the path
 % and skip this node as we unroll the stack as
 % the full path will be further down the branch.
-norm_fields({[{Field, Cond}]}, <<>>) ->
-    % Don't include the '.' for the first element of
-    % the path.
-    norm_fields(Cond, Field);
-norm_fields({[{Field, Cond}]}, Path) ->
-    norm_fields(Cond, <<Path/binary, ".", Field/binary>>);
+norm_fields({[{Field, Cond}]}, Path) when is_binary(Field) ->
+    {ok, F} = mango_util:parse_field(Field),
+    norm_fields({[{F, Cond}]}, Path);
+norm_fields({[{Field, Cond}]}, Path) when is_list(Field) ->
+    norm_fields(Cond, Path ++ Field);
 % An empty selector
 norm_fields({[]}, Path) ->
     {Path, {[]}};
@@ -575,7 +574,14 @@ match({[_, _ | _] = _Props} = Sel, _Value, _Cmp) ->
 % match against.
 
 has_required_fields(Selector, RequiredFields) ->
-    Remainder = has_required_fields_int(Selector, RequiredFields),
+    Paths = lists:map(
+        fun(Field) ->
+            {ok, Path} = mango_util:parse_field(Field),
+            Path
+        end,
+        RequiredFields
+    ),
+    Remainder = has_required_fields_int(Selector, Paths),
     Remainder == [].
 
 % Empty selector
@@ -634,6 +640,9 @@ has_required_fields_int([{[{Field, Cond}]} | Rest], RequiredFields) ->
     end.
 
 % Returns true if a field in the selector is a constant value e.g. {a: {$eq: 1}}
+is_constant_field(Selector, Field) when not is_list(Field) ->
+    {ok, Path} = mango_util:parse_field(Field),
+    is_constant_field(Selector, Path);
 is_constant_field({[]}, _Field) ->
     false;
 is_constant_field(Selector, Field) when not is_list(Selector) ->
@@ -653,7 +662,7 @@ is_constant_field([{[{_UnMatched, _}]} | Rest], Field) ->
 fields({[{<<"$", _/binary>>, Args}]}) when is_list(Args) ->
     lists:flatmap(fun fields/1, Args);
 fields({[{Field, _Cond}]}) ->
-    [Field];
+    [mango_util:join_field(Field)];
 fields({[]}) ->
     [].
 
@@ -1516,7 +1525,7 @@ match_object_test() ->
 
     % an inner empty object selector matches only empty objects
     SelEmptyField = normalize({[{<<"x">>, {[]}}]}),
-    ?assertEqual({[{<<"x">>, {[{<<"$eq">>, {[]}}]}}]}, SelEmptyField),
+    ?assertEqual({[{[<<"x">>], {[{<<"$eq">>, {[]}}]}}]}, SelEmptyField),
     ?assertEqual(false, match_int(SelEmptyField, Doc1)),
     ?assertEqual(true, match_int(SelEmptyField, Doc2)),
     ?assertEqual(false, match_int(SelEmptyField, Doc3)),
@@ -1525,7 +1534,7 @@ match_object_test() ->
 
     % negated empty object selector matches a value which is present and is not the empty object
     SelNotEmptyField = normalize({[{<<"$not">>, {[{<<"x">>, {[]}}]}}]}),
-    ?assertEqual({[{<<"x">>, {[{<<"$ne">>, {[]}}]}}]}, SelNotEmptyField),
+    ?assertEqual({[{[<<"x">>], {[{<<"$ne">>, {[]}}]}}]}, SelNotEmptyField),
     ?assertEqual(false, match_int(SelNotEmptyField, Doc1)),
     ?assertEqual(false, match_int(SelNotEmptyField, Doc2)),
     ?assertEqual(true, match_int(SelNotEmptyField, Doc3)),
@@ -1534,7 +1543,7 @@ match_object_test() ->
 
     % inner object selectors with fields match objects with at least those fields
     Sel1Field = normalize({[{<<"x">>, {[{<<"a">>, 1}]}}]}),
-    ?assertEqual({[{<<"x.a">>, {[{<<"$eq">>, 1}]}}]}, Sel1Field),
+    ?assertEqual({[{[<<"x">>, <<"a">>], {[{<<"$eq">>, 1}]}}]}, Sel1Field),
     ?assertEqual(false, match_int(Sel1Field, Doc1)),
     ?assertEqual(false, match_int(Sel1Field, Doc2)),
     ?assertEqual(true, match_int(Sel1Field, Doc3)),
@@ -1546,8 +1555,8 @@ match_object_test() ->
     ?assertEqual(
         {[
             {<<"$and">>, [
-                {[{<<"x.a">>, {[{<<"$eq">>, 1}]}}]},
-                {[{<<"x.b">>, {[{<<"$eq">>, 2}]}}]}
+                {[{[<<"x">>, <<"a">>], {[{<<"$eq">>, 1}]}}]},
+                {[{[<<"x">>, <<"b">>], {[{<<"$eq">>, 2}]}}]}
             ]}
         ]},
         Sel2Field
@@ -1560,7 +1569,7 @@ match_object_test() ->
 
     % check shorthand syntax
     SelShort = normalize({[{<<"x.b">>, 2}]}),
-    ?assertEqual({[{<<"x.b">>, {[{<<"$eq">>, 2}]}}]}, SelShort),
+    ?assertEqual({[{[<<"x">>, <<"b">>], {[{<<"$eq">>, 2}]}}]}, SelShort),
     ?assertEqual(false, match_int(SelShort, Doc1)),
     ?assertEqual(false, match_int(SelShort, Doc2)),
     ?assertEqual(false, match_int(SelShort, Doc3)),

src/mango/src/mango_selector_text.erl

@@ -177,10 +177,10 @@ convert(_Path, {[{<<"$", _/binary>> = Op, _}]}) ->
 convert(Path, {[{Field0, Cond}]}) ->
     {ok, PP0} =
         case Field0 of
-            <<>> ->
-                {ok, []};
-            _ ->
-                mango_util:parse_field(Field0)
+            F when is_binary(F) ->
+                mango_util:parse_field(F);
+            F when is_list(F) ->
+                {ok, F}
         end,
     % Later on, we perform a lucene_escape_user call on the
     % final Path, which calls parse_field again. Calling the function
@@ -394,7 +394,7 @@ append_sort_type(RawSortField, Selector) ->
     end.
 
 get_sort_type(Field, Selector) ->
-    Types = get_sort_types(Field, Selector, []),
+    Types = get_sort_types(Field, mango_util:join_keys(Selector), []),
     case lists:usort(Types) of
         [str] -> <<"_3astring<string>">>;
         [num] -> <<"_3anumber<number>">>;

src/mango/src/mango_util.erl

@@ -41,6 +41,8 @@
     join/2,
 
     parse_field/1,
+    join_field/1,
+    join_keys/1,
 
     cached_re/2
 ]).
@@ -370,6 +372,31 @@ parse_field_slow(Field) ->
     ),
     {ok, Path}.
 
+join_keys({Sel}) when is_list(Sel) ->
+    Pairs = [{join_field(K), join_keys(V)} || {K, V} <- Sel],
+    {Pairs};
+join_keys(Sel) when is_list(Sel) ->
+    [join_keys(S) || S <- Sel];
+join_keys(Sel) ->
+    Sel.
+
+join_field(Field) when is_list(Field) ->
+    Parts = [field_to_binary(F) || F <- Field],
+    binary_join(Parts, <<".">>);
+join_field(Field) ->
+    Field.
+
+% binary:join/2 is not available on all Erlang versions we support; it was
+% added in 28.0. For now, we use this function in its place, c.f.
+% https://www.erlang.org/doc/apps/stdlib/binary.html#join/2
+binary_join(Binaries, Separator) when is_list(Binaries), is_binary(Separator) ->
+    iolist_to_binary(lists:join(Separator, Binaries)).
+
+field_to_binary(Field) when is_list(Field) ->
+    list_to_binary(Field);
+field_to_binary(Field) when is_binary(Field) ->
+    Field.
+
 check_non_empty(Field, Parts) ->
     case lists:member(<<>>, Parts) of
         true ->