feat: Normalise selector fields into a pre-parsed format to speed up lookups

Previously, an operator like {"x.y": 1} in a selector would require
"x.y" to be parsed using `mango_util:parse_field` into the form ["x",
"y"] first, before the path could be traversed. This happens every time
the selector is evaluated.

If we instead parse the field name during normalisation, then we can
speed up selector evaluation considerably. This does mean that the
normalised form is no longer a valid ejson form; instead of

    {[{<<"x.y">>, 1}]}

it is:

    {[{[<<"x">>, <<"y">>], 1}]}

There are a few places where a normalised selector needs to be turned
back into serialised JSON and written to disk or to an HTTP response, so
we have had to insert logic for turning these array keys back into
strings for such situations.

Author: jcoglan

src/mango/src/mango_cursor.erl

@@ -358,7 +358,7 @@ explain(#cursor{} = Cursor) ->
             {dbname, DbName},
             {index, JSON},
             {partitioned, Partitioned},
-            {selector, Selector},
+            {selector, mango_util:join_keys(Selector)},
             {opts, {Opts}},
             {limit, Limit},
             {skip, Skip},

src/mango/src/mango_cursor_view.erl

@@ -1016,7 +1016,7 @@ required_fields_disjoint_fields_test() ->
     ?assertEqual([<<"field1">>, <<"field2">>, <<"field3">>], required_fields(Cursor1)),
     Fields2 = [<<"field1">>, <<"field2">>],
     Selector2 = to_selector(#{<<"field3">> => undefined, <<"field4">> => undefined}),
-    Cursor2 = #cursor{fields = Fields2, selector = to_selector(Selector2)},
+    Cursor2 = #cursor{fields = Fields2, selector = Selector2},
     ?assertEqual(
         [<<"field1">>, <<"field2">>, <<"field3">>, <<"field4">>], required_fields(Cursor2)
     ).

src/mango/src/mango_idx_view.erl

@@ -233,7 +233,7 @@ opts() ->
 make_view(Idx) ->
     View =
         {[
-            {<<"map">>, Idx#idx.def},
+            {<<"map">>, mango_util:join_keys(Idx#idx.def)},
             {<<"reduce">>, <<"_count">>},
             {<<"options">>, {Idx#idx.opts}}
         ]},
@@ -271,23 +271,26 @@ validate_ddoc(VProps) ->
 % the equivalent of a multi-query. But that's for another
 % day.
 
+indexable_fields(Selector) ->
+    [mango_util:join_field(F) || F <- indexable_paths(Selector)].
+
 % We can see through '$and' trivially
-indexable_fields({[{<<"$and">>, Args}]}) ->
-    lists:usort(lists:flatten([indexable_fields(A) || A <- Args]));
+indexable_paths({[{<<"$and">>, Args}]}) ->
+    lists:usort(lists:flatmap(fun(A) -> indexable_paths(A) end, Args));
 % So far we can't see through any other operator
-indexable_fields({[{<<"$", _/binary>>, _}]}) ->
+indexable_paths({[{<<"$", _/binary>>, _}]}) ->
     [];
 % If we have a field with a terminator that is locatable
 % using an index then the field is a possible index
-indexable_fields({[{Field, Cond}]}) ->
+indexable_paths({[{Field, Cond}]}) ->
     case indexable(Cond) of
         true ->
             [Field];
         false ->
             []
     end;
 % An empty selector
-indexable_fields({[]}) ->
+indexable_paths({[]}) ->
     [].
 
 % Check if a condition is indexable. The logical
@@ -320,8 +323,8 @@ indexable({[{<<"$", _/binary>>, _}]}) ->
 
 % For each field, return {Field, Range}
 field_ranges(Selector) ->
-    Fields = indexable_fields(Selector),
-    field_ranges(Selector, Fields).
+    Fields = indexable_paths(Selector),
+    [{mango_util:join_field(F), R} || {F, R} <- field_ranges(Selector, Fields)].
 
 field_ranges(Selector, Fields) ->
     field_ranges(Selector, Fields, []).

src/mango/src/mango_selector.erl

@@ -38,7 +38,7 @@ normalize(Selector) ->
     ],
     {NProps} = lists:foldl(fun(Step, Sel) -> Step(Sel) end, Selector, Steps),
     FieldNames = [Name || {Name, _} <- NProps],
-    case lists:member(<<>>, FieldNames) of
+    case lists:member([], FieldNames) of
         true ->
             ?MANGO_ERROR({invalid_selector, missing_field_name});
         false ->
@@ -210,7 +210,7 @@ norm_ops(Value) ->
 norm_fields({[]}) ->
     {[]};
 norm_fields(Selector) ->
-    norm_fields(Selector, <<>>).
+    norm_fields(Selector, []).
 
 % Operators where we can push the field names further
 % down the operator tree
@@ -237,7 +237,7 @@ norm_fields({[{<<"$keyMapMatch">>, Arg}]}, Path) ->
 % $default field. This also asserts that the $default
 % field is at the root as well as that it only has
 % a $text operator applied.
-norm_fields({[{<<"$default">>, {[{<<"$text">>, _Arg}]}}]} = Sel, <<>>) ->
+norm_fields({[{<<"$default">>, {[{<<"$text">>, _Arg}]}}]} = Sel, []) ->
     Sel;
 norm_fields({[{<<"$default">>, _}]} = Selector, _) ->
     ?MANGO_ERROR({bad_field, Selector});
@@ -249,12 +249,11 @@ norm_fields({[{<<"$", _/binary>>, _}]} = Cond, Path) ->
 % We've found a field name. Append it to the path
 % and skip this node as we unroll the stack as
 % the full path will be further down the branch.
-norm_fields({[{Field, Cond}]}, <<>>) ->
-    % Don't include the '.' for the first element of
-    % the path.
-    norm_fields(Cond, Field);
-norm_fields({[{Field, Cond}]}, Path) ->
-    norm_fields(Cond, <<Path/binary, ".", Field/binary>>);
+norm_fields({[{Field, Cond}]}, Path) when is_binary(Field) ->
+    {ok, F} = mango_util:parse_field(Field),
+    norm_fields({[{F, Cond}]}, Path);
+norm_fields({[{Field, Cond}]}, Path) when is_list(Field) ->
+    norm_fields(Cond, Path ++ Field);
 % An empty selector
 norm_fields({[]}, Path) ->
     {Path, {[]}};
@@ -575,7 +574,9 @@ match({[_, _ | _] = _Props} = Sel, _Value, _Cmp) ->
 % match against.
 
 has_required_fields(Selector, RequiredFields) ->
-    Remainder = has_required_fields_int(Selector, RequiredFields),
+    Paths0 = [mango_util:parse_field(F) || F <- RequiredFields],
+    Paths = [P || {ok, P} <- Paths0],
+    Remainder = has_required_fields_int(Selector, Paths),
     Remainder == [].
 
 % Empty selector
@@ -634,6 +635,9 @@ has_required_fields_int([{[{Field, Cond}]} | Rest], RequiredFields) ->
     end.
 
 % Returns true if a field in the selector is a constant value e.g. {a: {$eq: 1}}
+is_constant_field(Selector, Field) when not is_list(Field) ->
+    {ok, Path} = mango_util:parse_field(Field),
+    is_constant_field(Selector, Path);
 is_constant_field({[]}, _Field) ->
     false;
 is_constant_field(Selector, Field) when not is_list(Selector) ->
@@ -653,7 +657,7 @@ is_constant_field([{[{_UnMatched, _}]} | Rest], Field) ->
 fields({[{<<"$", _/binary>>, Args}]}) when is_list(Args) ->
     lists:flatmap(fun fields/1, Args);
 fields({[{Field, _Cond}]}) ->
-    [Field];
+    [mango_util:join_field(Field)];
 fields({[]}) ->
     [].
 

src/mango/src/mango_selector_text.erl

@@ -177,10 +177,10 @@ convert(_Path, {[{<<"$", _/binary>> = Op, _}]}) ->
 convert(Path, {[{Field0, Cond}]}) ->
     {ok, PP0} =
         case Field0 of
-            <<>> ->
-                {ok, []};
-            _ ->
-                mango_util:parse_field(Field0)
+            F when is_binary(F) ->
+                mango_util:parse_field(F);
+            F when is_list(F) ->
+                {ok, F}
         end,
     % Later on, we perform a lucene_escape_user call on the
     % final Path, which calls parse_field again. Calling the function

src/mango/src/mango_util.erl

@@ -41,6 +41,8 @@
     join/2,
 
     parse_field/1,
+    join_field/1,
+    join_keys/1,
 
     cached_re/2
 ]).
@@ -370,6 +372,25 @@ parse_field_slow(Field) ->
     ),
     {ok, Path}.
 
+join_keys({Sel}) when is_list(Sel) ->
+    Pairs = [{join_field(K), join_keys(V)} || {K, V} <- Sel],
+    {Pairs};
+join_keys(Sel) when is_list(Sel) ->
+    [join_keys(S) || S <- Sel];
+join_keys(Sel) ->
+    Sel.
+
+join_field(Field) when is_list(Field) ->
+    Parts = [field_to_binary(F) || F <- Field],
+    binary:join(Parts, <<".">>);
+join_field(Field) ->
+    Field.
+
+field_to_binary(Field) when is_list(Field) ->
+    list_to_binary(Field);
+field_to_binary(Field) when is_binary(Field) ->
+    Field.
+
 check_non_empty(Field, Parts) ->
     case lists:member(<<>>, Parts) of
         true ->