Ensure with is tail call optimizable

Closes #6251

Author: José Valim

lib/elixir/src/elixir.hrl

@@ -12,7 +12,7 @@
   vars=#{},                %% a map of defined variables and their alias
   backup_vars=nil,         %% a copy of vars to be used on ^var
   export_vars=nil,         %% a dict of all variables defined in a particular clause
-  extra_guards=nil,        %% extra guards from args expansion
+  extra_guards=[],         %% extra guards from args expansion
   counter=#{},             %% a map counting the variables defined
   file=(<<"nofile">>)      %% the current scope filename
 }).

lib/elixir/src/elixir_clauses.erl

@@ -2,7 +2,7 @@
 %% fn, receive and friends. try is handled in elixir_try.
 -module(elixir_clauses).
 -export([match/3, clause/5, def/2, head/2,
-         'case'/3, 'receive'/3, 'try'/3, 'cond'/3,
+         'case'/3, 'receive'/3, 'try'/3, 'cond'/3, with/3,
          format_error/1]).
 -import(elixir_errors, [form_error/4]).
 -include("elixir.hrl").
@@ -112,6 +112,66 @@ expand_receive(Meta, {'after', _}, _Acc, E) ->
 expand_receive(Meta, {Key, _}, _Acc, E) ->
   form_error(Meta, ?key(E, file), ?MODULE, {unexpected_option, 'receive', Key}).
 
+%% With
+
+with(Meta, Args, E) ->
+  {Exprs, Opts0} =
+    case elixir_utils:split_last(Args) of
+      {_, LastArg} = SplitResult when is_list(LastArg) ->
+        SplitResult;
+      _ ->
+        {Args, []}
+    end,
+
+  {EExprs, {EE, HasMatch}} = lists:mapfoldl(fun expand_with/2, {E, false}, Exprs),
+  {EDo, Opts1} = expand_with_do(Meta, Opts0, EE),
+  {EOpts, Opts2} = expand_with_else(Meta, Opts1, E, HasMatch),
+
+  case Opts2 of
+    [{Key, _} | _] ->
+      form_error(Meta, ?key(E, file), elixir_clauses, {unexpected_option, with, Key});
+    [] ->
+      ok
+  end,
+
+  {{with, Meta, EExprs ++ [[{do, EDo} | EOpts]]}, E}.
+
+expand_with({'<-', Meta, [{Name, _, Ctx}, _] = Args}, Acc) when is_atom(Name), is_atom(Ctx) ->
+  expand_with({'=', Meta, Args}, Acc);
+expand_with({'<-', Meta, [Left, Right]}, {E, _HasMatch}) ->
+  {ERight, ER} = elixir_expand:expand(Right, E),
+  {[ELeft], EL}  = head([Left], E),
+  {{'<-', Meta, [ELeft, ERight]}, {elixir_env:mergev(EL, ER), true}};
+expand_with(Expr, {E, HasMatch}) ->
+  {EExpr, EE} = elixir_expand:expand(Expr, E),
+  {EExpr, {EE, HasMatch}}.
+
+expand_with_do(Meta, Opts, E) ->
+  case lists:keytake(do, 1, Opts) of
+    {value, {do, Expr}, RestOpts} ->
+      {EExpr, _} = elixir_expand:expand(Expr, E),
+      {EExpr, RestOpts};
+    false ->
+      form_error(Meta, ?key(E, file), elixir_expand, {missing_option, 'with', [do]})
+  end.
+
+expand_with_else(Meta, Opts, E, HasMatch) ->
+  case lists:keytake(else, 1, Opts) of
+    {value, Pair, RestOpts} ->
+      if
+        HasMatch ->
+          ok;
+        true ->
+          Message = "\"else\" clauses will never match because all patterns in \"with\" will always match",
+          elixir_errors:warn(?line(Meta), ?key(E, file), Message)
+      end,
+      Fun = expand_one(Meta, 'with', 'else', fun head/2),
+      EPair = expand_without_export(Meta, 'with', Fun, Pair, E),
+      {[EPair], RestOpts};
+    false ->
+      {[], Opts}
+  end.
+
 %% Try
 
 'try'(Meta, [], E) ->

lib/elixir/src/elixir_erl_clauses.erl

@@ -1,20 +1,16 @@
 %% Handle code related to args, guard and -> matching for case,
 %% fn, receive and friends. try is handled in elixir_erl_try.
 -module(elixir_erl_clauses).
--export([match/3, clause/6, clauses/3, guards/3, get_clauses/3, get_clauses/4]).
+-export([match/3, clause/6, clauses/3, guards/3, get_clauses/3]).
 -include("elixir.hrl").
 
 %% Get clauses under the given key.
 
 get_clauses(Key, Keyword, As) ->
-  get_clauses(Key, Keyword, As, false).
-get_clauses(Key, Keyword, As, AllowNil) ->
   case lists:keyfind(Key, 1, Keyword) of
     {Key, Clauses} when is_list(Clauses) ->
       [{As, Meta, Left, Right} || {'->', Meta, [Left, Right]} <- Clauses];
-    {Key, nil} when AllowNil ->
-      [];
-    false ->
+    _ ->
       []
   end.
 
@@ -28,19 +24,16 @@ match(Fun, Args, S) -> Fun(Args, S).
 %% Translate clauses with args, guards and expressions
 
 clause(Meta, Fun, Args, Expr, Guards, S) when is_list(Meta) ->
-  {TArgs, SA} = match(Fun, Args, S#elixir_erl{extra_guards=[]}),
-  {TExpr, SE} = elixir_erl_pass:translate(Expr,
-                  SA#elixir_erl{extra_guards=nil, export_vars=S#elixir_erl.export_vars}),
-
-  Extra = SA#elixir_erl.extra_guards,
-  TGuards = guards(Guards, Extra, SA),
+  {TArgs, SA} = match(Fun, Args, S),
+  SG = SA#elixir_erl{extra_guards=[]},
+  TGuards = guards(Guards, SA#elixir_erl.extra_guards, SG),
+  {TExpr, SE} = elixir_erl_pass:translate(Expr, SG#elixir_erl{export_vars=S#elixir_erl.export_vars}),
   {{clause, ?ann(Meta), TArgs, TGuards, unblock(TExpr)}, SE}.
 
 % Translate/Extract guards from the given expression.
 
 guards(Guards, Extra, S) ->
-  SG = S#elixir_erl{context=guard, extra_guards=nil},
-
+  SG = S#elixir_erl{context=guard},
   case Guards of
     [] -> case Extra of [] -> []; _ -> [Extra] end;
     _  -> [translate_guard(Guard, Extra, SG) || Guard <- Guards]

lib/elixir/src/elixir_erl_for.erl

@@ -50,11 +50,11 @@ translate_gen(_Meta, Left, Right, T, S) ->
   {TRight, SR} = elixir_erl_pass:translate(Right, S),
   {LeftArgs, LeftGuards} = elixir_utils:extract_guards(Left),
   {TLeft, SL} = elixir_erl_clauses:match(fun elixir_erl_pass:translate/2, LeftArgs,
-                                     SR#elixir_erl{extra=pin_guard, extra_guards=[]}),
+                                         SR#elixir_erl{extra=pin_guard}),
 
   TLeftGuards = elixir_erl_clauses:guards(LeftGuards, [], SL),
   ExtraGuards = [{nil, X} || X <- SL#elixir_erl.extra_guards],
-  SF = SL#elixir_erl{extra=S#elixir_erl.extra, extra_guards=nil},
+  SF = SL#elixir_erl{extra=S#elixir_erl.extra, extra_guards=[]},
 
   {TT, {TFilters, TS}} = translate_filters(T, SF),
 

lib/elixir/src/elixir_erl_pass.erl

@@ -46,7 +46,7 @@ translate({'__CALLER__', Meta, Atom}, #elixir_erl{def=Kind}=S) when is_atom(Atom
       {{atom, ?ann(Meta), nil}, S}
   end;
 
-translate({'super', Meta, [{Kind,Name}|Args]}, S) ->
+translate({'super', Meta, [{Kind, Name} | Args]}, S) ->
   %% In the expanded AST, super is used to invoke a function
   %% in the current module originated from a default clause
   %% or a super call.
@@ -76,8 +76,8 @@ translate({'&', Meta, [{'/', _, [{Fun, _, Atom}, Arity]}]}, S)
 translate({fn, Meta, Clauses}, S) ->
   Transformer = fun({'->', CMeta, [ArgsWithGuards, Expr]}, Acc) ->
     {Args, Guards} = elixir_utils:extract_splat_guards(ArgsWithGuards),
-    {TClause, TS } = elixir_erl_clauses:clause(CMeta, fun translate_fn_match/2,
-                                            Args, Expr, Guards, Acc),
+    {TClause, TS} = elixir_erl_clauses:clause(CMeta, fun translate_fn_match/2,
+                                              Args, Expr, Guards, Acc),
     {TClause, elixir_erl_var:mergec(S, TS)}
   end,
   {TClauses, NS} = lists:mapfoldl(Transformer, S, Clauses),
@@ -107,29 +107,28 @@ translate({'case', Meta, [Expr, Opts]}, S) ->
 %% Try
 
 translate({'try', Meta, [Opts]}, S) ->
-  SN = S#elixir_erl{extra=nil},
   Do = proplists:get_value('do', Opts, nil),
-  {TDo, SB} = translate(Do, SN),
+  {TDo, SB} = translate(Do, S),
 
   Catch = [Tuple || {X, _} = Tuple <- Opts, X == 'rescue' orelse X == 'catch'],
-  {TCatch, SC} = elixir_erl_try:clauses(Meta, Catch, mergec(SN, SB)),
+  {TCatch, SC} = elixir_erl_try:clauses(Meta, Catch, mergec(S, SB)),
 
   {TAfter, SA} = case lists:keyfind('after', 1, Opts) of
     {'after', After} ->
-      {TBlock, SAExtracted} = translate(After, mergec(SN, SC)),
+      {TBlock, SAExtracted} = translate(After, mergec(S, SC)),
       {unblock(TBlock), SAExtracted};
     false ->
-      {[], mergec(SN, SC)}
+      {[], mergec(S, SC)}
   end,
 
   Else = elixir_erl_clauses:get_clauses(else, Opts, match),
-  {TElse, SE} = elixir_erl_clauses:clauses(Meta, Else, mergec(SN, SA)),
+  {TElse, SE} = elixir_erl_clauses:clauses(Meta, Else, mergec(S, SA)),
   {{'try', ?ann(Meta), unblock(TDo), TElse, TCatch, TAfter}, mergec(S, SE)};
 
 %% Receive
 
 translate({'receive', Meta, [Opts]}, S) ->
-  Do = elixir_erl_clauses:get_clauses(do, Opts, match, true),
+  Do = elixir_erl_clauses:get_clauses(do, Opts, match),
 
   case lists:keyfind('after', 1, Opts) of
     false ->
@@ -143,11 +142,17 @@ translate({'receive', Meta, [Opts]}, S) ->
       {{'receive', ?ann(Meta), FClauses, FExpr, FAfter}, SC}
   end;
 
-%% Comprehensions
+%% Comprehensions and with
 
 translate({for, Meta, [_ | _] = Args}, S) ->
   elixir_erl_for:translate(Meta, Args, true, S);
 
+translate({with, Meta, [_ | _] = Args}, S) ->
+  {Exprs, [{do, Do} | Opts]} = elixir_utils:split_last(Args),
+  {ElseClause, SE} = translate_with_else(Meta, Opts, S),
+  {With, SD} = translate_with_do(Exprs, Do, ElseClause, elixir_erl_var:mergec(S, SE)),
+  {With, elixir_erl_var:mergec(S, SD)};
+
 %% Variables
 
 translate({'^', Meta, [{Name, VarMeta, Kind}]}, #elixir_erl{context=match, file=File} = S) when is_atom(Name), is_atom(Kind) ->
@@ -240,10 +245,10 @@ translate(Other, S) ->
 translate_case(true, Meta, Expr, Opts, S) ->
   Clauses = elixir_erl_clauses:get_clauses(do, Opts, match),
   {TExpr, SE} = translate(Expr, S),
-  {TClauses, SC} = elixir_erl_clauses:clauses(Meta, Clauses, SE#elixir_erl{extra=nil}),
-  {{'case', ?ann(Meta), TExpr, TClauses}, SC#elixir_erl{extra=SE#elixir_erl.extra}};
+  {TClauses, SC} = elixir_erl_clauses:clauses(Meta, Clauses, SE),
+  {{'case', ?ann(Meta), TExpr, TClauses}, SC};
 translate_case(false, Meta, Expr, Opts, S) ->
-  {Case, SC} = translate_case(true, Meta, Expr, Opts, S#elixir_erl{extra=nil}),
+  {Case, SC} = translate_case(true, Meta, Expr, Opts, S),
   {Case, elixir_erl_var:mergec(S, SC)}.
 
 translate_list([{'|', _, [_, _]=Args}], Fun, Acc, List) ->
@@ -355,6 +360,43 @@ returns_boolean(Condition, Body) ->
     false -> false
   end.
 
+%% with
+
+translate_with_else(Meta, [], S) ->
+  Ann = ?ann(?generated(Meta)),
+  {VarName, _, SC} = elixir_erl_var:build('_', S),
+  Var = {var, Ann, VarName},
+  {{clause, Ann, [Var], [], [Var]}, SC};
+translate_with_else(Meta, [{else, Else}], S) ->
+  Generated = ?generated(Meta),
+  ElseVarEx = {else, Generated, ?var_context},
+  {ElseVarErl, SV} = elixir_erl_var:assign(Generated, else, ?var_context, S),
+
+  RaiseVar = {catch_all, Generated, ?var_context},
+  RaiseExpr = {{'.', Generated, [erlang, error]}, Generated, [{with_clause, RaiseVar}]},
+  RaiseClause = {'->', Generated, [[RaiseVar], RaiseExpr]},
+
+  Case = {'case', [{export_vars, false} | Meta], [ElseVarEx, [{do, Else ++ [RaiseClause]}]]},
+  {TranslatedCase, SC} = elixir_erl_pass:translate(Case, SV),
+  {{clause, ?ann(Generated), [ElseVarErl], [], [TranslatedCase]}, SC}.
+
+translate_with_do([{'<-', Meta, [Left, Expr]} | Rest], Do, Else, S) ->
+  {Args, Guards} = elixir_utils:extract_guards(Left),
+  {TExpr, SR} = elixir_erl_pass:translate(Expr, S),
+  {TArgs, SA} = elixir_erl_clauses:match(fun elixir_erl_pass:translate/2, Args, SR),
+  TGuards = elixir_erl_clauses:guards(Guards, [], SA),
+  {TBody, SB} = translate_with_do(Rest, Do, Else, SA),
+
+  Ann = ?ann(?generated(Meta)),
+  Clause = {clause, Ann, [TArgs], TGuards, unblock(TBody)},
+  {{'case', Ann, TExpr, [Clause, Else]}, SB};
+translate_with_do([Expr | Rest], Do, Else, S) ->
+  {TExpr, TS} = elixir_erl_pass:translate(Expr, S),
+  {TRest, RS} = translate_with_do(Rest, Do, Else, TS),
+  {{block, 0, [TExpr | unblock(TRest)]}, RS};
+translate_with_do([], Do, _Else, S) ->
+  elixir_erl_pass:translate(Do, S).
+
 %% Maps and structs
 
 translate_map(Meta, [{'|', _Meta, [Update, Assocs]}], S) ->

lib/elixir/src/elixir_erl_try.erl

@@ -28,14 +28,14 @@ each_clause({'catch', Meta, Raw, Expr}, S) ->
 
 each_clause({rescue, Meta, [{in, _, [Left, Right]}], Expr}, S) ->
   {TempName, _, CS} = elixir_erl_var:build('_', S),
-  TempVar = {TempName, Meta, 'Elixir'},
+  TempVar = {TempName, Meta, ?var_context},
   {Parts, Safe, FS} = rescue_guards(Meta, TempVar, Right, CS),
   Body = rescue_clause_body(Left, Expr, Safe, TempVar, Meta),
   build_rescue(Meta, Parts, Body, FS);
 
 each_clause({rescue, Meta, [{VarName, _, Context} = Left], Expr}, S) when is_atom(VarName), is_atom(Context) ->
   {TempName, _, CS} = elixir_erl_var:build('_', S),
-  TempVar = {TempName, Meta, 'Elixir'},
+  TempVar = {TempName, Meta, ?var_context},
   Body = rescue_clause_body(Left, Expr, false, TempVar, Meta),
   build_rescue(Meta, [{TempVar, []}], Body, CS).
 

lib/elixir/src/elixir_erl_var.erl

@@ -1,6 +1,6 @@
 %% Convenience functions used to manipulate scope and its variables.
 -module(elixir_erl_var).
--export([translate/4, build/2,
+-export([translate/4, build/2, assign/4,
   load_binding/2, dump_binding/2,
   mergev/2, mergec/2, merge_vars/2, merge_opt_vars/2,
   warn_unsafe_var/4, format_error/1
@@ -10,57 +10,59 @@
 %% VAR HANDLING
 
 translate(Meta, Name, Kind, S) when is_atom(Kind); is_integer(Kind) ->
-  Ann = ?ann(Meta),
   Tuple = {Name, Kind},
-  Vars = S#elixir_erl.vars,
-  BackupVars = S#elixir_erl.backup_vars,
 
   {Current, Safe} =
-    case maps:find({Name, Kind}, Vars) of
+    case maps:find(Tuple, S#elixir_erl.vars) of
       {ok, {VarC, _, VarS}} -> {VarC, VarS};
       error -> {nil, true}
     end,
 
   case S#elixir_erl.context of
     match ->
       Previous =
-        case maps:find({Name, Kind}, BackupVars) of
+        case maps:find(Tuple, S#elixir_erl.backup_vars) of
           {ok, {BackupVarC, _, _}} -> BackupVarC;
           error -> nil
         end,
 
       if
         Current /= nil, Current /= Previous ->
-          {{var, Ann, Current}, S};
+          {{var, ?ann(Meta), Current}, S};
         true ->
-          %% We attempt to give vars a nice name because we
-          %% still use the unused vars warnings from erl_lint.
-          %%
-          %% Once we move the warning to Elixir compiler, we
-          %% can name vars as _@COUNTER.
-          {NewVar, Counter, NS} =
-            if
-              Kind /= nil ->
-                build('_', S);
-              true ->
-                build(Name, S)
-            end,
-
-          FS = NS#elixir_erl{
-            vars=maps:put(Tuple, {NewVar, Counter, true}, Vars),
-            export_vars=case S#elixir_erl.export_vars of
-              nil -> nil;
-              EV  -> maps:put(Tuple, {NewVar, Counter, true}, EV)
-            end
-          },
-
-          {{var, Ann, NewVar}, FS}
+          assign(Meta, Name, Kind, S)
       end;
     _  when Current /= nil ->
       warn_unsafe_var(Meta, S#elixir_erl.file, Name, Safe),
-      {{var, Ann, Current}, S}
+      {{var, ?ann(Meta), Current}, S}
   end.
 
+assign(Meta, Name, Kind, S) ->
+  Tuple = {Name, Kind},
+
+  %% We attempt to give vars a nice name because we
+  %% still use the unused vars warnings from erl_lint.
+  %%
+  %% Once we move the warning to Elixir compiler, we
+  %% can name vars as _@COUNTER.
+  {NewVar, Counter, NS} =
+    if
+      Kind /= nil ->
+        build('_', S);
+      true ->
+        build(Name, S)
+    end,
+
+  FS = NS#elixir_erl{
+    vars=maps:put(Tuple, {NewVar, Counter, true}, S#elixir_erl.vars),
+    export_vars=case S#elixir_erl.export_vars of
+      nil -> nil;
+      EV  -> maps:put(Tuple, {NewVar, Counter, true}, EV)
+    end
+  },
+
+  {{var, ?ann(Meta), NewVar}, FS}.
+
 build(Key, #elixir_erl{counter=Counter} = S) ->
   Cnt =
     case maps:find(Key, Counter) of