Return 'eagain' when request queue is full and then try the request again.

In the worst case is the request queue remains full and we loop between
the NIF and Erlang forever trying over and over to enqueue the request. If
that happens we shouldn't take schedulers offline as the NIF calls are fast
and we shouldn't run out of memory as that is bounded.  CPU will show a lot
of activity, but progress will continue in Erlang.

c_src/async_nif.h

@@ -96,7 +96,7 @@ struct async_nif_state {
     new_env = req->env;                                                 \
     if (!req)                                                           \
       return enif_make_tuple2(env, enif_make_atom(env, "error"),        \
-                              enif_make_atom(env, "enomem"));           \
+                              enif_make_atom(env, "eagain"));           \
     do pre_block while(0);                                              \
     copy_of_args = (struct decl ## _args *)enif_alloc(sizeof(struct decl ## _args)); \
     if (!copy_of_args) {                                                \

src/async_nif.hrl

@@ -21,22 +21,25 @@
 %%
 %% -------------------------------------------------------------------
 
--define(ASYNC_NIF_CALL(Fun, Args),
-        begin
-            NIFRef = erlang:make_ref(),
-            case erlang:apply(Fun, [NIFRef|Args]) of
-                {ok, enqueued} ->
-                    receive
-                        {NIFRef, {error, shutdown}=Error} ->
-                            %% Work unit was queued, but not executed.
-                            Error;
-                        {NIFRef, {error, _Reason}=Error} ->
-                            %% Work unit returned an error.
-                            Error;
-                        {NIFRef, Reply} ->
-                            Reply
-                    end;
-                Other ->
-                    Other
-            end
-        end).
+-spec async_nif_enqueue(reference(), function(), [term()]) -> term() | {error, term()}.
+async_nif_enqueue(R, F, A) ->
+    case erlang:apply(F, [R|A]) of
+        {ok, enqueued} ->
+            receive
+                {R, {error, eagain}} ->
+                    %% Work unit was not queued, try again.
+                    async_nif_enqueue(R, F, A);
+                {R, {error, shutdown}=Error} ->
+                    %% Work unit was queued, but not executed.
+                    Error;
+                {R, {error, _Reason}=Error} ->
+                    %% Work unit returned an error.
+                    Error;
+                {R, Reply} ->
+                    Reply
+            end;
+        Other ->
+            Other
+    end.
+
+-define(ASYNC_NIF_CALL(Fun, Args), async_nif_enqueue(erlang:make_ref(), Fun, Args)).

src/wterl.erl

@@ -69,8 +69,6 @@
 
 -export([set_event_handler_pid/1]).
 
--include("async_nif.hrl").
-
 -ifdef(TEST).
 -ifdef(EQC).
 -include_lib("eqc/include/eqc.hrl").
@@ -90,6 +88,7 @@
 
 -on_load(init/0).
 
+-include("async_nif.hrl").
 -define(nif_stub, nif_stub_error(?LINE)).
 nif_stub_error(Line) ->
     erlang:nif_error({nif_not_loaded,module,?MODULE,line,Line}).