n-api: detect deadlocks in thread-safe function

We introduce status `napi_would_deadlock` to be used as a return status
by `napi_call_threadsafe_function` if the call is made with
`napi_tsfn_blocking` on the main thread and the queue is full.

PR-URL: https://github.com/nodejs/node/pull/32689
Fixes: https://github.com/nodejs/node/issues/32615
Signed-off-by: Gabriel Schulhof <gabriel.schulhof@intel.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Anna Henningsen <anna@addaleax.net>
Reviewed-By: Colin Ihrig <cjihrig@gmail.com>
Reviewed-By: David Carlier <devnexen@gmail.com>
Reviewed-By: Chengzhong Wu <legendecas@gmail.com>
Reviewed-By: Michael Dawson <michael_dawson@ca.ibm.com>

test/node-api/test_threadsafe_function/binding.c

@@ -267,6 +267,60 @@ static napi_value StartThreadNoJsFunc(napi_env env, napi_callback_info info) {
     /** block_on_full */true, /** alt_ref_js_cb */true);
 }
 
+static void DeadlockTestDummyMarshaller(napi_env env,
+                                        napi_value empty0,
+                                        void* empty1,
+                                        void* empty2) {}
+
+static napi_value TestDeadlock(napi_env env, napi_callback_info info) {
+  napi_threadsafe_function tsfn;
+  napi_status status;
+  napi_value async_name;
+  napi_value return_value;
+
+  // Create an object to store the returned information.
+  NAPI_CALL(env, napi_create_object(env, &return_value));
+
+  // Create a string to be used with the thread-safe function.
+  NAPI_CALL(env, napi_create_string_utf8(env,
+                                     "N-API Thread-safe Function Deadlock Test",
+                                     NAPI_AUTO_LENGTH,
+                                     &async_name));
+
+  // Create the thread-safe function with a single queue slot and a single thread.
+  NAPI_CALL(env, napi_create_threadsafe_function(env,
+                                                 NULL,
+                                                 NULL,
+                                                 async_name,
+                                                 1,
+                                                 1,
+                                                 NULL,
+                                                 NULL,
+                                                 NULL,
+                                                 DeadlockTestDummyMarshaller,
+                                                 &tsfn));
+
+  // Call the threadsafe function. This should succeed and fill the queue.
+  NAPI_CALL(env, napi_call_threadsafe_function(tsfn, NULL, napi_tsfn_blocking));
+
+  // Call the threadsafe function. This should not block, but return
+  // `napi_would_deadlock`. We save the resulting status in an object to be
+  // returned.
+  status = napi_call_threadsafe_function(tsfn, NULL, napi_tsfn_blocking);
+  add_returned_status(env,
+                      "deadlockTest",
+                      return_value,
+                      "Main thread would deadlock",
+                      napi_would_deadlock,
+                      status);
+
+  // Clean up the thread-safe function before returning.
+  NAPI_CALL(env, napi_release_threadsafe_function(tsfn, napi_tsfn_release));
+
+  // Return the result.
+  return return_value;
+}
+
 // Module init
 static napi_value Init(napi_env env, napi_value exports) {
   size_t index;
@@ -305,6 +359,7 @@ static napi_value Init(napi_env env, napi_value exports) {
     DECLARE_NAPI_PROPERTY("StopThread", StopThread),
     DECLARE_NAPI_PROPERTY("Unref", Unref),
     DECLARE_NAPI_PROPERTY("Release", Release),
+    DECLARE_NAPI_PROPERTY("TestDeadlock", TestDeadlock),
   };
 
   NAPI_CALL(env, napi_define_properties(env, exports,

test/node-api/test_threadsafe_function/binding.gyp

@@ -2,7 +2,10 @@
   'targets': [
     {
       'target_name': 'binding',
-      'sources': ['binding.c']
+      'sources': [
+        'binding.c',
+        '../../js-native-api/common.c'
+      ]
     }
   ]
 }

test/node-api/test_threadsafe_function/test.js

@@ -210,8 +210,13 @@ new Promise(function testWithoutJSMarshaller(resolve) {
 }))
 .then((result) => assert.strictEqual(result.indexOf(0), -1))
 
-// Start a child process to test rapid teardown
+// Start a child process to test rapid teardown.
 .then(() => testUnref(binding.MAX_QUEUE_SIZE))
 
-// Start a child process with an infinite queue to test rapid teardown
-.then(() => testUnref(0));
+// Start a child process with an infinite queue to test rapid teardown.
+.then(() => testUnref(0))
+
+// Test deadlock prevention.
+.then(() => assert.deepStrictEqual(binding.TestDeadlock(), {
+  deadlockTest: 'Main thread would deadlock'
+}));