From dc3f8a52d393b72bed950ccf6d586da1b1bfaf6d Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Mon, 21 Sep 2020 17:09:41 +0900
Subject: [PATCH v31 07/11] Introduce foreign transaction launcher and resolver
 processes.

This commits introduces to new background processes: foreign
transaction launcher and resolvers. With this change, users no longer
need to use pg_resolve_foreign_xact() to resolve foreign transaction
prepared by PREPARE TRANSACTION and left by COMMIT/ROLLBACK
TRANSACTION. These foreign transactions are resolved in background by
foreign transaction resolver process.

Co-authored-by: Masahiko Sawada, Ashutosh Bapat
---
 src/backend/access/fdwxact/Makefile           |   5 +-
 src/backend/access/fdwxact/fdwxact.c          |  33 +-
 src/backend/access/fdwxact/launcher.c         | 567 ++++++++++++++++++
 src/backend/access/fdwxact/resolver.c         | 352 +++++++++++
 src/backend/access/transam/twophase.c         |  16 +
 src/backend/postmaster/bgworker.c             |   8 +
 src/backend/postmaster/pgstat.c               |   6 +
 src/backend/postmaster/postmaster.c           |  13 +-
 src/backend/storage/ipc/ipci.c                |   3 +
 src/backend/storage/lmgr/lwlocknames.txt      |   1 +
 src/backend/tcop/postgres.c                   |  14 +
 src/backend/utils/misc/guc.c                  |  37 ++
 src/backend/utils/misc/postgresql.conf.sample |  12 +
 src/include/access/fdwxact.h                  |   6 +
 src/include/access/fdwxact_launcher.h         |  28 +
 src/include/access/fdwxact_resolver.h         |  23 +
 src/include/access/resolver_internal.h        |  63 ++
 src/include/catalog/pg_proc.dat               |   5 +
 src/include/pgstat.h                          |   2 +
 src/include/utils/guc_tables.h                |   2 +
 20 files changed, 1183 insertions(+), 13 deletions(-)
 create mode 100644 src/backend/access/fdwxact/launcher.c
 create mode 100644 src/backend/access/fdwxact/resolver.c
 create mode 100644 src/include/access/fdwxact_launcher.h
 create mode 100644 src/include/access/fdwxact_resolver.h
 create mode 100644 src/include/access/resolver_internal.h

diff --git a/src/backend/access/fdwxact/Makefile b/src/backend/access/fdwxact/Makefile
index aacab1d729..151e3ae336 100644
--- a/src/backend/access/fdwxact/Makefile
+++ b/src/backend/access/fdwxact/Makefile
@@ -12,6 +12,9 @@ subdir = src/backend/access/fdwxact
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = fdwxact.o
+OBJS = \
+	fdwxact.o \
+	resolver.o \
+	launcher.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
index 7b3a2f1fba..b4cab71c3d 100644
--- a/src/backend/access/fdwxact/fdwxact.c
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -22,10 +22,10 @@
  * At PREPARE TRANSACTION, we prepare all transactions on foreign servers by executing
  * PrepareForeignTransaction() API regardless of data on the foreign server having been
  * modified.  At COMMIT PREPARED and ROLLBACK PREPARED, we commit or rollback only the
- * local transaction but not do anything for involved foreign transactions.  To resolve
- * these foreign transactions the user needs to use pg_resolve_foreign_xact() SQL
- * function that resolve a foreign transaction according to the result of the
- * corresponding local transaction.
+ * local transaction but not do anything for involved foreign transactions.  The preapred
+ * foreign transactions are resolved by a resolver process asynchronously.  Also, the
+ * user can use pg_resolve_foreign_xact() SQL function to resolve a foreign transaction
+ * manually.
  *
  * LOCKING
  *
@@ -76,7 +76,10 @@
 #include <unistd.h>
 
 #include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/twophase.h"
+#include "access/resolver_internal.h"
 #include "access/xact.h"
 #include "access/xlog.h"
 #include "access/xloginsert.h"
@@ -157,6 +160,7 @@ static bool fdwXactExitRegistered = false;
 
 /* Guc parameter */
 int			max_prepared_foreign_xacts = 0;
+int			max_foreign_xact_resolvers = 0;
 
 static void AtProcExit_FdwXact(int code, Datum arg);
 static void FdwXactPrepareForeignTransactions(TransactionId xid);
@@ -165,7 +169,6 @@ static void FdwXactParticipantEndTransaction(FdwXactParticipant *fdw_part,
 											 bool commit);
 static FdwXact FdwXactInsertFdwXactEntry(TransactionId xid,
 										 FdwXactParticipant *fdw_part);
-static void FdwXactResolveFdwXacts(int *fdwxact_idxs, int nfdwxacts);
 static void FdwXactComputeRequiredXmin(void);
 static FdwXactStatus FdwXactGetTransactionFate(TransactionId xid);
 static void FdwXactResolveOneFdwXact(FdwXact fdwxact);
@@ -772,12 +775,13 @@ ForgetAllFdwXactParticipants(void)
 
 	/*
 	 * If we leave any FdwXact entries, update the oldest local transaction of
-	 * unresolved distributed transaction.
+	 * unresolved distributed transaction and notify the launcher.
 	 */
 	if (nlefts > 0)
 	{
 		elog(DEBUG1, "left %u foreign transactions", nlefts);
 		FdwXactComputeRequiredXmin();
+		FdwXactLaunchOrWakeupResolver();
 	}
 
 	list_free_deep(FdwXactParticipants);
@@ -785,7 +789,9 @@ ForgetAllFdwXactParticipants(void)
 }
 
 /*
- * Commit or rollback all foreign transactions.
+ * Close in-progress involved foreign transactions.  We don't perform the second
+ * phase of two-phase commit protocol here.  All prepared foreign transactions
+ * enter in-doubt state and a resolver process will process them.
  */
 void
 AtEOXact_FdwXact(bool is_commit)
@@ -889,7 +895,7 @@ PrePrepare_FdwXact(void)
  * The caller must hold the given foreign transactions in advance to prevent
  * concurrent update.
  */
-static void
+void
 FdwXactResolveFdwXacts(int *fdwxact_idxs, int nfdwxacts)
 {
 	for (int i = 0; i < nfdwxacts; i++)
@@ -924,6 +930,17 @@ FdwXactExists(Oid dbid, Oid serverid, Oid userid)
 
 	return (idx >= 0);
 }
+bool
+FdwXactExistsXid(TransactionId xid)
+{
+	int			idx;
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	idx = get_fdwxact(InvalidOid, xid, InvalidOid, InvalidOid);
+	LWLockRelease(FdwXactLock);
+
+	return (idx >= 0);
+}
 
 /*
  * Return the index of first found FdwXact entry that matched to given arguments.
diff --git a/src/backend/access/fdwxact/launcher.c b/src/backend/access/fdwxact/launcher.c
new file mode 100644
index 0000000000..916b9af2f7
--- /dev/null
+++ b/src/backend/access/fdwxact/launcher.c
@@ -0,0 +1,567 @@
+/*-------------------------------------------------------------------------
+ *
+ * launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
+#include "access/resolver_internal.h"
+#include "access/twophase.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "nodes/pg_list.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+static volatile sig_atomic_t got_SIGUSR2 = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+/*
+ * Wake up the launcher process to request launching new resolvers
+ * immediately.
+ */
+void
+FdwXactLauncherRequestToLaunch(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR2);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, SizeOfFdwXactRslvCtlData);
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool		found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int			slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+		SHMQueueInit(&(FdwXactRslvCtl->fdwxact_queue));
+		FdwXactRslvCtl->launcher_pid = InvalidPid;
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			memset(resolver, 0, sizeof(FdwXactResolver));
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/* SIGUSR2: set flag to launch new resolver process immediately */
+static void
+fdwxact_launcher_sigusr2(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGUSR2 = true;
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == InvalidPid);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+	FdwXactRslvCtl->launcher_latch = &MyProc->procLatch;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGUSR2, fdwxact_launcher_sigusr2);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz now;
+		long		wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int			rc;
+
+		CHECK_FOR_INTERRUPTS();
+		ResetLatch(MyLatch);
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Limit the start retry to once a
+		 * foreign_xact_resolution_retry_interval but always attempt to
+		 * start when requested.
+		 */
+		if (got_SIGUSR2 ||
+			TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			MemoryContext oldctx;
+			MemoryContext subctx;
+			bool		launched;
+
+			if (got_SIGUSR2)
+				got_SIGUSR2 = false;
+
+			subctx = AllocSetContextCreate(TopMemoryContext,
+										   "Foreign Transaction Launcher",
+										   ALLOCSET_DEFAULT_SIZES);
+			oldctx = MemoryContextSwitchTo(subctx);
+
+			/*
+			 * Launch foreign transaction resolvers that are requested but not
+			 * running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+			{
+				last_start_time = now;
+				wait_time = foreign_xact_resolution_retry_interval;
+			}
+
+			/* Switch back to original memory context. */
+			MemoryContextSwitchTo(oldctx);
+			/* Clean the temporary memory. */
+			MemoryContextDelete(subctx);
+		}
+		else
+		{
+			/*
+			 * The wait in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we should
+			 * retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDWXACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver process
+ * or wake up the resolver if it's already running.
+ */
+void
+FdwXactLaunchOrWakeupResolver(void)
+{
+	volatile FdwXactResolver *resolver;
+	bool		found = false;
+
+	/*
+	 * Looking for a resolver process that is running and working on the same
+	 * database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (int i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	if (found)
+	{
+		/* Found the running resolver */
+		elog(DEBUG1,
+			 "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		/*
+		 * Wakeup the resolver. It's possible that the resolver is starting up
+		 * and doesn't attach its slot yet. Since the resolver will find
+		 * FdwXact entry we inserted soon we don't anything.
+		 */
+		if (resolver->latch)
+			SetLatch(resolver->latch);
+
+		return;
+	}
+
+	/* Otherwise wake up the launcher to launch new resolver */
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid'.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int			unused_slot = -1;
+	int			i;
+
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	/* Find unused resolver slot */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			unused_slot = i;
+			break;
+		}
+	}
+
+	/* No unused found */
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign transaction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[unused_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_notify_pid = MyProcPid;
+	bgw.bgw_main_arg = Int32GetDatum(unused_slot);
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to
+	 * wait until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch or relaunch foreign transaction resolvers on database that has
+ * at least one FdwXact entry but no resolver is running on it.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	HTAB	   *fdwxact_dbs;
+	HTAB	   *resolver_dbs;
+	HASHCTL		ctl;
+	HASH_SEQ_STATUS status;
+	Oid		   *entry;
+	bool		launched;
+
+	memset(&ctl, 0, sizeof(ctl));
+	ctl.keysize = sizeof(Oid);
+	ctl.entrysize = sizeof(Oid);
+
+	/*
+	 * Create a hash map for the database that has at least one foreign
+	 * transaction to resolve.
+	 */
+	fdwxact_dbs = hash_create("fdwxact dblist",
+							  32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+	/* Collect database oids that has at least one FdwXact entry to resolve */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		/*
+		 * We need to launch resolver process if the foreign transaction
+		 * is not held by anyone and is not a part of the local prepared
+		 * transaction.
+		 */
+		if (fdwxact->locking_backend == InvalidBackendId &&
+			!TwoPhaseExists(fdwxact->local_xid))
+			hash_search(fdwxact_dbs, &(fdwxact->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* There is no foreign transaction to resolve, no need to launch new one */
+	if (hash_get_num_entries(fdwxact_dbs) == 0)
+	{
+		hash_destroy(fdwxact_dbs);
+		return false;
+	}
+
+	/* Create a hash map for databases on which a resolver is running */
+	resolver_dbs = hash_create("resolver dblist",
+							   32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+	/* Collect database oids on which resolvers are running */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (int i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+			continue;
+
+		hash_search(resolver_dbs, &(resolver->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/*
+	 * Find databases on which no resolver is running and launch new
+	 * resolver process on them.
+	 */
+	hash_seq_init(&status, fdwxact_dbs);
+	while ((entry = (Oid *) hash_seq_search(&status)) != NULL)
+	{
+		bool		found;
+
+		hash_search(resolver_dbs, entry, HASH_FIND, &found);
+
+		if (!found)
+		{
+			/* No resolver is running on this database, launch new one */
+			fdwxact_launch_resolver(*entry);
+			launched = true;
+		}
+	}
+
+	hash_destroy(fdwxact_dbs);
+	hash_destroy(resolver_dbs);
+
+	return launched;
+}
+
+/* Register a background worker running the foreign transaction launcher */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+/*
+ * Stop the fdwxact resolver running on the given database.
+ */
+Datum
+pg_stop_foreign_xact_resolver(PG_FUNCTION_ARGS)
+{
+	Oid			dbid = PG_GETARG_OID(0);
+	FdwXactResolver *resolver = NULL;
+	int			i;
+
+	/* Must be super user */
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 errmsg("permission denied to stop foreign transaction resolver")));
+
+	if (!OidIsValid(dbid))
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("invalid database id")));
+
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+
+	/* Find the running resolver process on the given database */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* found! */
+		if (resolver->in_use && resolver->dbid == dbid)
+			break;
+	}
+
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errmsg("there is no running foreign transaction resolver process on database %d",
+						dbid)));
+
+	/* Found the resolver, terminate it ... */
+	kill(resolver->pid, SIGTERM);
+
+	/* ... and wait for it to die */
+	for (;;)
+	{
+		int			rc;
+
+		/* is it gone? */
+		if (!resolver->in_use)
+			break;
+
+		LWLockRelease(FdwXactResolverLock);
+
+		/* Wait a bit --- we don't expect to have to wait long. */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
+					   10L, WAIT_EVENT_BGWORKER_SHUTDOWN);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	}
+
+	LWLockRelease(FdwXactResolverLock);
+
+	PG_RETURN_BOOL(true);
+}
diff --git a/src/backend/access/fdwxact/resolver.c b/src/backend/access/fdwxact/resolver.c
new file mode 100644
index 0000000000..c9d41428fc
--- /dev/null
+++ b/src/backend/access/fdwxact/resolver.c
@@ -0,0 +1,352 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for each databases.
+ *
+ * A resolver process continues to resolve foreign transactions on the
+ * database, which the backend process is waiting for resolution.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/resolver_internal.h"
+#include "access/transam.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+/* GUC parameters */
+int	foreign_xact_resolution_retry_interval;
+int	foreign_xact_resolver_timeout = 60 * 1000;
+
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FXRslvLoop(void);
+static long FXRslvComputeSleepTime(TimestampTz now, TimestampTz targetTime);
+static void FXRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+static void hold_indoubt_fdwxacts(void);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+static TimestampTz last_resolution_time = -1;
+
+/*
+ * held_fdwxacts has indexes of FdwXact which the resolver marked
+ * as in-processing. These mark is cleared on process exit.
+ */
+static int *held_fdwxacts = NULL;
+static int	nheld;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+
+	/* Release the held foreign transaction entries */
+	for (int i = 0; i < nheld; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[held_fdwxacts[i]];
+
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->locking_backend = InvalidBackendId;
+		LWLockRelease(FdwXactLock);
+	}
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	Assert(OidIsValid(MyFdwXactResolver->dbid));
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int			slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+	CommitTransactionCommand();
+
+	held_fdwxacts = palloc(sizeof(int) * max_prepared_foreign_xacts);
+	nheld = 0;
+
+	/* Initialize stats to a sanish value */
+	last_resolution_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FXRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FXRslvLoop(void)
+{
+	MemoryContext resolver_ctx;
+
+	resolver_ctx = AllocSetContextCreate(TopMemoryContext,
+										 "Foreign Transaction Resolver",
+										 ALLOCSET_DEFAULT_SIZES);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz resolutionTs = -1;
+		TimestampTz now;
+		int			rc;
+		long		sleep_time = DEFAULT_NAPTIME_PER_CYCLE;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		MemoryContextSwitchTo(resolver_ctx);
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		now = GetCurrentTimestamp();
+
+		/* Hold in-doubt foreign transaction to resolve */
+		hold_indoubt_fdwxacts();
+
+		if (nheld > 0)
+		{
+			/* Resolve in-doubt transactions */
+			StartTransactionCommand();
+			FdwXactResolveFdwXacts(held_fdwxacts, nheld);
+			CommitTransactionCommand();
+			last_resolution_time = now;
+		}
+
+		FXRslvCheckTimeout(now);
+
+		sleep_time = FXRslvComputeSleepTime(now, resolutionTs);
+
+		MemoryContextResetAndDeleteChildren(resolver_ctx);
+		MemoryContextSwitchTo(TopMemoryContext);
+
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   sleep_time,
+					   WAIT_EVENT_FDWXACT_RESOLVER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FXRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	timeout = TimestampTzPlusMilliseconds(last_resolution_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	/* Reached timeout, exit */
+	StartTransactionCommand();
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+					get_database_name(MyDatabaseId))));
+	CommitTransactionCommand();
+	fdwxact_resolver_detach();
+	proc_exit(0);
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. We can sleep until the time
+ * out or the next resolution time given by nextResolutionTs.
+ */
+static long
+FXRslvComputeSleepTime(TimestampTz now, TimestampTz nextResolutionTs)
+{
+	long		sleeptime = DEFAULT_NAPTIME_PER_CYCLE;
+
+	if (foreign_xact_resolver_timeout > 0)
+	{
+		TimestampTz timeout;
+		long		sec_to_timeout;
+		int			microsec_to_timeout;
+
+		/* Compute relative time until wakeup. */
+		timeout = TimestampTzPlusMilliseconds(last_resolution_time,
+											  foreign_xact_resolver_timeout);
+		TimestampDifference(now, timeout,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	if (nextResolutionTs > 0)
+	{
+		long		sec_to_timeout;
+		int			microsec_to_timeout;
+
+		TimestampDifference(now, nextResolutionTs,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
+
+/*
+ * Lock foreign transactions that are not held by anyone.
+ */
+static void
+hold_indoubt_fdwxacts(void)
+{
+	nheld = 0;
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->valid &&
+			fdwxact->locking_backend == InvalidBackendId &&
+			!TwoPhaseExists(fdwxact->local_xid))
+		{
+			held_fdwxacts[nheld++] = i;
+			fdwxact->locking_backend = MyBackendId;
+		}
+	}
+	LWLockRelease(FdwXactLock);
+}
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 265b03ba5a..29f11fb779 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -77,6 +77,8 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/htup_details.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
@@ -2286,6 +2288,13 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * If the prepared transaciton was a part of a distributed transaction
+	 * notify a resolver process to handle it.
+	 */
+	if (FdwXactExistsXid(xid))
+		FdwXactLaunchOrWakeupResolver();
 }
 
 /*
@@ -2345,6 +2354,13 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * If the prepared transaciton was a part of a distributed transaction
+	 * notify a resolver process to handle it.
+	 */
+	if (FdwXactExistsXid(xid))
+		FdwXactLaunchOrWakeupResolver();
 }
 
 /*
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index d209b69ec0..ee87e4a847 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -13,6 +13,8 @@
 #include "postgres.h"
 
 #include "access/parallel.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "libpq/pqsignal.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -128,6 +130,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index 4e53d40b13..db51f458b0 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3830,6 +3830,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDWXACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDWXACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_APPLY_MAIN:
 			event_name = "LogicalApplyMain";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 4fdf015c9b..a837cb8260 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -94,6 +94,7 @@
 #endif
 
 #include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/transam.h"
 #include "access/xlog.h"
 #include "bootstrap/bootstrap.h"
@@ -911,6 +912,9 @@ PostmasterMain(int argc, char *argv[])
 	if (max_wal_senders > 0 && wal_level == WAL_LEVEL_MINIMAL)
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers <= 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0")));
 
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
@@ -976,12 +980,13 @@ PostmasterMain(int argc, char *argv[])
 	LocalProcessControlFile(false);
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 2d7191d3cd..271fd35884 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -17,6 +17,7 @@
 #include "access/clog.h"
 #include "access/commit_ts.h"
 #include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/heapam.h"
 #include "access/multixact.h"
 #include "access/nbtree.h"
@@ -151,6 +152,7 @@ CreateSharedMemoryAndSemaphores(void)
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
 		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -270,6 +272,7 @@ CreateSharedMemoryAndSemaphores(void)
 	SyncScanShmemInit();
 	AsyncShmemInit();
 	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index dc29a7ea6f..9327394013 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -54,3 +54,4 @@ XactTruncationLock					44
 WrapLimitsVacuumLock				46
 NotifyQueueTailLock					47
 FdwXactLock							48
+FdwXactResolverLock					49
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 317d1aa573..7ac1488e33 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -36,6 +36,8 @@
 #include "rusagestub.h"
 #endif
 
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
 #include "access/parallel.h"
 #include "access/printtup.h"
 #include "access/xact.h"
@@ -3085,6 +3087,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 7ef7eef1b5..33e1b5884c 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -760,6 +760,10 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FOREIGN_TRANSACTION */
+	gettext_noop("Foreign Transaction"),
+	/* FOREIGN_TRANSACTION_RESOLVER */
+	gettext_noop("Foreign Transaction / Resolver"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -2459,6 +2463,39 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, FOREIGN_TRANSACTION_RESOLVER,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, FOREIGN_TRANSACTION_RESOLVER,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 13e7027fd4..36abadbc60 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -732,6 +732,18 @@
 #max_pred_locks_per_page = 2            # min 0
 
 
+#------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+
 #------------------------------------------------------------------------------
 # VERSION AND PLATFORM COMPATIBILITY
 #------------------------------------------------------------------------------
diff --git a/src/include/access/fdwxact.h b/src/include/access/fdwxact.h
index 9ba819e9d1..a3763e52c0 100644
--- a/src/include/access/fdwxact.h
+++ b/src/include/access/fdwxact.h
@@ -104,13 +104,19 @@ typedef struct FdwXactRslvState
 
 /* GUC parameters */
 extern int	max_prepared_foreign_xacts;
+extern int	max_foreign_xact_resolvers;
+extern int	foreign_xact_resolution_retry_interval;
+extern int	foreign_xact_resolver_timeout;
 
 /* Function declarations */
 extern Size FdwXactShmemSize(void);
 extern void FdwXactShmemInit(void);
 extern void AtEOXact_FdwXact(bool is_commit);
 extern void PrePrepare_FdwXact(void);
+extern bool FdwXactIsForeignTwophaseCommitRequired(void);
+extern void FdwXactResolveFdwXacts(int *fdwxact_idxs, int nfdwxacts);
 extern bool FdwXactExists(Oid dbid, Oid serverid, Oid userid);
+extern bool FdwXactExistsXid(TransactionId xid);
 extern void CheckPointFdwXacts(XLogRecPtr redo_horizon);
 extern void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
 								Oid userid, void *content, int len);
diff --git a/src/include/access/fdwxact_launcher.h b/src/include/access/fdwxact_launcher.h
new file mode 100644
index 0000000000..688b43b8d0
--- /dev/null
+++ b/src/include/access/fdwxact_launcher.h
@@ -0,0 +1,28 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.h
+ *	  PostgreSQL foreign transaction launcher definitions
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_launcher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FDWXACT_LAUNCHER_H
+#define FDWXACT_LAUNCHER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactLauncherRegister(void);
+extern void FdwXactLauncherMain(Datum main_arg);
+extern void FdwXactLauncherRequestToLaunch(void);
+extern void FdwXactLaunchOrWakeupResolver(void);
+extern Size FdwXactRslvShmemSize(void);
+extern void FdwXactRslvShmemInit(void);
+extern bool IsFdwXactLauncher(void);
+
+
+#endif							/* FDWXACT_LAUNCHER_H */
diff --git a/src/include/access/fdwxact_resolver.h b/src/include/access/fdwxact_resolver.h
new file mode 100644
index 0000000000..779848113c
--- /dev/null
+++ b/src/include/access/fdwxact_resolver.h
@@ -0,0 +1,23 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.h
+ *	  PostgreSQL foreign transaction resolver definitions
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_resolver.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_RESOLVER_H
+#define FDWXACT_RESOLVER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactResolverMain(Datum main_arg);
+extern bool IsFdwXactResolver(void);
+
+extern int	foreign_xact_resolver_timeout;
+
+#endif							/* FDWXACT_RESOLVER_H */
diff --git a/src/include/access/resolver_internal.h b/src/include/access/resolver_internal.h
new file mode 100644
index 0000000000..c935471936
--- /dev/null
+++ b/src/include/access/resolver_internal.h
@@ -0,0 +1,63 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver_internal.h
+ *	  Internal headers shared by fdwxact resolvers.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/resolver_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef RESOLVER_INTERNAL_H
+#define RESOLVER_INTERNAL_H
+
+#include "storage/latch.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/timestamp.h"
+
+/*
+ * Each foreign transaction resolver has a FdwXactResolver struct in
+ * shared memory.  This struct is protected by FdwXactResolverLaunchLock.
+ */
+typedef struct FdwXactResolver
+{
+	pid_t		pid;			/* this resolver's PID, or 0 if not active */
+	Oid			dbid;			/* database oid */
+
+	/* Indicates if this slot is used of free */
+	bool		in_use;
+
+	/* Protect shared variables shown above */
+	slock_t		mutex;
+
+	/*
+	 * Pointer to the resolver's patch. Used by backends to wake up this
+	 * resolver when it has work to do. NULL if the resolver isn't active.
+	 */
+	Latch	   *latch;
+} FdwXactResolver;
+
+/* There is one FdwXactRslvCtlData struct for the whole database cluster */
+typedef struct FdwXactRslvCtlData
+{
+	/* Foreign transaction resolution queue. Protected by FdwXactLock */
+	SHM_QUEUE	fdwxact_queue;
+
+	/* Supervisor process and latch */
+	pid_t		launcher_pid;
+	Latch	   *launcher_latch;
+
+	FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER];
+} FdwXactRslvCtlData;
+#define SizeOfFdwXactRslvCtlData \
+	(offsetof(FdwXactRslvCtlData, resolvers) + sizeof(FdwXactResolver))
+
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+extern FdwXactResolver *MyFdwXactResolver;
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+#endif							/* RESOLVER_INTERNAL_H */
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 71ebb034b2..091176ad72 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -6165,6 +6165,11 @@
   proname => 'pg_walfile_name', prorettype => 'text', proargtypes => 'pg_lsn',
   prosrc => 'pg_walfile_name' },
 
+{ oid => '9709',
+  descr => 'stop a foreign transaction resolver process running on the given database',
+  proname => 'pg_stop_foreign_xact_resolver', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'oid', prosrc => 'pg_stop_foreign_xact_resolver'},
+
 { oid => '3165', descr => 'difference in bytes, given two wal locations',
   proname => 'pg_wal_lsn_diff', prorettype => 'numeric',
   proargtypes => 'pg_lsn pg_lsn', prosrc => 'pg_wal_lsn_diff' },
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index 1ed1987fa5..382913a790 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -883,6 +883,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDWXACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDWXACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index 7f36e1146f..cf2170cf5f 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -96,6 +96,8 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FOREIGN_TRANSACTION,
+	FOREIGN_TRANSACTION_RESOLVER,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
-- 
2.27.0

