SQL Server 2017

By using db<>fiddle, you agree to license everything you submit by Creative Commons CC0.
    create table #SiteInfo (ID int, SiteUserID INT, BrokerID int )
      insert into #SiteInfo values (
      5009,    19437,   765),
      (5008,    19437,   891),
      (5419,    26846,   891),
      (5421,    26846,   765),
      (5117,    40348,   765),
       (5914,    40389,   765),
       (5915,    40574,   891)
7 rows affected

   DECLARE @SiteUserId INT
         SET @SiteUserId = '40574';
         
SELECT DISTINCT SiteUserID  
FROM #SiteInfo

EXCEPT 

SELECT t1.SiteUserID
FROM #SiteInfo t1
LEFT JOIN #SiteInfo t2 ON  t1.BrokerID = t2.BrokerID
                            AND t2.SiteUserID = @SiteUserID
WHERE t2.SiteUserID IS NULL


SiteUserID


40574


   DECLARE @SiteUserId INT
         SET @SiteUserId = '40389';
         
SELECT DISTINCT SiteUserID  
FROM #SiteInfo

EXCEPT 

SELECT t1.SiteUserID
FROM #SiteInfo t1
LEFT JOIN #SiteInfo t2 ON  t1.BrokerID = t2.BrokerID
                            AND t2.SiteUserID = @SiteUserID
WHERE t2.SiteUserID IS NULL


SiteUserID


40348

40389


   DECLARE @SiteUserId INT
         SET @SiteUserId = '19437';
         
SELECT DISTINCT SiteUserID  
FROM #SiteInfo

EXCEPT 

SELECT t1.SiteUserID
FROM #SiteInfo t1
LEFT JOIN #SiteInfo t2 ON  t1.BrokerID = t2.BrokerID
                            AND t2.SiteUserID = @SiteUserID
WHERE t2.SiteUserID IS NULL


SiteUserID


19437

26846

40348

40389

40574