SQL Server 2017

By using db<>fiddle, you agree to license everything you submit by Creative Commons CC0.
CREATE TABLE [dbo].[BasketProduct](
    [BasketProductId] [int] IDENTITY(1,1) NOT NULL,
    [BasketId] [int] NOT NULL,
    [ProductId] [int] NOT NULL,
    [Quantity] [int] NOT NULL,
 CONSTRAINT [PK_BasketProduct] PRIMARY KEY CLUSTERED 
([BasketProductId] ASC));
CREATE PROC AddBasketProduct @BasketID int, @ProductID int, @Quantity int AS
BEGIN

    DECLARE @DistinctProducts int;

    SELECT @DistinctProducts = COUNT(DISTINCT ProductID)
    FROM dbo.BasketProduct WITH (UPDLOCK)
    WHERE BasketId = @BasketID
      AND ProductID != @ProductID ;

    IF @DistinctProducts >= 10
        THROW 71245, N'Cannot have more than 10 different products in a single basket.',16; --Choose an error number and state appropraite for your applciation
    ELSE
        INSERT INTO dbo.BasketProduct (BasketId,
                                       ProductId,
                                       Quantity)
        VALUES(@BasketID,@ProductID,@Quantity);
END;
--Make some sample data
INSERT INTO dbo.BasketProduct (BasketId,
                               ProductId,
                               Quantity)
VALUES(1,1,1),
      (1,2,1),
      (1,3,1),
      (1,4,1),
      (1,5,1),
      (1,6,1),
      (1,7,1),
      (1,8,1),
      (1,9,1),
      (1,10,1); --10 products. Will work
10 rows affected

--11th product, will fail
EXEC dbo.AddBasketProduct @BasketID = 1,
                          @ProductID = 11,
                          @Quantity = 1;
Msg 71245 Level 16 State 16 Line 12
Cannot have more than 10 different products in a single basket.

--Repetition of product 2, will work
EXEC dbo.AddBasketProduct @BasketID = 1,
                          @ProductID = 2,
                          @Quantity = 1;
1 rows affected