MySQL 8.0

By using db<>fiddle, you agree to license everything you submit by Creative Commons CC0.

CREATE TABLE IF NOT EXISTS `purchases` (
  `category` varchar(50) NOT NULL,
  `subcategory` varchar(50) NOT NULL,
  `vendor` varchar(120) NOT NULL,
  `amount` decimal(18,2) DEFAULT NULL
) DEFAULT CHARSET=utf8;
INSERT INTO `purchases` (`category`, `subcategory`, `vendor`, `amount`) VALUES
  ('Paper', 'Napkins','Brawny', 20.01),
  ('Paper', 'Napkins','Paper Tiger', 20.43),
  ('Paper', 'Napkins','Paper LLC', 20.13),
  ('Paper', 'Napkins','Dunder Mifflin', 20.33),
  ('Paper', 'Napkins','Brawny', 40.01),
  ('Paper', 'Napkins','Paper Tiger', 30.43),
  ('Paper', 'Napkins','Paper LLC', 40.13),
  ('Paper', 'Napkins','Paper LLC', 50.23),
  ('Paper', 'Napkins','Dunder Mifflin', 20.33),
  ('Paper', 'Napkins','Dunder Mifflin', 30.33),
  ('Paper', 'Napkins','Staples', 22.63),
  ('Paper', 'Towels','Acme LLC', 52.01),
  ('Paper', 'Towels','Towel Direct', 40.43),
  ('Paper', 'Towels','Michigan Paper', 41.23),
  ('Paper', 'Towels','Cascades LLC', 40.55),
  ('Paper', 'Towels','Brawny', 26.33),
  ('Food', 'Apples','Orchards LLC', 52.31),
  ('Food', 'Apples','Orchards LLC', 32.35),
  ('Food', 'Apples','Orchards LLC', 12.43),
  ('Food', 'Apples','Apples LLC', 40.43),
  ('Food', 'Apples','Northstar Orchard', 41.23),
  ('Food', 'Pizza','Pizza Hut', 10.25),
  ('Food', 'Pizza','Dominos', 16.53);

Records: 23  Duplicates: 0  Warnings: 0

WITH purchases_cat_sum AS
(
  SELECT *,
         SUM(amount) OVER (PARTITION BY category) category_total_amount,
         SUM(amount) OVER (PARTITION BY category, subcategory) subcategory_total_amount
  FROM purchases
)
SELECT category, category_total_amount,
       subcategory, subcategory_total_amount,
       CASE grp WHEN 'Others'
         THEN CONCAT(COUNT(*),' ', grp)
         ELSE grp
       END AS vendor,
       SUM(sum_amount) AS total_amount
FROM
(
  SELECT category, category_total_amount,
         subcategory, subcategory_total_amount,
         vendor, SUM(amount) sum_amount,
       CASE 
         WHEN ROW_NUMBER() OVER (PARTITION BY category, subcategory ORDER BY SUM(amount) DESC) <=3
         THEN vendor
         ELSE 'Others'
       END AS grp
  FROM purchases_cat_sum
  GROUP BY category,category_total_amount, 
           subcategory, subcategory_total_amount, 
           vendor
) T
GROUP BY category,category_total_amount, 
         subcategory, subcategory_total_amount, 
         grp
ORDER BY category DESC, subcategory, 
         CASE WHEN grp<>'Others'THEN 1 ELSE 2 END, 
         SUM(sum_amount) DESC

category	category_total_amount	subcategory	subcategory_total_amount	vendor	total_amount
Paper	515.54	Napkins	314.99	Paper LLC	110.49
Paper	515.54	Napkins	314.99	Dunder Mifflin	70.99
Paper	515.54	Napkins	314.99	Brawny	60.02
Paper	515.54	Napkins	314.99	2 Others	73.49
Paper	515.54	Towels	200.55	Acme LLC	52.01
Paper	515.54	Towels	200.55	Michigan Paper	41.23
Paper	515.54	Towels	200.55	Cascades LLC	40.55
Paper	515.54	Towels	200.55	2 Others	66.76
Food	205.53	Apples	178.75	Orchards LLC	97.09
Food	205.53	Apples	178.75	Northstar Orchard	41.23
Food	205.53	Apples	178.75	Apples LLC	40.43
Food	205.53	Pizza	26.78	Dominos	16.53
Food	205.53	Pizza	26.78	Pizza Hut	10.25