MariaDB 10.2

By using db<>fiddle, you agree to license everything you submit by Creative Commons CC0.
create table orderbuffertest (type_id int, price decimal (5, 2), is_buy_order bit)
insert into orderbuffertest values
(1, 2.3, 1),
(1, 3.3, 1),
(1, 4.3, 1),
(1, 5.3, 1),
(1, 6.3, 1),
(1, 7.3, 1),
(1, 8.3, 1),
(1, 9.3, 1),
(1, 12.4, 1),
(1, 12.3, 1),
(1, 22.3, 1),
(1, 32.3, 1),
(1, 42.3, 1),
(1, 52.3, 1),
(1, 62.3, 1),
(1, 72.3, 1),
(1, 82.3, 1),
(1, 92.3, 1),
(1, 2.3, 1),
(1, 3.3, 1),
(1, 4.3, 1),
(1, 5.3, 1),
(1, 6.3, 1),
(1, 7.3, 1),
(1, 8.3, 1),
(1, 9.3, 1),
(1, 22.3, 0),
(1, 32.3, 0),
(1, 42.3, 0),
(1, 52.3, 0),
(1, 62.3, 0),
(1, 72.3, 0),
(1, 92.3, 0),
(1, 2.3, 0),
(1, 3.3, 0),
(1, 4.3, 0),
(1, 5.3, 0),
(1, 6.3, 0),
(1, 7.3, 0),
(1, 8.3, 0),
(1, 9.3, 0),
(1, 12.4, 0),
(1, 22.4, 0),
(1, 32.4, 0),
(1, 42.4, 0),
(1, 52.4, 0),
(1, 62.4, 0),
(1, 82.3, 0),
(1, 72.4, 0),
(1, 82.4, 0),
(1, 92.4, 0)
WITH prices AS (
  SELECT type_id, price, is_buy_order,
         ROW_NUMBER() OVER (PARTITION BY type_id, is_buy_order ORDER BY price DESC) AS rownr,
         COUNT(*) OVER (PARTITION BY type_id, is_buy_order) AS num_rows
  FROM   orderbuffertest
)
SELECT type_id,
       MAX(CASE WHEN is_buy_order = 1 THEN price END) AS buy,
       COALESCE(MIN(CASE WHEN is_buy_order = 1 AND 100.0 * (rownr - 1) / num_rows <= 5 AND rownr != 1 THEN price END), 
                MAX(CASE WHEN is_buy_order = 1 AND rownr = 2 THEN price END)) AS `95th%buy`,
       MIN(CASE WHEN is_buy_order = 0 THEN price END) AS sell,
       COALESCE(MAX(CASE WHEN is_buy_order = 0 AND 100.0 * rownr / num_rows >= 95 AND rownr != num_rows THEN price END), 
                MAX(CASE WHEN is_buy_order = 0 AND rownr = num_rows - 1 THEN price END)) AS `95th%sell`
FROM prices
GROUP BY type_id


type_id
buy
95th%buy
sell
95th%sell


1
92.30
82.30
2.30
3.30


SELECT type_id,
       MAX(CASE WHEN is_buy_order = 1 THEN price END) AS buy,
       COALESCE(MIN(CASE WHEN is_buy_order = 1 AND 100.0 * (rownr - 1) / num_rows <= 5 AND rownr != 1 THEN price END), 
                MAX(CASE WHEN is_buy_order = 1 AND rownr = 2 THEN price END)) AS `95th%buy`,
       MIN(CASE WHEN is_buy_order = 0 THEN price END) AS sell,
       COALESCE(MAX(CASE WHEN is_buy_order = 0 AND 100.0 * rownr / num_rows >= 95 AND rownr != num_rows THEN price END), 
                MAX(CASE WHEN is_buy_order = 0 AND rownr = num_rows - 1 THEN price END)) AS `95th%sell`
FROM (
  SELECT type_id, price, is_buy_order,
         ROW_NUMBER() OVER (PARTITION BY type_id, is_buy_order ORDER BY price DESC) AS rownr,
         COUNT(*) OVER (PARTITION BY type_id, is_buy_order) AS num_rows
  FROM   orderbuffertest
) prices
GROUP BY type_id


type_id
buy
95th%buy
sell
95th%sell


1
92.30
82.30
2.30
3.30