By using db<>fiddle, you agree to license everything you submit by Creative Commons CC0.
CREATE TABLE tablename (
a INT NOT NULL,
b INT NOT NULL,
x INT GENERATED ALWAYS AS (LEAST(a, b)),
y INT GENERATED ALWAYS AS (GREATEST(a, b)),
UNIQUE (x, y)
);
INSERT INTO tablename (`a`, `b`) VALUES ('1', '2');
INSERT INTO tablename (`a`, `b`) VALUES ('2', '1');
Duplicate entry '1-2' for key 'tablename.x'