SQL Server 2019

By using db<>fiddle, you agree to license everything you submit by Creative Commons CC0.
create table relations (id1 int, id2 int)
alter table relations add  least_id1_id2 as
    (case when id1 < id2 then id1 else id2 end) persisted;

alter table relations add greatest_id1_id2 as
    (case when id1 < id2 then id2 else id1 end) persisted;


create unique index unq_relations_id1_id2 on relations(least_id1_id2, greatest_id1_id2);
insert into relations (id1, id2) values  (1, 2)
1 rows affected

-- error expected
insert into relations (id1, id2) values  (1, 2)
Msg 2601 Level 14 State 1 Line 2
Cannot insert duplicate key row in object 'dbo.relations' with unique index 'unq_relations_id1_id2'. The duplicate key value is (1, 2).
Msg 3621 Level 0 State 0 Line 2
The statement has been terminated.

-- error expected
insert into relations (id1, id2) values (2, 1)
Msg 2601 Level 14 State 1 Line 2
Cannot insert duplicate key row in object 'dbo.relations' with unique index 'unq_relations_id1_id2'. The duplicate key value is (1, 2).
Msg 3621 Level 0 State 0 Line 2
The statement has been terminated.

select * from relations


id1
id2
least_id1_id2
greatest_id1_id2


1
2
1
2