Postgres 13

By using db<>fiddle, you agree to license everything you submit by Creative Commons CC0.
CREATE TABLE tbl (
  a text
, b text
);

ALTER TABLE tbl ADD CONSTRAINT a_not_equal_b CHECK (a <> b);
ALTER TABLE tbl ADD CONSTRAINT ab_unique
   EXCLUDE USING gist ((ARRAY[hashtext(COALESCE(a, ''))
                            , hashtext(COALESCE(b, ''))]) gist__int_ops WITH &&);
INSERT INTO tbl(a,b) VALUES ('x', 'y');
INSERT INTO tbl(a,b) VALUES ('xx', 'yy');
INSERT INTO tbl(a,b) VALUES ('a', 'b'), ('A', 'B');
1 rows affected

1 rows affected

2 rows affected

-- NULL values are allowed
INSERT INTO tbl(a,b) VALUES (NULL, NULL);
1 rows affected

TABLE tbl;


a
b


x
y

xx
yy

a
b

A
B

null
null


-- NULL collides with ''
INSERT INTO tbl(a,b) VALUES ('', 'foo');
ERROR:  conflicting key value violates exclusion constraint "ab_unique"
DETAIL:  Key ((ARRAY[hashtext(COALESCE(a, ''::text)), hashtext(COALESCE(b, ''::text))]))=({-1477818771,1955559433}) conflicts with existing key ((ARRAY[hashtext(COALESCE(a, ''::text)), hashtext(COALESCE(b, ''::text))]))=({-1477818771,-1477818771}).


-- not allowed
INSERT INTO tbl(a,b) VALUES ('x', 'y');
ERROR:  conflicting key value violates exclusion constraint "ab_unique"
DETAIL:  Key ((ARRAY[hashtext(COALESCE(a, ''::text)), hashtext(COALESCE(b, ''::text))]))=({1074944137,32626184}) conflicts with existing key ((ARRAY[hashtext(COALESCE(a, ''::text)), hashtext(COALESCE(b, ''::text))]))=({1074944137,32626184}).


-- not allowed
INSERT INTO tbl(a,b) VALUES ('y', NULL)
ERROR:  conflicting key value violates exclusion constraint "ab_unique"
DETAIL:  Key ((ARRAY[hashtext(COALESCE(a, ''::text)), hashtext(COALESCE(b, ''::text))]))=({32626184,-1477818771}) conflicts with existing key ((ARRAY[hashtext(COALESCE(a, ''::text)), hashtext(COALESCE(b, ''::text))]))=({1074944137,32626184}).


-- not allowed
INSERT INTO tbl(a,b) VALUES (NULL, 'x')
ERROR:  conflicting key value violates exclusion constraint "ab_unique"
DETAIL:  Key ((ARRAY[hashtext(COALESCE(a, ''::text)), hashtext(COALESCE(b, ''::text))]))=({-1477818771,1074944137}) conflicts with existing key ((ARRAY[hashtext(COALESCE(a, ''::text)), hashtext(COALESCE(b, ''::text))]))=({1074944137,32626184}).


-- not allowed
INSERT INTO tbl(a,b) VALUES ('z', 'z');
ERROR:  new row for relation "tbl" violates check constraint "a_not_equal_b"
DETAIL:  Failing row contains (z, z).
a	b
x	y
xx	yy
a	b
A	B
null	null