SQLite 3.16

By using db<>fiddle, you agree to license everything you submit by Creative Commons CC0.

WITH entry AS
(
  SELECT 'this is help { some help }' input
  UNION
  SELECT 'help'
  UNION
  SELECT 'another help [ help ]'
  UNION  
  SELECT 'some text ( some words surrounding help . just for example )'
  UNION
  SELECT '[ see help ] some text'
  UNION
  SELECT 'this is a sentence. { help }' 
  UNION
  SELECT 'Asking for [help]. help me'
)
SELECT input
FROM
(
    SELECT input,
           CASE WHEN InStr(input, '[') > 0 THEN
                     InStr(input, '[')
                ELSE
                (CASE WHEN InStr(input, '(') > 0 THEN
                           InStr(input, '(')
                      ELSE InStr(input, '{')
                 END)
           END Idx,
           CASE WHEN InStr(input, ']') > 0 THEN
                     InStr(input, ']')
                ELSE
                (CASE WHEN InStr(input, ')') > 0 THEN
                           InStr(input, ')')
                      ELSE InStr(input, '}')
                 END)
           END Ix,
           InStr(input, 'help') IdxH
   FROM entry
)
WHERE (Idx = 0 OR Idx > IdxH OR InStr(SubStr(input, Ix), 'help') > 0)

input
Asking for [help]. help me
another help [ help ]
help
this is help { some help }