\documentclass{article}
\usepackage{amsmath}
\makeatletter
\def\resetMathstrut@{%
  \setbox\z@\hbox{%
    \mathchardef\@tempa\mathcode`\|\relax
    \def\@tempb##1"##2##3{\the\textfont"##3\char"}%
    \expandafter\@tempb\meaning\@tempa \relax
  }%
  \ht\Mathstrutbox@\ht\z@ \dp\Mathstrutbox@\dp\z@}
\makeatother
\begingroup
  \catcode`(\active \xdef({\left\string(}
  \catcode`)\active \xdef){\right\string)}
  \catcode`[\active \xdef[{\left\string[}
  \catcode`]\active \xdef]{\right\string]}
\endgroup
\mathcode`(="8000 \mathcode`)="8000
\mathcode`[="8000 \mathcode`]="8000
\DeclareRobustCommand{\{}{\ifmmode\left\lbrace\else\textbraceleft\fi}
\DeclareRobustCommand{\}}{\ifmmode\right\rbrace\else\textbraceright\fi}

\begin{document}

\begin{align}
 f(x) &= (\frac{(3-\frac{1}{x})^2}{(\frac4x-1)x}-1)x^2 \\
 g(x) &= x(1+x(2+x(3+x)))
\end{align}

\begin{align}
y = \{\begin{aligned}
  a \\b\\c
\end{aligned} \}
\end{align}

\begin{align}
y = \{\begin{aligned}
  a \\\relax[\frac{b_1}{b_2}]\\c % \relax hier wichtig
\end{aligned} \}
\end{align}

\end{document}