We consider ##0 < x < 1##. The cases ##x=0,1## can be dealt with separately, taken as a standard result. We have:
\begin{align*}
& \sum_{n=0}^\infty \dfrac{\cos (2 \pi n x)}{n^2}
\nonumber \\
& = \sum_{n=1}^\infty \dfrac{\cos (2 \pi n x)}{n^2} \int_0^\infty u e^{-u} du
\nonumber \\
& = \sum_{n=1}^\infty \cos (2 \pi n x) \int_0^\infty u e^{-nu} du
\nonumber \\
& = \frac{1}{2} \sum_{n=1}^\infty \int_0^\infty (e^{-nu + i n 2 \pi x} + e^{-nu - i n 2 \pi x}) u du
\nonumber \\
& = \frac{1}{2} \int_0^\infty \sum_{n=1}^\infty (e^{-nu + i n 2 \pi x} + e^{-nu - i n 2 \pi x}) u du
\nonumber \\
& = \frac{1}{2} \int_0^\infty \left( \dfrac{1}{e^{u - 2 \pi x i} - 1} + \dfrac{1}{e^{u + 2 \pi x i} - 1} \right) u du
\end{align*}
To justify interchanging summation and integration in the above, we can use Fubini:
\begin{align*}
\sum_{n=1}^\infty \int_0^\infty | \cos (2 \pi n x)u e^{-nu} | du & \leq \sum_{n=1}^\infty \int_0^\infty u e^{-nu} du
\nonumber \\
& = \sum_{n=1}^\infty \frac{1}{n^2}
\nonumber \\
& < 1 + \lim_{N \rightarrow \infty} \sum_{n=2}^N \frac{1}{n (n-1)}
\nonumber \\
& = 1 + \lim_{N \rightarrow \infty} \sum_{n=2}^N \left( \frac{1}{n-1} - \frac{1}{n} \right)
\nonumber \\
& = 1 + 1 - \lim_{N \rightarrow \infty} \frac{1}{N} = 2 < \infty
\end{align*}
From the above
\begin{align*}
\sum_{n=0}^\infty \dfrac{\cos (2 \pi n x)}{n^2} & = \frac{1}{2} \int_0^\infty \left( \dfrac{1}{e^{u - 2 \pi x i} - 1} + \dfrac{1}{e^{u + 2 \pi x i} - 1} \right) u du
\nonumber \\
& = \frac{1}{2} \int_0^\infty \left( \dfrac{e^{2 \pi x i} u}{e^u - e^{2 \pi x i}} + \dfrac{e^{-2 \pi x i} u}{e^u - e^{-2 \pi x i}} \right) du
\nonumber \\
& = \frac{1}{4} \int_0^\infty \left( \dfrac{e^{2 \pi x i} e^u u^2}{(e^u - e^{2 \pi x i})^2} + \dfrac{e^{-2 \pi x i} e^u u^2}{(e^u - e^{-2 \pi x i})^2} \right) du
\nonumber \\
& = \frac{1}{8} \int_{-\infty}^\infty \left( \dfrac{e^{2 \pi x i} e^u u^2}{(e^u - e^{2 \pi x i})^2} + \dfrac{e^{-2 \pi x i} e^u u^2}{(e^u - e^{-2 \pi x i})^2} \right) du
\nonumber \\
& = \frac{1}{4} Re \int_{-\infty}^\infty \dfrac{e^{2 \pi x i} e^u u^2}{(e^u - e^{2 \pi x i})^2} du
\end{align*}
where I done an integration by parts. Extending the range of integration is valid if you use both terms.
Define
\begin{align*}
I (\alpha) = \frac{1}{4} e^{2 \pi x i} \int_{-\infty}^\infty \dfrac{e^{\alpha u}}{(e^u - e^{2 \pi x i})^2} du
\end{align*}
for ##\frac{1}{2} \leq \alpha \leq \frac{3}{2}##. Then
\begin{align*}
\sum_{n=1}^\infty \dfrac{\cos (2 \pi n x)}{n^2} = \left. \dfrac{\partial^2}{\partial \alpha^2} Re I (\alpha) \right|_{\alpha =1}
\end{align*}
In order to evaluate ##I(\alpha)##, consider the contour in the figure
and the integral
\begin{align*}
\oint_C \dfrac{e^{2 \pi x i} e^{\alpha z}}{(e^z - e^{2 \pi x i})^2}
\end{align*}
whose integrand has pole at ##2 \pi x i##.
The integral along the vertical edges vanishes as:
\begin{align*}
f(z) = \dfrac{e^{2 \pi x i} e^{\alpha (u+iv)}}{(e^{u+iv} + e^{2 \pi x i})^2} =
\begin{cases}
e^{2 \pi x i} e^{(\alpha - 2) (u+iv)} & u \rightarrow \infty \\
\dfrac{e^{\alpha (u+iv)}}{e^{2 \pi x i}} & u \rightarrow - \infty \\
\end{cases}
\end{align*}
So that
\begin{align*}
& \frac{1}{4} \oint_C \dfrac{e^{2 \pi x i} e^{\alpha z}}{(e^z - e^{2 \pi x i})^2}
\nonumber \\
& = \frac{1}{4} \int_{-\infty}^\infty \dfrac{e^{2 \pi x i} e^{\alpha u}}{(e^u - e^{2 \pi x i})^2} du - \frac{1}{4} e^{\alpha 2 \pi i} \int_{-\infty + 2 \pi i}^{\infty + 2 \pi i} \dfrac{e^{2 \pi x i} e^{\alpha u}}{(e^u - e^{2 \pi x i})^2} du
\nonumber \\
& = (1-e^{\alpha 2 \pi i}) \frac{1}{4} \int_{-\infty}^\infty \dfrac{e^{2 \pi x i} e^{\alpha u}}{(e^u - e^{2 \pi x i})^2} du
\end{align*}
Which rearranged is
\begin{align*}
\frac{1}{4} e^{2 \pi x i} \int_{-\infty}^\infty \dfrac{e^{\alpha u}}{(e^u - e^{2 \pi x i})^2} du & = \frac{i \pi}{2} \frac{1}{1-e^{\alpha 2 \pi i}} \frac{1}{2 \pi i} \oint_C \dfrac{e^{2 \pi x i} e^{\alpha z}}{(e^z - e^{2 \pi x i})^2} dz
\nonumber \\
& = \frac{i \pi}{2} \frac{1}{1-e^{\alpha 2 \pi i}} Res [f(z)]
\end{align*}
We have
\begin{align*}
\frac{1}{(e^z - e^{z_0})^2} & = \frac{e^{-2 z_0}}{(e^{z-z_0} - 1)^2}
\nonumber \\
& = \dfrac{e^{-2 z_0}}{(z-z_0)^2 [1 + \frac{1}{2!} (z-z_0) + \cdots]^2}
\nonumber \\
& = \dfrac{e^{-2z_0}}{(z-z_0)^2} [1 - (z-z_0) + \cdots]
\nonumber \\
& = \dfrac{e^{-2z_0}}{(z-z_0)^2} - \dfrac{e^{-2z_0}}{z-z_0} + \cdots
\end{align*}
where ##z_0## is the position of the pole ##2 \pi x i##. We use this in extracting the residue of ##f(z)## at ##z_0##:
\begin{align*}
e^{z_0} \dfrac{e^{\alpha z}}{(e^z - e^{z_0})^2} & = e^{-z_0} \dfrac{ e^{\alpha z_0 + \alpha (z-z_0)} }{ (z-z_0)^2 } - e^{-z_0} \dfrac{ e^{\alpha z_0} }{ z-z_0 } + \cdots
\nonumber \\
& = e^{-z_0} \dfrac{ e^{\alpha z_0} [1 + \alpha (z-z_0) + \cdots] }{ (z-z_0)^2 } - e^{-z_0} \dfrac{ e^{\alpha z_0} }{ z-z_0 } + \cdots
\nonumber \\
& = e^{-z_0} \dfrac{ e^{\alpha z_0} }{ (z-z_0)^2 } + e^{-z_0} \dfrac{ (\alpha - 1) e^{\alpha z_0} }{ z-z_0 } + \cdots
\end{align*}
Combining everything
\begin{align*}
I(\alpha) & = \frac{1}{4} e^{2 \pi x i} \int_{-\infty}^\infty \dfrac{e^{\alpha u}}{(e^u - e^{2 \pi x i})^2} du
\nonumber \\
& = \frac{i \pi}{2} \frac{1}{1-e^{\alpha 2 \pi i}} \cdot e^{-2 \pi x i} (\alpha - 1) e^{\alpha 2 \pi x i}
\nonumber \\
& = - \frac{\pi}{4} e^{(\alpha - 1) 2 \pi x i - \alpha \pi i} \frac{(\alpha - 1) 2i}{e^{\alpha \pi i} - e^{-\alpha \pi i}}
\end{align*}
So that
\begin{align*}
Re I(\alpha) & = - \frac{\pi}{4} \cos [(\alpha - 1) 2 x \pi - \alpha \pi] \frac{\alpha - 1}{\sin \alpha \pi}
\end{align*}
Taking second derivative w.r.t. ##\alpha## results in
\begin{align*}
& \dfrac{\partial^2}{\partial \alpha^2} Re I(\alpha)
\nonumber \\
& = \frac{\pi}{4} \pi^2 (2x-1)^2 \cos [(\alpha - 1) 2 x \pi - \alpha \pi] \frac{\alpha - 1}{\sin \alpha \pi}
\nonumber \\
& \frac{\pi}{2} \pi (2x-1) \sin [(\alpha - 1) 2 x \pi - \alpha \pi] \dfrac{\partial}{\partial \alpha} \frac{\alpha - 1}{\sin \alpha \pi}
\nonumber \\
& - \frac{\pi}{4} \cos [(\alpha - 1) 2 x \pi - \alpha \pi] \dfrac{\partial^2}{\partial \alpha^2} \frac{\alpha - 1}{\sin \alpha \pi}
\nonumber \\
& = \frac{\pi}{4} \pi^2 (2x-1)^2 \cos [(\alpha - 1) 2 x \pi - \alpha \pi] \frac{\alpha - 1}{\sin \alpha \pi}
\nonumber \\
& \frac{\pi}{2} \pi (2x-1) \sin [(\alpha - 1) 2 x \pi - \alpha \pi] \frac{\sin \alpha \pi - (\alpha - 1) \pi \cos \alpha \pi}{\sin^2 \alpha \pi}
\nonumber \\
& - \frac{\pi}{4} \cos [(\alpha - 1) 2 x \pi - \alpha \pi] \frac{(\alpha - 1) \pi^2\sin^3 \alpha \pi - [\sin \alpha \pi - (\alpha - 1) \pi \cos \alpha \pi] 2 \pi \cos \alpha \pi \sin \alpha \pi}{\sin^4 \alpha \pi}
\end{align*}
I handled all but the last term with L'Hopital. With the last term, initially we do the Taylor expansion:
\begin{align*}
& \frac{[\sin \alpha \pi - (\alpha - 1) \pi \cos \alpha \pi] 2 \pi \cos \alpha \pi \sin \alpha \pi}{\sin^4 \alpha \pi}
\nonumber \\
& = 2 \pi \left( \dfrac{\cos \alpha \pi}{\sin^2 \alpha \pi} \left( 1 - \pi \dfrac{\alpha - 1}{\sin \alpha \pi} \cos \alpha \pi \right) \right)
\nonumber \\
& = 2 \pi \left( \dfrac{\cos \alpha \pi}{\sin^2 \alpha \pi} \left( 1 - \pi \dfrac{\alpha - 1}{\sin (\alpha - 1) \pi} \cos (\alpha - 1) \pi \right) \right)
\nonumber \\
& = 2 \pi \left( \dfrac{\cos \alpha \pi}{\sin^2 \alpha \pi} \left( 1 - \pi \dfrac{\alpha - 1}{(\alpha - 1) \pi [ 1 - \frac{1}{3!} (\alpha - 1)^2 \pi^2 \cdots]} [ 1 - \frac{1}{2!} (\alpha - 1)^2 \pi^2 + \cdots \right) \right)
\nonumber \\
& = 2 \pi \left( \dfrac{\cos \alpha \pi}{\sin^2 \alpha \pi} \left( 1 - [ 1 + \frac{1}{3!} (\alpha - 1)^2 \pi^2 + \cdots] [ 1 - \frac{1}{2!} (\alpha - 1)^2 \pi^2 + \cdots \right) \right)
\nonumber \\
& = 2 \pi \left( \dfrac{\cos \alpha \pi}{\sin^2 \alpha \pi} \left( \frac{1}{3} (\alpha - 1)^2 \pi^2 + \cdots \right) \right)
\nonumber \\
& = \frac{2}{3} \pi^3 \cos \alpha \pi \dfrac{(\alpha - 1)^2}{\sin^2 \alpha \pi} + \text{terms that vanish as } \alpha \rightarrow 1
\end{align*}
So that
\begin{align*}
& \lim_{\alpha \rightarrow 1} \frac{\pi}{4} \cos [(\alpha - 1) 2 x \pi - \alpha \pi] \frac{[\sin \alpha \pi - (\alpha - 1) \pi \cos \alpha \pi] 2 \pi \cos \alpha \pi \sin \alpha \pi}{\sin^4 \alpha \pi}
\nonumber \\
& = \lim_{\alpha \rightarrow 1} \frac{\pi}{6} \cos [(\alpha - 1) 2 x \pi - \alpha \pi] \pi^3 \cos \alpha \pi \dfrac{(\alpha - 1)^2}{\sin^2 \alpha \pi}
\nonumber \\
& = \frac{\pi^2}{6}
\end{align*}
From using L'Hopital on the first terms and using the result just proven:
\begin{align*}
\left. \dfrac{\partial^2}{\partial \alpha^2} Re I(\alpha) \right|_{\alpha = 1} & = \frac{\pi}{4} \pi^2 (2x-1)^2 \frac{1}{\pi} + \frac{\pi^2}{2} (2x-1)^2 - \frac{\pi^2}{2} (2x-1)^2
\nonumber \\
& - \frac{\pi^2}{4} + \frac{\pi^2}{6}
\end{align*}
So finally:
\begin{align*}
\sum_{n=1}^\infty \dfrac{\cos (2 \pi n x)}{n^2} = \pi^2 \left( x^2 - x + \frac{1}{6} \right) .
\end{align*}
2nd method for calculating integral
We use the same rectangular contour. In the complex contour integrals here, the integral along vertical edges vanishes.
Consider
\begin{align*}
\oint_C \dfrac{e^{2 \pi x i} e^z z^3}{(e^z - e^{2 \pi x i})^2} dz & = \int_{-\infty}^\infty \dfrac{e^{2 \pi x i} e^u u^3}{(e^u - e^{2 \pi x i})^2} du - \int_{-\infty+2 \pi i}^{\infty + 2 \pi i} \dfrac{e^{2 \pi x i} e^u (u+ 2 \pi i)^3}{(e^u - e^{2 \pi x i})^2} du
\nonumber \\
& = - \int_{-\infty}^\infty \dfrac{e^{2 \pi x i} e^u (3 u^2 (2 \pi i) + 3u (2 \pi i)^2 + (2\pi i)^3)}{(e^u - e^{2 \pi x i})^2} du
\nonumber \\
& = - 3 (2 \pi i) \int_{-\infty}^\infty \dfrac{e^{2 \pi x i} u^2 e^u}{(e^u - e^{2 \pi x i})^2} du - 3 (2 \pi i)^2 \int_{-\infty}^\infty \dfrac{e^{2 \pi x i} u e^u}{(e^u - e^{2 \pi x i})^2} du
\nonumber \\
& - (2 \pi i)^3 \int_{-\infty}^\infty \dfrac{e^{2 \pi x i} e^u}{(e^u - e^{2 \pi x i})^2} du
\nonumber \\
& - 3 (2 \pi i) I_2 - 3 (2 \pi i)^2 I_1 - (2 \pi i)^3 I_0
\end{align*}
So that
\begin{align*}
\frac{1}{2 \pi i} \oint_C \dfrac{e^{2 \pi x i} e^z z^3}{(e^z - e^{2 \pi x i})^2} dz = - 3 I_2 - 3 (2 \pi i) I_1 - (2 \pi i)^2 I_0
\end{align*}
The integral ##I_0## is straightforward to do:
\begin{align*}
I_0 & = \int_{-\infty}^\infty \dfrac{e^{2 \pi x i} e^u}{(e^u - e^{2 \pi x i})^2} du
\nonumber \\
& = \int_{-\infty}^\infty \left( - \frac{d}{du} \dfrac{e^{2 \pi x i}}{e^u - e^{2 \pi x i}} \right) du
\nonumber \\
& = \left[ - \dfrac{e^{2 \pi x i}}{e^u - e^{2 \pi x i}} \right]_{-\infty}^\infty = - 1
\end{align*}
We have the following residue (see below for details)
\begin{align*}
\text{Res}_{z = 2 \pi x i} \dfrac{e^{2 \pi x i} e^z z^3}{(e^z - e^{2 \pi x i})^2} = - 12 \pi^2 x^2 = - 3 I_2 - 3 (2 \pi i) I_1 - (2 \pi i)^2 I_0
\end{align*}
Consider
\begin{align*}
\oint_C \dfrac{e^{2 \pi x i} e^z z^2}{(e^z - e^{2 \pi x i})^2} dz & = \int_{-\infty}^\infty \dfrac{e^{2 \pi x i} e^u u^2}{(e^u - e^{2 \pi x i})^2} du - \int_{-\infty+2 \pi i}^{\infty + 2 \pi i} \dfrac{e^{2 \pi x i} e^u (u+ 2 \pi i)^2}{(e^u - e^{2 \pi x i})^2} du
\nonumber \\
& = - \int_{-\infty}^\infty \dfrac{e^{2 \pi x i} e^u (2 u (2 \pi i) + (2\pi i)^2}{(e^u - e^{2 \pi x i})^2} du
\nonumber \\
& = - 2 (2 \pi i) \int_{-\infty}^\infty \dfrac{e^{2 \pi x i} u e^u}{(e^u - e^{2 \pi x i})^2} du - (2 \pi i)^2 \int_{-\infty}^\infty \dfrac{e^{2 \pi x i} e^u}{(e^u - e^{2 \pi x i})^2} du
\nonumber \\
& - 2 (2 \pi i) I_1 + (2 \pi i)^2
\end{align*}
So that
\begin{align*}
\frac{1}{2 \pi i} \oint_C \dfrac{e^{2 \pi x i} e^z z^2}{(e^z - e^{2 \pi x i})^2} dz = - 2 I_1 + (2 \pi i)
\end{align*}
We have the following residue
\begin{align*}
\text{Res}_{z = 2 \pi x i} \dfrac{e^{2 \pi x i} e^z z^2}{(e^z - e^{2 \pi x i})^2} = i 4 \pi x = - 2 I_1 + (2 \pi i)
\end{align*}
Collecting results
\begin{align*}
- 12 \pi^2 x^2 & = - 3 I_2 - 3 (2 \pi i) I_1 + (2 \pi i)^2
\nonumber \\
i 2 \pi x & = - I_1 + (\pi i)
\nonumber \\
I_0 & = -1
\end{align*}
So that
\begin{align*}
I_2 & = 4 \pi^2 x^2 - 2 \pi i I_1 - \frac{4}{3} \pi^2
\nonumber \\
& = 4 \pi^2 x^2 - 2 \pi i (- i 2 \pi x + \pi i) - \frac{4}{3} \pi^2
\nonumber \\
& = 4 \pi^2 x^2 - 4 \pi^2 x + \frac{2}{3} \pi^2
\end{align*}
So finally,
\begin{align*}
\sum_{n=0}^\infty \dfrac{\cos (2 \pi n x)}{n^2} & = \frac{1}{4} Re \int_{-\infty}^\infty \dfrac{e^{2 \pi x i} e^u u^2}{(e^u - e^{2 \pi x i})^2} du
\nonumber \\
& = \frac{1}{4} Re I_2
\nonumber \\
& = \pi^2 \left( x^2 - x + \frac{1}{6} \right) .
\end{align*}
Calculating residues
We calculate
\begin{align*}
\text{Res}_{z = 2 \pi x i} \dfrac{e^{2 \pi x i} e^z z^k}{(e^z - e^{2 \pi x i})^2} .
\end{align*}
Again,
\begin{align*}
\frac{1}{(e^z - e^{z_0})^2} = \dfrac{e^{-2z_0}}{(z-z_0)^2} - \dfrac{e^{-2z_0}}{z-z_0} + \cdots
\end{align*}
So
\begin{align*}
\frac{e^{z_0} e^z z^k}{(e^z - e^{z_0})^2} & = \frac{e^{2z_0} e^{z-z_0} [z_0 +(z-z_0)]^k}{(e^z - e^{z_0})^2}
\nonumber \\
& = \dfrac{[1 + (z-z_0) + \cdots] [z_0^k + kz_0^{k-1} (z-z_0) + \cdots]}{(z-z_0)^2} - \dfrac{z_0^k}{z-z_0} + \cdots
\nonumber \\
& = \dfrac{z_0^k}{(z-z_0)^2} + \dfrac{kz_0^{k-1}}{z-z_0} + \cdots
\end{align*}
So
\begin{align*}
\text{Res}_{z = 2 \pi x i} \dfrac{e^{2 \pi x i} e^z z^k}{(e^z - e^{2 \pi x i})^2} & = k (2 \pi x i)^{k-1}
\nonumber \\
& = i^{k-1} 2^{k-1} k \pi^{k-1} x^{k-1} .
\end{align*}