Begin with integration by parts:
#intudv = uv - intvdu#
let #u = (ln(x))^2# and #dv = x^-3dx#
then #du = (2ln(x))/xdx# and #v = -1/2x^-2#
#int(ln(x))^2/x^3dx = -1/2(ln(x)/x)^2 -int(-1/(2x^2))((2ln(x))/x)dx #
#int(ln(x))^2/x^3dx = -1/2(ln(x)/x)^2 +int(ln(x))/x^3dx #
Integrate by parts:
#intudv = uv - intvdu#
let #u = ln(x)# and #dv = x^-3dx#
then #du = 1/xdx# and #v = -1/2x^-2#
#int(ln(x))^2/x^3dx = -1/2(ln(x)/x)^2 - ln(x)/(2x^2) - int (-1/2x^-2)(1/x)dx#
#int(ln(x))^2/x^3dx = -1/2(ln(x)/x)^2 - ln(x)/(2x^2) +1/2int 1/x^3dx#
We have already done the last integral:
#int(ln(x))^2/x^3dx = -1/2(ln(x)/x)^2 - ln(x)/(2x^2) -1/(4x^2)+ C#
Simplify over a common denominator:
#int(ln(x))^2/x^3dx = -(2(ln(x))^2 + 2ln(x) +1)/(4x^2)+ C#