cot^2(pi/7)+cot^2((2pi)/7)+cot^2((3pi)/7)
=csc^2(pi/7)+csc^2((2pi)/7)+csc^2((3pi)/7)-3
=1/sin^2(pi/7)+1/sin^2((2pi)/7)+1/sin^2((3pi)/7)-3
=2/(1-cos((2pi)/7))+2/(1-cos((4pi)/7))+2/(1-cos((6pi)/7))-3
=2/(1+cos((5pi)/7))+2/(1+cos((3pi)/7))+2/(1+cos(pi/7))-3
=2[((1+cos((3pi)/7))(1+cos(pi/7))+(1+cos((5pi)/7))(1+cos(pi/7)+(1+cos((5pi)/7))(1+cos((3pi)/7))))/((1+cos((5pi)/7))(1+cos((3pi)/7))(1+cos(pi/7)))]-3
=2[(3+2cos((3pi)/7)+2cos(pi/7)+2cos((5pi)/7)+cos(pi/7)cos((3pi)/7)+cos((3pi)/7)cos((5pi)/7)+cos((5pi)/7)cos(pi/7))/((1+cos((5pi)/7))(1+cos((3pi)/7))(1+cos(pi/7)))]-3
=2[(3+2cos((3pi)/7)+2cos(pi/7)+2cos((5pi)/7)+1/2(cos((4pi)/7)+cos((2pi)/7)+cos((8pi)/7)+cos((2pi)/7)+cos((6pi)/7)+cos((4pi)/7)))/((1+cos((5pi)/7))(1+cos((3pi)/7))(1+cos(pi/7)))]-3
=2[(3+2cos((3pi)/7)+2cos(pi/7)+2cos((5pi)/7)+1/2(-cos((3pi)/7)-cos((5pi)/7)-cos(pi/7)-cos((5pi)/7)-cos(pi/7)-cos((3pi)/7)))/((1+cos((5pi)/7))(1+cos((3pi)/7))(1+cos(pi/7)))]-3
=2[(3+cos((3pi)/7)+cos(pi/7)+cos((5pi)/7))/((1+cos((5pi)/7))(1+cos((3pi)/7))(1+cos(pi/7)))]-3
=2[(3+cos((3pi)/7)+cos(pi/7)+cos((5pi)/7))/(1+cos((3pi)/7)+cos(pi/7)+cos((5pi)/7)+cos(pi/7)cos((3pi)/7)+cos((3pi)/7)cos((5pi)/7)+cos((5pi)/7)cos(pi/7)+cos((5pi)/7)cos((3pi)/7)cos(pi/7))]-3
=2[(3+cos((3pi)/7)+cos(pi/7)+cos((5pi)/7))/((1+cos((3pi)/7)+cos(pi/7)+cos((5pi)/7)+1/2(-cos((3pi)/7)-cos((5pi)/7)-cos(pi/7)-cos((5pi)/7)-cos(pi/7)-cos((3pi)/7))+cos((5pi)/7)cos((3pi)/7))cos(pi/7))]-3
=color(red)(2[(3+cos((3pi)/7)+cos(pi/7)+cos((5pi)/7))/(1+cos((5pi)/7)cos((3pi)/7)cos(pi/7))]-3)
=2((3+1/2)/(1-1/8))-3 [ please see the note below ]
=(2xx7/2)/(7/8)-3=5
cos((3pi)/7)+cos(pi/7)+cos((5pi)/7))
=1/(2sin(pi/7))[2sin(pi/7)cos(pi/7)+2sin((3pi)/7)cos(pi/7)+2sin(pi/7)cos((5pi)/7))]
=1/(2sin(pi/7))[sin((2pi)/7)+sin((4pi)/7)-sin((2pi)/7)+sin((6pi)/7)-sin((4pi)/7)]
=1/(2sin(pi/7))[sin(pi-pi/7)]
=1/(2sin(pi/7))*sin(pi/7)
=1/2
Again
cos((5pi)/7)cos((3pi)/7)cos(pi/7)
=1/(2sin(pi/7))[2sin(pi/7)cos(pi-(2pi)/7)cos(pi-(4pi)/7)cos(pi/7)]
=1/(4sin(pi/7))[2sin((2pi)/7)cos((2pi)/7)cos((4pi)/7)]
=1/(8sin(pi/7))[2sin((4pi)/7)cos((4pi)/7)]
=1/(8sin(pi/7))sin((8pi)/7)
=1/(8sin(pi/7))sin(pi+pi/7)
=-1/(8sin(pi/7))sin(pi/7)=-1/8