You need to chmod every dir in order for Passenger to work correctly.
chmod 700 /home/deploy/.rvm chmod 700 /home/deploy/.rvm/gems/ chmod 700 /home/deploy/.rvm/gems/[email protected]/ chmod 700 /home/deploy/.rvm/gems/[email protected]/gems/ chmod 700 /home/deploy/.rvm/gems/[email protected]/gems/passenger-5.2.0