Made something that should act like the old code, but run faster, and use less code. You might also want to do a raycast where you set the target variable and find the actualdistance, instead of using 20.

Feel free to change the way limit is calculated back to your way. I just like that when limitRadius is larger, the circle is larger.

var crosshairTex : Texture2D;
var sensitivity : float;
var limitRadius : float;

private var limit : float;
private var origo = Vector2(Screen.width / 2, Screen.height / 2);
private var currentPoint = Vector2.zero;
private var oldPoint = Vector2.zero;
private var particleGun : Transform;
private var crosshairPos : Rect;
private var target : Vector3;

function Start ()
{
    particleGun = GameObject.Find("ParticleGun").transform;
    crosshairPos = Rect(origo.x - (crosshairTex.width / 2), (origo.y - (crosshairTex.height / 2)), crosshairTex.width, crosshairTex.height);
    limit = Screen.width * limitRadius;
}

function Update ()
{
    oldPoint = currentPoint;
    currentPoint.x += Input.GetAxis ("Mouse X") * sensitivity;
    currentPoint.y += Input.GetAxis ("Mouse Y") * sensitivity;
    currentPoint = Vector2.ClampMagnitude(currentPoint, limit);
    crosshairPos.x += currentPoint.x-oldPoint.x;
    crosshairPos.y -= currentPoint.y-oldPoint.y;

    target = camera.ScreenToWorldPoint (Vector3(currentPoint.x+origo.x, currentPoint.y+origo.y, 20.0));
    particleGun.LookAt(target, transform.up);
}

function OnGUI () 
{
    GUI.DrawTexture (crosshairPos, crosshairTex);
}

Well this is how I solved this problem. Since I needed a crosshair for the gun in my scene anyway, I just control the crosshair GUI instead of the gun. I limited its movement to a circle around the center of the screen, converted screen coordinates to world coordinates and used the transform.LookAt function to turn the gun towards that point.

There must be a better way to do it, or maybe there isn't. I'm still interested in ideas.

var crosshairTex : Texture2D;
var sensitivity : float;
var limitRadius : float;

private var limit : float;
private var origo : Vector3;
private var targetCenter : Vector3;
private var crosshairPos : Rect;
private var start : boolean = true;

function Start ()
{
    targetCenter = Vector3(Screen.width / 2, Screen.height / 2, 0);
}

function Update ()
{
    target = camera.ScreenToWorldPoint (Vector3(targetCenter.x, targetCenter.y, 20.0));
    GameObject.Find ("ParticleGun").transform.LookAt (target, transform.up);
}

function OnGUI () 
{
    origo = Vector3(Screen.width / 2, Screen.height / 2, 0);
    limit = Screen.width / limitRadius;

    targetCenter.x += Input.GetAxis ("Mouse X") * sensitivity;
    targetCenter.y += Input.GetAxis ("Mouse Y") * sensitivity;

    var currentRadius = targetCenter - origo;

    if (currentRadius.sqrMagnitude > Mathf.Pow (limit, 2))
    {
        currentRadius = Vector3.ClampMagnitude (currentRadius, limit);
        targetCenter.x = origo.x + currentRadius.x;
        targetCenter.y = origo.y + currentRadius.y;
    }

    crosshairPos = Rect(targetCenter.x - (crosshairTex.width / 2), Screen.height - (targetCenter.y + (crosshairTex.height / 2)), crosshairTex.width, crosshairTex.height);

    GUI.DrawTexture (crosshairPos, crosshairTex);
}

I think that this will work:

Vector3 CurRotation = transform.rotation.eulerAngles;

if(CurRotation.x > 60)//this will limit looking up to 60
    transform.rotation = Quaternion.Euler(new Vector3(60,CurRotation.y,CurRotation.z));
else if(CurRotation.x < -60)//this will limit looking down to -60
    transform.rotation = Quaternion.Euler(new Vector3(-60,CurRotation.y,CurRotation.z));

p.s.

I din't test this code.

p.p.s.

Yes it works ;) I just made a small test.

I just wrote this little ditty the other night. According to the diagram in your post, and if my understanding (that you want to limit the angle of deviation from a single axis/vector) is correct, this script should prove useful for you.

using UnityEngine;
using System.Collections;

/**
 * ConeMouseLook.cs
 * 
 * Restricts the basic mouse look script to keep the view within a cone.  The cone 
 * is defined by the maxAngle and the rotation at the moment the look is frozen, 
 * unless one of the overloaded Restrict methods is used instead of the no-argument 
 * Restrict method.
 * 
 * Author: Robert Grant
 * Version: 11/22/2012
 */
public class ConeMouseLook : MonoBehaviour {
    public float maxAngle = 45.0f;     // maximum deviation
    public float xSensitivity = 1.0f;  // horizontal sensitivity
    public float ySensitivity = -1.0f; // vertical sensitivity

    private Quaternion baseRotation;
    private bool isRestricted;
    private Quaternion requestedRotation;

    void Start () {
       isRestricted = true;
       baseRotation = transform.rotation;
       requestedRotation = Quaternion.identity;
       if (rigidbody) {
         rigidbody.freezeRotation = true;
       }
    }

    void LateUpdate() {
       if (Time.timeScale != 0.0f) {
         // Save for correcting the roll later.
         float z = transform.localEulerAngles.z;

         // Find the requested rotation - we may not use it or all of it.
         requestedRotation = Quaternion.Euler(
              Input.GetAxis("Mouse Y") * ySensitivity, 
              Input.GetAxis("Mouse X") * xSensitivity, 
              0
          );
         float requestedAngle = Quaternion.Angle(
              baseRotation, 
              transform.rotation * requestedRotation
          );

         // Do the rotation.
         if (isRestricted && requestedAngle > maxAngle) {
          // Limit the rotation since too much was requested.
          transform.rotation = Quaternion.RotateTowards(
                 transform.rotation * requestedRotation, 
                 baseRotation, 
                 requestedAngle - maxAngle
              );
         } else {
          // Just rotate freely!
          transform.rotation = Quaternion.Slerp(
                 transform.rotation, 
                 transform.rotation * requestedRotation, 
                 .25f
              );
         }

         // Correct any un-wanted roll.
         Vector3 angles = transform.localEulerAngles; 
         angles.z = z;
         transform.localEulerAngles = angles;
       }
    }

    /**
     *     Restricts the rotation to at most maxAngle degrees from the current rotation.
     */
    public void Restrict(float maxAngle) {
       this.maxAngle = maxAngle;
       isRestricted = true;
       baseRotation = transform.rotation;
    }

    /**
     *     Restricts the rotation to at most maxAngle degrees from the given direction.
     */
    public void Restrict(Vector3 direction, float maxAngle) {
       this.maxAngle = maxAngle;
       baseRotation = Quaternion.LookRotation(direction, Vector3.up);
       isRestricted = true;
    }

    /**
     *     Restricts the rotation relative to the given direction using the current maxAngle.
     */
    public void Restrict(Vector3 direction) {
       baseRotation = Quaternion.LookRotation(direction, Vector3.up);
       isRestricted = true;
    }

    /**
     *     Restricts the rotation using the current rotation and maxAngle.
     */
    public void Restrict() {
       baseRotation = transform.rotation;
       isRestricted = true;
    }

    /**
     *     Frees the rotation entirely.
     */
    public void Free() {
       isRestricted = false;
    }
}

The rotation is restricted on Start(), so you might want to change that.

This worked well for me:

void Update () {
    float ang = Time.deltaTime * speed * Input.GetAxis("Horizontal");
    myTransform.RotateAround(myCamera.ScreenToWorldPoint(new Vector3(Screen.width * 0.5f, Screen.height * 0.5f, myCamera.nearClipPlane)), -myTransform.right, ang);
}

Note: In my case, -myTransform.right is the vector that faces the camera