Why is my code updating my vel variable outside of an if statement?

I made a 2d unity project with a circle inside of it. I am trying to get the circle to move by clicking dragging then releasing with your mouse. My code gets the initial mouse down action and reads the mouse position then gets the mouse position when i release the mouse. This works and gives the ball a velocity which lets it move in a direction but when I continue to move the mouse even after releasing the button it changes the balls velocity. I cannot figure out why it is doing this.

using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class NewBehaviourScript : MonoBehaviour
{
    public Rigidbody2D rb;

    private Vector2 vel;

    public int div;

    private int mouseDown;

    private Vector2 mousePos;
    private Vector2 mousePos2;

    // Start is called before the first frame update
    void Start()
    {
        mouseDown = 0;
    }

    // Update is called once per frame
    void Update()
    {
        if (Input.GetMouseButtonDown(0) && mouseDown == 0)
        {
            mousePos = Input.mousePosition;
            mouseDown = 1;
        }
        if (Input.GetMouseButtonUp(0) && mouseDown == 1)
        {
            mousePos2 = Input.mousePosition;
            vel = new Vector2((mousePos2.x - mousePos.x)/div,
                (mousePos2.y - mousePos.y)/div);
            mouseDown = 0;
            print(vel);
        }
        rb.MovePosition(rb.position + vel * Time.deltaTime);
    }
}

Note1: When working with Unity physics, namely rigidbody, make sure to use it in the FixUpdate. Documents: Here.

Note2: If you are sharing your code with others, make sure not to use abbreviated variable name. Its hard to know your intention in the code on first sight. e.g. vel , div.

Note3: It would help alot if you could describe what you are trying to archieve more clearly. Meaning,e.g. step1, what's your input action, step2, whats the output you wish to see. In the code provided, I made the assumption that the ball is drag towards your mouse drag direction instead of reverse, hope it matches to what you want.

Note4: Input.GetMouseButtonDown(0) and Input.GetMouseButtonUp(0) only captures once, so I don't think you need to have a seperate variable to count the mouseClick (your code -> int mouseDown). There is a Input.GetMouseButton(0) that will captures the holding of the button if that's what you need.

using UnityEngine;

public class CircleController : MonoBehaviour
{
    [SerializeField] private float _speed = 0.05f;
    private Rigidbody2D _rigidbody;
    private Vector3 _mouseDownPosition;
    private Vector3 _mouseUpPosition; 

    void Start()
    {
        _rigidbody = GetComponent<Rigidbody2D>();
    }

    void Update()
    {
        if (Input.GetMouseButtonDown(0))
        {
            _mouseDownPosition = Input.mousePosition;
            Debug.Log($"mouseDownPosition: {_mouseDownPosition}");
     
        }
        if (Input.GetMouseButtonUp(0))
        {
            _mouseUpPosition = Input.mousePosition;
            Debug.Log($"mouseUpPosition: {_mouseUpPosition}");
        }
    }
    
    void FixedUpdate()
    {
        Vector3 inputVector3 = _mouseUpPosition - _mouseDownPosition;
        _rigidbody.MovePosition(transform.position + inputVector3 * Time.deltaTime * _speed);
    }
}

My Scene Setup