Running Selenium Webdriver with a proxy in Python

I am trying to run a Selenium Webdriver script in Python to do some basic tasks. I can get the robot to function perfectly when running it through the Selenium IDE inteface (ie: when simply getting the GUI to repeat my actions). However when I export the code as a Python script and try to execute it from the command line, the Firefox browser will open but cannot ever access the starting URL (an error is returned to command line and the program stops). This is happening me regardless of what website etc I am trying to access.

I have included a very basic code here for demonstration purposes. I don't think that I have included the proxy section of the code correctly as the error being returned seems to be generated by the proxy.

Any help would be hugely appreciated.

The below code is simply meant to open www.google.ie and search for the word "selenium". For me it opens a blank firefox browser and stops.

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import NoSuchElementException
import unittest, time, re
from selenium.webdriver.common.proxy import *

class Testrobot2(unittest.TestCase):
    def setUp(self):

	    myProxy = "http://149.215.113.110:70"

	    proxy = Proxy({
	    'proxyType': ProxyType.MANUAL,
	    'httpProxy': myProxy,
	    'ftpProxy': myProxy,
	    'sslProxy': myProxy,
	    'noProxy':''})

        self.driver = webdriver.Firefox(proxy=proxy)
        self.driver.implicitly_wait(30)
        self.base_url = "https://www.google.ie/"
        self.verificationErrors = []
        self.accept_next_alert = True
    
    def test_robot2(self):
        driver = self.driver
        driver.get(self.base_url + "/#gs_rn=17&gs_ri=psy-ab&suggest=p&cp=6&gs_id=ix&xhr=t&q=selenium&es_nrs=true&pf=p&output=search&sclient=psy-ab&oq=seleni&gs_l=&pbx=1&bav=on.2,or.r_qf.&bvm=bv.47883778,d.ZGU&fp=7c0d9024de9ac6ab&biw=592&bih=665")
        driver.find_element_by_id("gbqfq").clear()
        driver.find_element_by_id("gbqfq").send_keys("selenium")
    
    def is_element_present(self, how, what):
        try: self.driver.find_element(by=how, value=what)
        except NoSuchElementException, e: return False
        return True
    
    def is_alert_present(self):
        try: self.driver.switch_to_alert()
        except NoAlertPresentException, e: return False
        return True
    
    def close_alert_and_get_its_text(self):
        try:
            alert = self.driver.switch_to_alert()
            alert_text = alert.text
            if self.accept_next_alert:
                alert.accept()
            else:
                alert.dismiss()
            return alert_text
        finally: self.accept_next_alert = True
    
    def tearDown(self):
        self.driver.quit()
        self.assertEqual([], self.verificationErrors)

if __name__ == "__main__":
    unittest.main()

Works for me this way (similar to @Amey and @user4642224 code, but shorter a bit):

from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy, ProxyType

prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = "ip_addr:port"
prox.socks_proxy = "ip_addr:port"
prox.ssl_proxy = "ip_addr:port"

capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)

driver = webdriver.Chrome(desired_capabilities=capabilities)

How about something like this

PROXY = "149.215.113.110:70"

webdriver.DesiredCapabilities.FIREFOX['proxy'] = {
    "httpProxy":PROXY,
    "ftpProxy":PROXY,
    "sslProxy":PROXY,
    "noProxy":None,
    "proxyType":"MANUAL",
    "class":"org.openqa.selenium.Proxy",
    "autodetect":False
}

# you have to use remote, otherwise you'll have to code it yourself in python to 
driver = webdriver.Remote("http://localhost:4444/wd/hub", webdriver.DesiredCapabilities.FIREFOX)

You can read more about it here.

My solution:

def my_proxy(PROXY_HOST,PROXY_PORT):
        fp = webdriver.FirefoxProfile()
        # Direct = 0, Manual = 1, PAC = 2, AUTODETECT = 4, SYSTEM = 5
        print PROXY_PORT
        print PROXY_HOST
        fp.set_preference("network.proxy.type", 1)
        fp.set_preference("network.proxy.http",PROXY_HOST)
        fp.set_preference("network.proxy.http_port",int(PROXY_PORT))
        fp.set_preference("general.useragent.override","whater_useragent")
        fp.update_preferences()
        return webdriver.Firefox(firefox_profile=fp)

Then call in your code:

my_proxy(PROXY_HOST,PROXY_PORT)

I had issues with this code because I was passing a string as a port #:

 PROXY_PORT="31280"

This is important:

int("31280")

You must pass an integer instead of a string or your firefox profile will not be set to a properly port and connection through proxy will not work.

Try setting up sock5 proxy too. I was facing the same problem and it is solved by using the socks proxy

def install_proxy(PROXY_HOST,PROXY_PORT):
        fp = webdriver.FirefoxProfile()
        print PROXY_PORT
        print PROXY_HOST
        fp.set_preference("network.proxy.type", 1)
        fp.set_preference("network.proxy.http",PROXY_HOST)
        fp.set_preference("network.proxy.http_port",int(PROXY_PORT))
	    fp.set_preference("network.proxy.https",PROXY_HOST)
        fp.set_preference("network.proxy.https_port",int(PROXY_PORT))
	    fp.set_preference("network.proxy.ssl",PROXY_HOST)
        fp.set_preference("network.proxy.ssl_port",int(PROXY_PORT))  
	    fp.set_preference("network.proxy.ftp",PROXY_HOST)
        fp.set_preference("network.proxy.ftp_port",int(PROXY_PORT))   
	    fp.set_preference("network.proxy.socks",PROXY_HOST)
        fp.set_preference("network.proxy.socks_port",int(PROXY_PORT))   
	    fp.set_preference("general.useragent.override","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A")
        fp.update_preferences()
        return webdriver.Firefox(firefox_profile=fp)

Then call install_proxy ( ip , port ) from your program.

If anyone is looking for a solution here's how :

from selenium import webdriver
PROXY = "YOUR_PROXY_ADDRESS_HERE"
webdriver.DesiredCapabilities.FIREFOX['proxy']={
    "httpProxy":PROXY,
    "ftpProxy":PROXY,
    "sslProxy":PROXY,
    "noProxy":None,
    "proxyType":"MANUAL",
    "autodetect":False
}
driver = webdriver.Firefox()
driver.get('http://www.whatsmyip.org/')

Proxy with verification. This is a whole new python script in reference from a Mykhail Martsyniuk sample script.

# Load webdriver
from selenium import webdriver

# Load proxy option
from selenium.webdriver.common.proxy import Proxy, ProxyType

# Configure Proxy Option
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL

# Proxy IP & Port
prox.http_proxy = “0.0.0.0:00000”
prox.socks_proxy = “0.0.0.0:00000”
prox.ssl_proxy = “0.0.0.0:00000”

# Configure capabilities 
capabilities = webdriver.DesiredCapabilities.CHROME
prox.add_to_capabilities(capabilities)

# Configure ChromeOptions
driver = webdriver.Chrome(executable_path='/usr/local/share chromedriver',desired_capabilities=capabilities)

# Verify proxy ip
driver.get("http://www.whatsmyip.org/")

It's quite an old post, however, for others, it might still benefit by providing the answer as of today, and yet originally author was extremely close to a working solution.

First of all, the ftpProxy setting is no longer supported at this time and will throw an error

proxy = Proxy({
        'proxyType': ProxyType.MANUAL,
        'httpProxy': myProxy,
        'ftpProxy': myProxy, # this will throw an error
        'sslProxy': myProxy,
        'noProxy':''})

Next, instead of setting the proxy property, you should be using firefox options like so

proxy = Proxy({
    'proxyType': ProxyType.MANUAL,
    'httpProxy': myProxy,
    'sslProxy': myProxy,
    'noProxy': ''})

options = Options()
options.proxy = proxy
driver = webdriver.Firefox(options=options)

Additionally, don't define the scheme when specifying the proxy, especially if you want to use the same proxy for multiple protocols

myProxy = "149.215.113.110:70"

All together it looks like this

from selenium import webdriver
from selenium.webdriver.common.proxy import *
from selenium.webdriver.firefox.options import Options

myProxy = "149.215.113.110:70"
proxy = Proxy({
    'proxyType': ProxyType.MANUAL,
    'httpProxy': myProxy,
    'sslProxy': myProxy,
    'noProxy': ''})

options = Options()
options.proxy = proxy
driver = webdriver.Firefox(options=options)
driver.get("https://www.google.ie")

The result stated above may be correct, but isn't working with the latest webdriver. Here is my solution for the above question. Simple and sweet

		http_proxy  = "ip_addr:port"
		https_proxy = "ip_addr:port"

		webdriver.DesiredCapabilities.FIREFOX['proxy']={
			"httpProxy":http_proxy,
			"sslProxy":https_proxy,
			"proxyType":"MANUAL"
		}

        driver = webdriver.Firefox()

OR

	http_proxy  = "http://ip:port"
	https_proxy = "https://ip:port"

	proxyDict = {
					"http"  : http_proxy,
					"https" : https_proxy,
				}

	driver = webdriver.Firefox(proxy=proxyDict)

Try by Setting up FirefoxProfile

from selenium import webdriver
import time


"Define Both ProxyHost and ProxyPort as String"
ProxyHost = "54.84.95.51" 
ProxyPort = "8083"



def ChangeProxy(ProxyHost ,ProxyPort):
	"Define Firefox Profile with you ProxyHost and ProxyPort"
	profile = webdriver.FirefoxProfile()
	profile.set_preference("network.proxy.type", 1)
	profile.set_preference("network.proxy.http", ProxyHost )
	profile.set_preference("network.proxy.http_port", int(ProxyPort))
	profile.update_preferences()
	return webdriver.Firefox(firefox_profile=profile)


def FixProxy():
	""Reset Firefox Profile""
	profile = webdriver.FirefoxProfile()
	profile.set_preference("network.proxy.type", 0)
	return webdriver.Firefox(firefox_profile=profile)


driver = ChangeProxy(ProxyHost ,ProxyPort)
driver.get("http://whatismyipaddress.com")

time.sleep(5)

driver = FixProxy()
driver.get("http://whatismyipaddress.com")

This program tested on both Windows 8 and Mac OSX. If you are using Mac OSX and if you don't have selenium updated then you may face selenium.common.exceptions.WebDriverException. If so, then try again after upgrading your selenium

pip install -U selenium

The answers above and on this question either didn't work for me with Selenium 3.14 and Firefox 68.9 on Linux, or are unnecessarily complex. I needed to use a WPAD configuration, sometimes behind a proxy (on a VPN), and sometimes not. After studying the code a bit, I came up with:

from selenium import webdriver
from selenium.webdriver.common.proxy import Proxy
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile

proxy = Proxy({'proxyAutoconfigUrl': 'http://wpad/wpad.dat'})
profile = FirefoxProfile()
profile.set_proxy(proxy)
driver = webdriver.Firefox(firefox_profile=profile)

The Proxy initialization sets proxyType to ProxyType.PAC (autoconfiguration from a URL) as a side-effect.

It also worked with Firefox's autodetect, using:

from selenium.webdriver.common.proxy import ProxyType

proxy = Proxy({'proxyType': ProxyType.AUTODETECT})

But I don't think this would work with both internal URLs (not proxied) and external (proxied) the way WPAD does. Similar proxy settings should work for manual configuration as well. The possible proxy settings can be seen in the code here.

Note that directly passing the Proxy object as proxy=proxy to the driver does NOT work--it's accepted but ignored (there should be a deprecation warning, but in my case I think Behave is swallowing it).

As stated by @Dugini, some config entries have been removed. Maximal:

webdriver.DesiredCapabilities.FIREFOX['proxy'] = {
    "httpProxy":PROXY,
    "ftpProxy":PROXY,
    "sslProxy":PROXY,
    "noProxy":[],
    "proxyType":"MANUAL"
 }

This worked for me and allow to use an headless browser, you just need to call the method passing your proxy.

def setProxy(proxy):
        options = Options()
        options.headless = True
        #options.add_argument("--window-size=1920,1200")
        options.add_argument("--disable-dev-shm-usage")
        options.add_argument("--no-sandbox")
        prox = Proxy()
        prox.proxy_type = ProxyType.MANUAL
        prox.http_proxy = proxy
        prox.ssl_proxy = proxy
        capabilities = webdriver.DesiredCapabilities.CHROME
        prox.add_to_capabilities(capabilities)
        return webdriver.Chrome(desired_capabilities=capabilities, options=options, executable_path=DRIVER_PATH)

Scraping the data from any online source is quite easy when scraping APIs are used. You can try using scraper API to scrape the information from webpages and it automatically parses the web data. API can be integrated into your source code as well. Other than using API to scrape data, you can try the under-mentioned source code in beautiful soup to scrape data using CSS selectors. Before trying this code, please note that the select() method can be utilized to find numerous elements. Along with that, select_one() to be used search single element.

> Source Code:

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
PROXY = "177.46.141.143:59393" #your proxy  (ip address: port no)
chrome_options = WebDriverWait.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % PROXY)
chrome = webdriver.Chrome(chrome_options=chrome_options)
chrome.get("https://www.ipchicken.com/")

try running tor service, add the following function to your code.

def connect_tor(port):

socks.set_default_proxy(socks.PROXY_TYPE_SOCKS5, '127.0.0.1', port, True)
socket.socket = socks.socksocket

def main():

connect_tor()
driver = webdriver.Firefox()