CA
Crawlee & Apify•2mo ago
stormy-gold

proxy_config.new_url() does not return new proxy

Here is my selenium python script, where i try to rotate proxies using the proxy_config.new_url(): 
# Standard libraries
import asyncio
import logging
import json

# Installed libraries
from selenium.common.exceptions import TimeoutException, WebDriverException
from selenium.webdriver.chrome.options import Options as ChromeOptions
from selenium.webdriver.common.proxy import ProxyType, Proxy
from selenium.webdriver.common.by import By
from selenium import webdriver
from apify import Actor

async def main() -> None:
    async with Actor:
        Actor.log.setLevel(logging.DEBUG)
        proxy_config = await Actor.create_proxy_configuration(groups=['RESIDENTIAL'])
        url = "https://api.ipify.org?format=json"
        for _ in range(10):
            proxy = await proxy_config.new_url()
            Actor.log.info(f'Using proxy: {proxy}')
            chrome_options = ChromeOptions()
            chrome_options.add_argument("--headless")
            chrome_options.add_argument("--no-sandbox")
            chrome_options.add_argument("--disable-dev-shm-usage")
            chrome_options.proxy = Proxy({'proxyType': ProxyType.MANUAL, 'httpProxy': proxy})
            try:
                with webdriver.Chrome(options=chrome_options) as driver:
                    driver.set_page_load_timeout(20)
                    driver.get(url)
                    content = driver.find_element(By.TAG_NAME, 'pre').text
                    ip = json.loads(content).get("ip")
                    Actor.log.info(f"IP = {ip}")
            except (TimeoutException, WebDriverException, json.JSONDecodeError):
                Actor.log.exception("An error occured")
# Standard libraries
import asyncio
import logging
import json

# Installed libraries
from selenium.common.exceptions import TimeoutException, WebDriverException
from selenium.webdriver.chrome.options import Options as ChromeOptions
from selenium.webdriver.common.proxy import ProxyType, Proxy
from selenium.webdriver.common.by import By
from selenium import webdriver
from apify import Actor

async def main() -> None:
    async with Actor:
        Actor.log.setLevel(logging.DEBUG)
        proxy_config = await Actor.create_proxy_configuration(groups=['RESIDENTIAL'])
        url = "https://api.ipify.org?format=json"
        for _ in range(10):
            proxy = await proxy_config.new_url()
            Actor.log.info(f'Using proxy: {proxy}')
            chrome_options = ChromeOptions()
            chrome_options.add_argument("--headless")
            chrome_options.add_argument("--no-sandbox")
            chrome_options.add_argument("--disable-dev-shm-usage")
            chrome_options.proxy = Proxy({'proxyType': ProxyType.MANUAL, 'httpProxy': proxy})
            try:
                with webdriver.Chrome(options=chrome_options) as driver:
                    driver.set_page_load_timeout(20)
                    driver.get(url)
                    content = driver.find_element(By.TAG_NAME, 'pre').text
                    ip = json.loads(content).get("ip")
                    Actor.log.info(f"IP = {ip}")
            except (TimeoutException, WebDriverException, json.JSONDecodeError):
                Actor.log.exception("An error occured")
Due to discord message size limitation i attach the log output of the above code in a new message below...
5 Replies
Hall
Hall•2mo ago
Someone will reply to you shortly. In the meantime, this might help: -# This post was marked as solved by Matous. View answer.
stormy-gold
stormy-goldOP•2mo ago
The logs of the above code show the following: 
Actor is running on the Apify platform, `disable_browser_sandbox` was changed to True.
INFO  Initializing Actor...
INFO  System info ({"apify_sdk_version": "2.5.0", "apify_client_version": "1.9.2", "crawlee_version": "0.6.5", "python_version": "3.13.2", "os": "linux"})
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Exiting Actor ({"exit_code": 0})
Actor is running on the Apify platform, `disable_browser_sandbox` was changed to True.
INFO  Initializing Actor...
INFO  System info ({"apify_sdk_version": "2.5.0", "apify_client_version": "1.9.2", "crawlee_version": "0.6.5", "python_version": "3.13.2", "os": "linux"})
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Using proxy: ***10.0.38.50:8011
INFO  IP = 13.217.115.135
INFO  Exiting Actor ({"exit_code": 0})
As you can see the script queries the browser IP from https://api.ipify.org?format=json and logs it in output. Even though the proxy_config.new_url() is used to get a new proxy for each selenium instance, the same IP is always returned. The same is happening if i use RESIDENTIAL proxy group. Am i missing something?
magic-amber
magic-amber•2w ago
Hi @huey louie dewey, to get various proxy URLs you will have to specify sessionID parameter. You can just use something like 
for i in range(10):
  proxy = await proxy_config.new_url(str(i))
for i in range(10):
  proxy = await proxy_config.new_url(str(i))
stormy-gold
stormy-goldOP•2w ago
Hi @Matous . Thanks for your suggestion. Trying your proposed method without success. Here is the log messages after implementing your correction above: 
Actor is running on the Apify platform, `disable_browser_sandbox` was changed to True.
INFO  Initializing Actor...
INFO  System info ({"apify_sdk_version": "2.5.0", "apify_client_version": "1.9.3", "crawlee_version": "0.6.7", "python_version": "3.13.3", "os": "linux"})
INFO  Using proxy: http://groups-RESIDENTIAL,session-0,country-GR:*********@10.0.93.207:8011
INFO  IP = 44.197.114.92
INFO  Using proxy: http://groups-RESIDENTIAL,session-1,country-GR:*********@10.0.93.207:8011
INFO  IP = 44.197.114.92
INFO  Using proxy: http://groups-RESIDENTIAL,session-2,country-GR:*********@10.0.93.207:8011
INFO  IP = 44.197.114.92
INFO  Using proxy: http://groups-RESIDENTIAL,session-3,country-GR:*********@10.0.93.207:8011
INFO  IP = 44.197.114.92
INFO  Using proxy: http://groups-RESIDENTIAL,session-4,country-GR:*********@10.0.93.207:8011
...
same for the rest 5 results
...
Actor is running on the Apify platform, `disable_browser_sandbox` was changed to True.
INFO  Initializing Actor...
INFO  System info ({"apify_sdk_version": "2.5.0", "apify_client_version": "1.9.3", "crawlee_version": "0.6.7", "python_version": "3.13.3", "os": "linux"})
INFO  Using proxy: http://groups-RESIDENTIAL,session-0,country-GR:*********@10.0.93.207:8011
INFO  IP = 44.197.114.92
INFO  Using proxy: http://groups-RESIDENTIAL,session-1,country-GR:*********@10.0.93.207:8011
INFO  IP = 44.197.114.92
INFO  Using proxy: http://groups-RESIDENTIAL,session-2,country-GR:*********@10.0.93.207:8011
INFO  IP = 44.197.114.92
INFO  Using proxy: http://groups-RESIDENTIAL,session-3,country-GR:*********@10.0.93.207:8011
INFO  IP = 44.197.114.92
INFO  Using proxy: http://groups-RESIDENTIAL,session-4,country-GR:*********@10.0.93.207:8011
...
same for the rest 5 results
...
In the above log i was using RESIDENTIAL type. Though the same result occur if i use DATACENTER type. I am a free user if that matters (but i don't think it matters here - In the free plan I should have access to 5 DATACENTER proxies and the RESIDENTIAL proxies have their cost on my monthly balance). Could this be an API issue? I could open an issue in github if needed.
magic-amber
magic-amber•2w ago
Hey, thanks a lot for pointing this out. I’ve reached out to the team, and I’ll keep you in the loop once I hear back. Hi @huey louie dewey, so I got a response:
There is no need to do the new_url in a loop, it always gives you the same URL (you just can pass session for formatting but no session == random session). I think Selenium couldn't support authenticated proxies so maybe they are not using proxies at all. Can they try Playwright? That should work if they spawn a new browser in a loop
And I also tested the new_url this way: 
# Installed libraries
import httpx
from apify import Actor

async def main() -> None:
    async with Actor:
        proxy_config = await Actor.create_proxy_configuration()
        url = "https://api.ipify.org?format=json"

        # note this is outside from the for-loop
        proxy = await proxy_config.new_url()

        for _ in range(10):
            Actor.log.info(f'Using proxy: {proxy}')
            async with httpx.AsyncClient(proxy=proxy) as client:
                response = await client.get(url)
                content = response.json()
                ip = content.get("ip")
                Actor.log.info(f"IP = {ip}")
# Installed libraries
import httpx
from apify import Actor

async def main() -> None:
    async with Actor:
        proxy_config = await Actor.create_proxy_configuration()
        url = "https://api.ipify.org?format=json"

        # note this is outside from the for-loop
        proxy = await proxy_config.new_url()

        for _ in range(10):
            Actor.log.info(f'Using proxy: {proxy}')
            async with httpx.AsyncClient(proxy=proxy) as client:
                response = await client.get(url)
                content = response.json()
                ip = content.get("ip")
                Actor.log.info(f"IP = {ip}")

Did you find this page helpful?