c# HttpClient System. Net. WebException: Unable to connect to remote server
I'm trying to check a large list of domains for validity. I'm trying to do this in multithreaded mode. The robots principle: I send a GET request to the site, then check its HTML code. Code:
httpClient loadingPage = new httpClient();
private async void Button_Click_1Async(object sender, RoutedEventArgs e)
{
int j = 0;
IProgress<string> progress = new Progress<string>(p => AllDomains.Content = Convert.ToString(j++));
IProgress<string> progress1 = new Progress<string>(p => good.Text += p + "\n");
IProgress<string> progress2 = new Progress<string>(p => bad.Text += p + "\n");
IProgress<string> progress3 = new Progress<string>(p => Catch.Text += p + "\n");
string[] arrString = File.ReadAllLines(@"C:\Users\Desktop\domain.txt");
int arrStringCountForThread = (arrString.Length / 20);
int maxConcurrency = 30; // сколько одновременно может работать потоков
using (SemaphoreSlim semaphore = new SemaphoreSlim(maxConcurrency))
{
List<Task> tasks = new List<Task>();
for (int i = 1; i < 20; i++)
{
try
{
string[] NEWarrString = new string[arrStringCountForThread];
//Разделяем arrString на 20 массивов равного размера
// каждый tasks получает 1/20 элементов массива
Array.Copy(arrString, (i * arrStringCountForThread), NEWarrString, 0, arrStringCountForThread);
tasks.Add(loadingPage.ProcessPage(NEWarrString, progress, progress1, progress2, progress3, semaphore)); // запуск I/O-bound операции
}
finally
{
}
}
await Task.WhenAll(tasks); // ждать завершения всех
}
}
HttpClient.cs
searshCMS searsh = new searshCMS();
HttpClient client = new HttpClient();
public async Task ProcessPage(string[] url, IProgress<string> progress, IProgress<string> progress1, IProgress<string> progress2, IProgress<string> progress3, SemaphoreSlim semaphore)
{ string urllink = null;
// MessageBox.Show(Convert.ToString(url.Length));
for (int i = 0; i < url.Length - 1; i++)
{urllink = url[i];
try
{
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls;
string html = await client.GetStringAsync(url[i]);
searsh.SearchCMS(url[i], html, progress, progress1, progress2, progress3);
}
catch (Exception e)
{
progress3.Report(Convert.ToString(e));
}
}
}
In searsh.SearchCMS (url[i], html, progress, progress1, progress2, progress3); the HTML of the page is passed for further processing.
A lot of things appear in the ProcessPage method exceptions
catch (Exception e)
{
progress3.Report(Convert.ToString(e));
}
For example:
Https://hairline.hu System.Net.Http.HttpRequestException: An error occurred while sending the request. ---> System. Net. WebException: Unable to connect to remote server - - - > System. Net.Sockets.SocketException: The attempt to establish a connection was unsuccessful, because the desired response was not received from another computer in the required time, or the already established connection was terminated due to an incorrect response from an already connected computer 79.172.205.80:443 in System. Net.Sockets.Socket. InternalEndConnect (IAsyncResult AsyncResult) in System.Net.Sockets.Socket.EndConnect (IAsyncResult AsyncResult) in System. Net. ServicePoint. ConnectSocketInternal(Boolean connectFailure, Socket s4, Socket s6, Socket& socket, IPAddress& address, ConnectSocketState state, IAsyncResult AsyncResult, Exception& exception) --- End of the internal exception stack trace - - - in System. Net. HttpWebRequest.EndGetResponse (IAsyncResult AsyncResult) in System.Net.Http.HttpClientHandler.GetResponseCallback(IAsyncResult ar) - - - End of internal exception stack trace - - - in System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess (Task task) in System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification (Task task) in System.Runtime.CompilerServices.TaskAwaiter`1.GetResult()
At the same time, in the browser https://hairline.hu is loading. About 50% of all domains fall into the exception, most of them are working
1 answers
You did something strange with the code. I recognize the comments, I think I wrote them not so long ago.
Here's how to try
private async void Button_Click_1Async(object sender, RoutedEventArgs e)
{
IProgress<string> progress = new Progress<string>(p => AllDomains.Content = Convert.ToString(j++));
IProgress<string> progress1 = new Progress<string>(p => good.Text += p + "\n");
IProgress<string> progress2 = new Progress<string>(p => bad.Text += p + "\n");
IProgress<string> progress3 = new Progress<string>(p => Catch.Text += p + "\n");
string[] arrString = File.ReadAllLines(@"C:\Users\Desktop\domain.txt");
int maxConcurrency = 30; // сколько одновременно может работать потоков
using (SemaphoreSlim semaphore = new SemaphoreSlim(maxConcurrency))
{
List<Task> tasks = new List<Task>();
foreach (string s in arrString)
{
await semaphore.WaitAsync();
tasks.Add(loadingPage.ProcessPage(s, progress, progress1, progress2, progress3, semaphore)); // запуск I/O-bound операции
}
await Task.WhenAll(tasks); // ждать завершения всех
}
}
public async Task ProcessPage(string url, IProgress<string> progress, IProgress<string> progress1, IProgress<string> progress2, IProgress<string> progress3, SemaphoreSlim semaphore)
{
try
{
string html;
using (HttpResponseMessage response = await client.GetAsync(url, HttpCompleteionOption.ResponseHeadersRead).ConfigureAwait(false))
{
response.EnsureSuccessStatusCode();
html = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
searsh.SearchCMS(url, html, progress, progress1, progress2, progress3);
}
catch (Exception ex)
{
progress3.Report(ex.ToString());
}
finally
{
semaphore.Release();
}
}
Now 30 simultaneous requests will work.
I checked twice, the address you specified gives the page normally. I checked it with this code in the console application. Most likely the problem is in your Windows.
public class Program
{
private static HttpClient client = new HttpClient();
public static async Task Main()
{
string html = await client.GetStringAsync("https://hairline.hu");
Console.WriteLine(html);
}
}
A large HTML file is output to the console.