C#爬虫基础之HttpClient获取HTTP请求与响应
一、概述
Net4.5以上的提供基本类,用于发送 HTTP 请求和接收来自通过 URI 确认的资源的 HTTP 响应。
HttpClient是一个高级 API,用于包装其运行的每个平台上可用的较低级别功能。
// HttpClient is intended to be instantiated once per application, rather than per-use. See Remarks. static readonly HttpClient client = new HttpClient(); static async Task Main() { // Call asynchronous network methods in a try/catch block to handle exceptions. try { HttpResponseMessage response = await client.GetAsync("http://www.contoso.com/"); response.EnsureSuccessStatusCode(); string responseBody = await response.Content.ReadAsStringAsync(); // Above three lines can be replaced with new helper method below // string responseBody = await client.GetStringAsync(uri); Console.WriteLine(responseBody); } catch(HttpRequestException e) { Console.WriteLine("\nException Caught!"); Console.WriteLine("Message :{0} ",e.Message); } }
二、HttpClient的使用
1.使用HttpClient调用Oauth的授权接口获取access_token
1)OAuth使用的密码式
2)获取到access_token后才进行下一步
2.带着access_token调用接口
1)hearder上添加bearer方式的access_token
2)调用接口确保成功获取到返回的结果
try { string host = ConfigurationManager.AppSettings["api_host"]; string username = ConfigurationManager.AppSettings["api_username"]; string password = ConfigurationManager.AppSettings["api_password"]; HttpClient httpClient = new HttpClient(); // 设置请求头信息 httpClient.DefaultRequestHeaders.Add("Host", host); httpClient.DefaultRequestHeaders.Add("Method", "Post"); httpClient.DefaultRequestHeaders.Add("KeepAlive", "false"); // HTTP KeepAlive设为false,防止HTTP连接保持 httpClient.DefaultRequestHeaders.Add("UserAgent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11"); //获取token var tokenResponse = httpClient.PostAsync("http://" + host + "/token", new FormUrlEncodedContent(new Dictionary<string, string> { {"grant_type","password"}, {"username", username}, {"password", password} })); tokenResponse.Wait(); tokenResponse.Result.EnsureSuccessStatusCode(); var tokenRes = tokenResponse.Result.Content.ReadAsStringAsync(); tokenRes.Wait(); var token = Newtonsoft.Json.Linq.JObject.Parse(tokenRes.Result); var access_token = token["access_token"].ToString(); // 调用接口发起POST请求 var authenticationHeaderValue = new AuthenticationHeaderValue("bearer", access_token); httpClient.DefaultRequestHeaders.Authorization = authenticationHeaderValue; var content = new StringContent(parameter); content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); var response = httpClient.PostAsync("http://" + host + "/" + api_address, content); response.Wait(); response.Result.EnsureSuccessStatusCode(); var res = response.Result.Content.ReadAsStringAsync(); res.Wait();return Newtonsoft.Json.JsonConvert.DeserializeObject(res.Result); } catch (Exception ex) { return ResultEx.Init(ex.Message); }
HttpClient 获取图片并保存到本机
class Program { static void Main() { //图片路径:https://img.infinitynewtab.com/wallpaper/1.jpg string imgSourceURL = "https://img.infinitynewtab.com/wallpaper/"; DownloadImags(imgSourceURL).Wait(); } private static async Task DownloadImags(string url) { var client = new HttpClient(); System.IO.FileStream fs; int a = 1; //文件名:序号+.jpg。可指定范围,以下是获取100.jpg~500.jpg. for (int i = 100; i <= 500; i++) { var uri = new Uri(Uri.EscapeUriString(url+i.ToString()+".jpg")); byte[] urlContents = await client.GetByteArrayAsync(uri); fs = new System.IO.FileStream(AppDomain.CurrentDomain.BaseDirectory + "\\images\\"+ i.ToString() + ".jpg",System.IO.FileMode.CreateNew); fs.Write(urlContents, 0, urlContents.Length); Console.WriteLine(a++); } } }
以下为封装的类库
using Newtonsoft.Json; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Threading.Tasks; using System.Xml.Serialization; public class HttpClientHelpClass { /// /// get请求 /// /// /// public static string GetResponse(string url, out string statusCode) { if (url.StartsWith("https")) System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; var httpClient = new HttpClient(); httpClient.DefaultRequestHeaders.Accept.Add( new MediaTypeWithQualityHeaderValue("application/json")); HttpResponseMessage response = httpClient.GetAsync(url).Result; statusCode = response.StatusCode.ToString(); if (response.IsSuccessStatusCode) { string result = response.Content.ReadAsStringAsync().Result; return result; } return null; } public static string RestfulGet(string url) { HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest; // Get response using (HttpWebResponse response = request.GetResponse() as HttpWebResponse) { // Get the response stream StreamReader reader = new StreamReader(response.GetResponseStream()); // Console application output return reader.ReadToEnd(); } } public static T GetResponse(string url) where T : class, new() { if (url.StartsWith("https")) System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; var httpClient = new HttpClient(); httpClient.DefaultRequestHeaders.Accept.Add( new MediaTypeWithQualityHeaderValue("application/json")); HttpResponseMessage response = httpClient.GetAsync(url).Result; T result = default(T); if (response.IsSuccessStatusCode) { Task<string> t = response.Content.ReadAsStringAsync(); string s = t.Result; result = JsonConvert.DeserializeObject(s); } return result; } /// /// post请求 /// /// /// post数据 /// public static string PostResponse(string url, string postData, out string statusCode) { if (url.StartsWith("https")) System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; HttpContent httpContent = new StringContent(postData); httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); httpContent.Headers.ContentType.CharSet = "utf-8"; HttpClient httpClient = new HttpClient(); //httpClient..setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "utf-8"); HttpResponseMessage response = httpClient.PostAsync(url, httpContent).Result; statusCode = response.StatusCode.ToString(); if (response.IsSuccessStatusCode) { string result = response.Content.ReadAsStringAsync().Result; return result; } return null; } /// /// 发起post请求 /// /// /// url /// post数据 /// public static T PostResponse(string url, string postData) where T : class, new() { if (url.StartsWith("https")) System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; HttpContent httpContent = new StringContent(postData); httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); HttpClient httpClient = new HttpClient(); T result = default(T); HttpResponseMessage response = httpClient.PostAsync(url, httpContent).Result; if (response.IsSuccessStatusCode) { Task<string> t = response.Content.ReadAsStringAsync(); string s = t.Result; result = JsonConvert.DeserializeObject(s); } return result; } /// /// 反序列化Xml /// /// /// /// public static T XmlDeserialize(string xmlString) where T : class, new() { try { XmlSerializer ser = new XmlSerializer(typeof(T)); using (StringReader reader = new StringReader(xmlString)) { return (T)ser.Deserialize(reader); } } catch (Exception ex) { throw new Exception("XmlDeserialize发生异常:xmlString:" + xmlString + "异常信息:" + ex.Message); } } public static string PostResponse(string url, string postData, string token, string appId, string serviceURL, out string statusCode) { if (url.StartsWith("https")) System.Net.ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; HttpContent httpContent = new StringContent(postData); httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/json"); httpContent.Headers.ContentType.CharSet = "utf-8"; httpContent.Headers.Add("token", token); httpContent.Headers.Add("appId", appId); httpContent.Headers.Add("serviceURL", serviceURL); HttpClient httpClient = new HttpClient(); //httpClient..setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "utf-8"); HttpResponseMessage response = httpClient.PostAsync(url, httpContent).Result; statusCode = response.StatusCode.ToString(); if (response.IsSuccessStatusCode) { string result = response.Content.ReadAsStringAsync().Result; return result; } return null; } /// /// 修改API /// /// /// /// public static string KongPatchResponse(string url, string postData) { var httpWebRequest = (HttpWebRequest)WebRequest.Create(url); httpWebRequest.ContentType = "application/x-www-form-urlencoded"; httpWebRequest.Method = "PATCH"; byte[] btBodys = Encoding.UTF8.GetBytes(postData); httpWebRequest.ContentLength = btBodys.Length; httpWebRequest.GetRequestStream().Write(btBodys, 0, btBodys.Length); HttpWebResponse httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse(); var streamReader = new StreamReader(httpWebResponse.GetResponseStream()); string responseContent = streamReader.ReadToEnd(); httpWebResponse.Close(); streamReader.Close(); httpWebRequest.Abort(); httpWebResponse.Close(); return responseContent; } /// /// 创建API /// /// /// /// public static string KongAddResponse(string url, string postData) { if (url.StartsWith("https")) ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; HttpContent httpContent = new StringContent(postData); httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded") { CharSet = "utf-8" }; var httpClient = new HttpClient(); HttpResponseMessage response = httpClient.PostAsync(url, httpContent).Result; if (response.IsSuccessStatusCode) { string result = response.Content.ReadAsStringAsync().Result; return result; } return null; } /// /// 删除API /// /// /// public static bool KongDeleteResponse(string url) { if (url.StartsWith("https")) ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; var httpClient = new HttpClient(); HttpResponseMessage response = httpClient.DeleteAsync(url).Result; return response.IsSuccessStatusCode; } /// /// 修改或者更改API /// /// /// /// public static string KongPutResponse(string url, string postData) { if (url.StartsWith("https")) ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; HttpContent httpContent = new StringContent(postData); httpContent.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded") { CharSet = "utf-8" }; var httpClient = new HttpClient(); HttpResponseMessage response = httpClient.PutAsync(url, httpContent).Result; if (response.IsSuccessStatusCode) { string result = response.Content.ReadAsStringAsync().Result; return result; } return null; } /// /// 检索API /// /// /// public static string KongSerchResponse(string url) { if (url.StartsWith("https")) ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; var httpClient = new HttpClient(); HttpResponseMessage response = httpClient.GetAsync(url).Result; if (response.IsSuccessStatusCode) { string result = response.Content.ReadAsStringAsync().Result; return result; } return null; } }
到此这篇关于C#使用HttpClient获取HTTP请求与响应的文章就介绍到这了。希望对大家的学习有所帮助,也希望大家多多支持我们。
赞 (0)