英文:
Failed to produce a 200 status after issuing a get requests with parameters
问题
I'm trying to send GET HTTP requests to a webpage and receive a 200 status in response. It is necessary to send appropriate parameters with the link while issuing requests. However, when I send a requests using the link
and payload
, I always receive a 410 status code.
When I use the URL copied from the dev tools directly in the following code, I get a 200 status.
Please note that I've modified the values of AnchorProdSetId
and ProdSetId
before sending payload with the link.
How can I get a 200 (403 when detected) status after issuing requests using the link with payload?
import json
import requests
complete_url_from_d_tools = 'https://www.mcmaster.com/mv1687879695/WebParts/Content/ContentWebPart.aspx?cntnrIDtxt=ProdPageContent&srchidtxt=5875341805879&cntnrwdth=769&srchrslttxt=screws&GrpUsrInps=[{%22AnchorProdSetId%22:%22-2674186034712253%22,%22AnchorStateIsSet%22:true,%22ClearAllProdSetId%22:-1,%22ClearAllStateIsSet%22:true,%22GrpEID%22:%22%22,%22ProdSetId%22:%22-2674186034712253%22}]&viewporthgt=192&cssAlias=undefined&usedynamicimageswapping=true&conditionalclientlandingpage=true&forcemdmfiltering=true&workstationdata=true&fwdSsToNav=true&fwdSsWpNav=true&fwdCrcWpNav=true&sortRowsOnServer=true&showbroadlandingpagecopy=true&filtSsAttrEarly=true&legImgContainsHash=true&envrmgrcharsetind=3&useEs6=true'
link = 'https://www.mcmaster.com/mv1687538569/WebParts/Content/ContentWebPart.aspx'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en-US,en;q=0.9,bn;q=0.8',
'Referer': 'https://www.mcmaster.com/products/screws/',
'X-Requested-With': 'XMLHttpRequest',
}
payload = {
'cntnrIDtxt': 'ProdPageContent',
'srchidtxt': '5875341805879',
'cntnrwdth': '769',
'srchrslttxt': 'screws',
'GrpUsrInps': '[{"AnchorProdSetId":"","AnchorStateIsSet":"true","ClearAllProdSetId":-1,"ClearAllStateIsSet":"true","GrpEID":"","ProdSetId":""}]',
'viewporthgt': '192',
'cssAlias': 'undefined',
'usedynamicimageswapping': 'true',
'conditionalclientlandingpage': 'true',
'forcemdmfiltering': 'true',
'workstationdata': 'true',
'fwdSsToNav': 'true',
'fwdSsWpNav': 'true',
'fwdCrcWpNav': 'true',
'sortRowsOnServer': 'true',
'showbroadlandingpagecopy': 'true',
'filtSsAttrEarly': 'true',
'legImgContainsHash': 'true',
'envrmgrcharsetind': '3',
'useEs6': 'true',
}
payload['GrpUsrInps'] = json.loads(payload['GrpUsrInps'])
payload['GrpUsrInps'][0]['AnchorProdSetId'] = '-2674186034712253'
payload['GrpUsrInps'][0]['ProdSetId'] = '-2674186034712253'
payload['GrpUsrInps'] = json.dumps(payload['GrpUsrInps'])
with requests.Session() as s:
s.headers.update(headers)
# res = s.get(complete_url_from_d_tools)
res = s.get(link, params=payload)
print(res.status_code)
(Note: The code provided is in English. If you need a translation into Chinese, please let me know.)
英文:
I'm trying to send GET HTTP requests to a webpage and receive a 200 status in response. It is necessary to send appropriate parameters with the link while issuing requests. However, when I send a requests using the link
and payload
, I always receive a 410 status code.
When I use the URL copied from the dev tools directly in the following code, I get a 200 status.
Please note that I've modified the values of AnchorProdSetId
and ProdSetId
before sending payload with the link.
How can I get a 200 (403 when detected) status after issuing requests using the link with payload?
import json
import requests
complete_url_from_d_tools = 'https://www.mcmaster.com/mv1687879695/WebParts/Content/ContentWebPart.aspx?cntnrIDtxt=ProdPageContent&srchidtxt=5875341805879&cntnrwdth=769&srchrslttxt=screws&GrpUsrInps=[{%22AnchorProdSetId%22:%22-2674186034712253%22,%22AnchorStateIsSet%22:true,%22ClearAllProdSetId%22:-1,%22ClearAllStateIsSet%22:true,%22GrpEID%22:%22%22,%22ProdSetId%22:%22-2674186034712253%22}]&viewporthgt=192&cssAlias=undefined&usedynamicimageswapping=true&conditionalclientlandingpage=true&forcemdmfiltering=true&workstationdata=true&fwdSsToNav=true&fwdSsWpNav=true&fwdCrcWpNav=true&sortRowsOnServer=true&showbroadlandingpagecopy=true&filtSsAttrEarly=true&legImgContainsHash=true&envrmgrcharsetind=3&useEs6=true'
link = 'https://www.mcmaster.com/mv1687538569/WebParts/Content/ContentWebPart.aspx'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en-US,en;q=0.9,bn;q=0.8',
'Referer': 'https://www.mcmaster.com/products/screws/',
'X-Requested-With': 'XMLHttpRequest',
}
payload = {
'cntnrIDtxt': 'ProdPageContent',
'srchidtxt': '5875341805879',
'cntnrwdth': '769',
'srchrslttxt': 'screws',
'GrpUsrInps': '[{"AnchorProdSetId":"","AnchorStateIsSet":"true","ClearAllProdSetId":-1,"ClearAllStateIsSet":"true","GrpEID":"","ProdSetId":""}]',
'viewporthgt': '192',
'cssAlias': 'undefined',
'usedynamicimageswapping': 'true',
'conditionalclientlandingpage': 'true',
'forcemdmfiltering': 'true',
'workstationdata': 'true',
'fwdSsToNav': 'true',
'fwdSsWpNav': 'true',
'fwdCrcWpNav': 'true',
'sortRowsOnServer': 'true',
'showbroadlandingpagecopy': 'true',
'filtSsAttrEarly': 'true',
'legImgContainsHash': 'true',
'envrmgrcharsetind': '3',
'useEs6': 'true',
}
payload['GrpUsrInps'] = json.loads(payload['GrpUsrInps'])
payload['GrpUsrInps'][0]['AnchorProdSetId'] = '-2674186034712253'
payload['GrpUsrInps'][0]['ProdSetId'] = '-2674186034712253'
payload['GrpUsrInps'] = json.dumps(payload['GrpUsrInps'])
with requests.Session() as s:
s.headers.update(headers)
# res = s.get(complete_url_from_d_tools)
res = s.get(link,params=payload)
print(res.status_code)
答案1
得分: 1
看起来这个服务器返回的是brotli压缩响应。要从服务器获取200
响应,请安装brotli
包:
pip install brotli
然后:
import requests
url = "https://www.mcmaster.com/mv1687879695/WebParts/Content/ContentWebPart.aspx?cntnrIDtxt=ProdPageContent&srchidtxt=5875341805879&cntnrwdth=1001&srchrslttxt=screws&GrpUsrInps=%5B%7B%22AnchorProdSetId%22%3A%22-3383254440195390%22%2C%22AnchorStateIsSet%22%3Atrue%2C%22ClearAllProdSetId%22%3A-1%2C%22ClearAllStateIsSet%22%3Atrue%2C%22GrpEID%22%3A%22%22%2C%22ProdSetId%22%3A%22-3383254440195390%22%7D%5D&viewporthgt=296&cssAlias=undefined&usedynamicimageswapping=true&conditionalclientlandingpage=true&forcemdmfiltering=true&workstationdata=true&fwdSsToNav=true&fwdSsWpNav=true&fwdCrcWpNav=true&sortRowsOnServer=true&showbroadlandingpagecopy=true&filtSsAttrEarly=true&legImgContainsHash=true&clientNavigationEvents=%5B%5D&envrmgrcharsetind=5&useEs6=true"
r = requests.get(url)
print(r.status_code)
打印:
200
编辑:要获取“Socket Head Screws”的内容:
import requests
AnchorProdSetId = '-3383254440195390'
ProdSetId = '-3383254440195390'
params = {
"cntnrIDtxt": "ProdPageContent",
"srchidtxt": "5875341805879",
"cntnrwdth": "1001",
"srchrslttxt": "screws",
"GrpUsrInps": '[{"AnchorProdSetId":"' + AnchorProdSetId + '","AnchorStateIsSet":true,"ClearAllProdSetId":-1,"ClearAllStateIsSet":true,"GrpEID":"","ProdSetId":"' + ProdSetId + '"}]',
"viewporthgt": "296",
"cssAlias": "undefined",
"usedynamicimageswapping": "true",
"conditionalclientlandingpage": "true",
"forcemdmfiltering": "true",
"workstationdata": "true",
"fwdSsToNav": "true",
"fwdSsWpNav": "true",
"fwdCrcWpNav": "true",
"sortRowsOnServer": "true",
"showbroadlandingpagecopy": "true",
"filtSsAttrEarly": "true",
"legImgContainsHash": "true",
"clientNavigationEvents": "[]",
"envrmgrcharsetind": "5",
"useEs6": "true",
}
url = 'https://www.mcmaster.com/mv1687879695/WebParts/Content/ContentWebPart.aspx'
r = requests.get(url, params=params)
print(r.text)
英文:
It seems that this server returns brotli compressed response. To get 200
response from the server, install brotli
package:
pip install brotli
Then:
import requests
url = "https://www.mcmaster.com/mv1687879695/WebParts/Content/ContentWebPart.aspx?cntnrIDtxt=ProdPageContent&srchidtxt=5875341805879&cntnrwdth=1001&srchrslttxt=screws&GrpUsrInps=%5B%7B%22AnchorProdSetId%22%3A%22-3383254440195390%22%2C%22AnchorStateIsSet%22%3Atrue%2C%22ClearAllProdSetId%22%3A-1%2C%22ClearAllStateIsSet%22%3Atrue%2C%22GrpEID%22%3A%22%22%2C%22ProdSetId%22%3A%22-3383254440195390%22%7D%5D&viewporthgt=296&cssAlias=undefined&usedynamicimageswapping=true&conditionalclientlandingpage=true&forcemdmfiltering=true&workstationdata=true&fwdSsToNav=true&fwdSsWpNav=true&fwdCrcWpNav=true&sortRowsOnServer=true&showbroadlandingpagecopy=true&filtSsAttrEarly=true&legImgContainsHash=true&clientNavigationEvents=%5B%5D&envrmgrcharsetind=5&useEs6=true"
r = requests.get(url)
print(r.status_code)
Prints:
200
EDIT: To get content for "Socket Head Screws":
import requests
AnchorProdSetId = '-3383254440195390'
ProdSetId = '-3383254440195390'
params = {
"cntnrIDtxt": "ProdPageContent",
"srchidtxt": "5875341805879",
"cntnrwdth": "1001",
"srchrslttxt": "screws",
"GrpUsrInps": '[{"AnchorProdSetId":"' + AnchorProdSetId + '","AnchorStateIsSet":true,"ClearAllProdSetId":-1,"ClearAllStateIsSet":true,"GrpEID":"","ProdSetId":"' + ProdSetId + '"}]',
"viewporthgt": "296",
"cssAlias": "undefined",
"usedynamicimageswapping": "true",
"conditionalclientlandingpage": "true",
"forcemdmfiltering": "true",
"workstationdata": "true",
"fwdSsToNav": "true",
"fwdSsWpNav": "true",
"fwdCrcWpNav": "true",
"sortRowsOnServer": "true",
"showbroadlandingpagecopy": "true",
"filtSsAttrEarly": "true",
"legImgContainsHash": "true",
"clientNavigationEvents": "[]",
"envrmgrcharsetind": "5",
"useEs6": "true",
}
url = 'https://www.mcmaster.com/mv1687879695/WebParts/Content/ContentWebPart.aspx'
r = requests.get(url, params=params)
print(r.text)
通过集体智慧和协作来改善编程学习和解决问题的方式。致力于成为全球开发者共同参与的知识库,让每个人都能够通过互相帮助和分享经验来进步。
评论