|
@@ -735,6 +735,7 @@
|
|
|
userDataDir = os.tmpdir() + separator + 'chrome-data-capture-jd';
|
|
|
}
|
|
|
|
|
|
+ puppeteer.use(StealthPlugin());
|
|
|
this.loginBrowser = await puppeteer.launch({
|
|
|
headless: false,
|
|
|
executablePath: this.initPath(),
|
|
@@ -1144,7 +1145,16 @@
|
|
|
|
|
|
let waitUntil = 'networkidle2';
|
|
|
waitUntil = this.initDevelop().waitUntil;
|
|
|
- await page.goto(urlInfo.url, {waitUntil : waitUntil});
|
|
|
+
|
|
|
+ let goUrl = urlInfo.url;
|
|
|
+ if(this.menuIndex == '1' && goUrl.indexOf('version=0') == -1){ // 阿里巴巴
|
|
|
+ if(goUrl.indexOf('?') > -1){
|
|
|
+ goUrl += "&version=0";
|
|
|
+ }else{
|
|
|
+ goUrl += "?verison=0";
|
|
|
+ }
|
|
|
+ }
|
|
|
+ await page.goto(goUrl, {waitUntil : waitUntil});
|
|
|
|
|
|
if(urlInfo.title){
|
|
|
if (fs.existsSync(this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title)) {
|
|
@@ -1645,17 +1655,6 @@
|
|
|
return {'scrollHeight': scrollHeight, 'cHeight': cHeight}
|
|
|
});
|
|
|
|
|
|
- if(urlInfo.title){
|
|
|
- if (fs.existsSync(this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title)) {
|
|
|
- urlInfo.newPath = this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title;
|
|
|
- } else {
|
|
|
- fs.mkdirSync(this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title);
|
|
|
- urlInfo.newPath = this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title;
|
|
|
- }
|
|
|
- }else{
|
|
|
- await this.getTitle(page, urlInfo); // 生成页面标题对应的文件夹
|
|
|
- }
|
|
|
-
|
|
|
let scrollHeight = pageInfo.scrollHeight;
|
|
|
let cHeight = pageInfo.cHeight;
|
|
|
let num = Math.ceil(scrollHeight / cHeight);
|
|
@@ -1733,6 +1732,17 @@
|
|
|
urlInfo.status = '3';
|
|
|
clearInterval(scrollInt);
|
|
|
|
|
|
+ if(urlInfo.title){
|
|
|
+ if (fs.existsSync(this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title)) {
|
|
|
+ urlInfo.newPath = this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title;
|
|
|
+ } else {
|
|
|
+ fs.mkdirSync(this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title);
|
|
|
+ urlInfo.newPath = this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title;
|
|
|
+ }
|
|
|
+ }else{
|
|
|
+ await this.getTitle(page, urlInfo); // 生成页面标题对应的文件夹
|
|
|
+ }
|
|
|
+
|
|
|
//detailImg:详情图;skuImg:sku图片;commentImg: 评论图;video: 视频
|
|
|
const imgInfo = await page.evaluate((authority, execNum) => {
|
|
|
let outObj = {
|
|
@@ -1810,7 +1820,7 @@
|
|
|
arr4 = document.querySelectorAll('div[class^=comments--] img');
|
|
|
}
|
|
|
for(let i=0; i< arr4.length; i++){
|
|
|
- if(arr4[i].src.indexOf('/avatar/sns/user/flag/sns_logo') == -1){ //过滤淘宝用户头像
|
|
|
+ if(arr4[i].src.indexOf('/avatar/sns/user/flag/sns_logo') == -1 && arr4[i].className.indexOf('creditImg') == -1){ //过滤淘宝用户头像
|
|
|
if(!authority && i < execNum){
|
|
|
outObj.commentImg.push(arr4[i].src);
|
|
|
}
|
|
@@ -1833,7 +1843,7 @@
|
|
|
}
|
|
|
return outObj;
|
|
|
}, authority, this.execNum);
|
|
|
-
|
|
|
+
|
|
|
if(responseVideo.length > 0){
|
|
|
for(let l=0; l<responseVideo.length; l++){
|
|
|
if(imgInfo.video.indexOf(responseVideo[l]) == -1){
|