|
@@ -338,6 +338,8 @@
|
|
import puppeteer from 'puppeteer-extra'
|
|
import puppeteer from 'puppeteer-extra'
|
|
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
|
|
const StealthPlugin = require('puppeteer-extra-plugin-stealth');
|
|
const listNameArr = ['alibaba','jd','tmall','tb','red','aliguoji','acaigou','amazon', '' ,'common'];
|
|
const listNameArr = ['alibaba','jd','tmall','tb','red','aliguoji','acaigou','amazon', '' ,'common'];
|
|
|
|
+ // const headless = true;
|
|
|
|
+ // const waitUntil = 'networkidle2';
|
|
|
|
|
|
let separator = '';
|
|
let separator = '';
|
|
if (os.platform == 'linux') {
|
|
if (os.platform == 'linux') {
|
|
@@ -480,6 +482,11 @@
|
|
this.$utils.setStorage('versionType', 2);
|
|
this.$utils.setStorage('versionType', 2);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
+
|
|
|
|
+ // 初始化开发者设置
|
|
|
|
+ this.$utils.setStorage('headless', 1);
|
|
|
|
+ this.$utils.setStorage('waitUntil', 'networkidle2');
|
|
|
|
+
|
|
},
|
|
},
|
|
methods: {
|
|
methods: {
|
|
// 实时获取浏览器路径
|
|
// 实时获取浏览器路径
|
|
@@ -491,6 +498,36 @@
|
|
}
|
|
}
|
|
return chromePath;
|
|
return chromePath;
|
|
},
|
|
},
|
|
|
|
+ // 实时获取页面滚动加载时间
|
|
|
|
+ initMs(){
|
|
|
|
+ let pageMs = this.$utils.getStorage('pageMs');
|
|
|
|
+ if(pageMs){
|
|
|
|
+ return Number(pageMs);
|
|
|
|
+ }else{
|
|
|
|
+ return 500;
|
|
|
|
+ }
|
|
|
|
+ },
|
|
|
|
+ // 实时获取淘宝的开发者设置
|
|
|
|
+ initDevelop(){
|
|
|
|
+ let develop = {
|
|
|
|
+ headless: true,
|
|
|
|
+ waitUntil: 'networkidle2'
|
|
|
|
+ };
|
|
|
|
+ let n1 = this.$utils.getStorage('headless');
|
|
|
|
+ let n2 = this.$utils.getStorage('waitUntil');
|
|
|
|
+ if(n1){
|
|
|
|
+ if(n1 == 1){
|
|
|
|
+ develop.headless = true;
|
|
|
|
+ }else if(n1 == 2){
|
|
|
|
+ develop.headless = false;
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ if(n2){
|
|
|
|
+ develop.waitUntil = n2;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ return develop;
|
|
|
|
+ },
|
|
// 删除文件夹内容
|
|
// 删除文件夹内容
|
|
deleteAll(folderPath, flag) {
|
|
deleteAll(folderPath, flag) {
|
|
if (fs.existsSync(folderPath)) {
|
|
if (fs.existsSync(folderPath)) {
|
|
@@ -831,14 +868,17 @@
|
|
browserName = 'tbBrowser';
|
|
browserName = 'tbBrowser';
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+ let headless = true;
|
|
|
|
+ headless = this.initDevelop().headless;
|
|
this[browserName] = await puppeteer.launch({
|
|
this[browserName] = await puppeteer.launch({
|
|
|
|
+ headless: headless,
|
|
executablePath: this.initPath(),
|
|
executablePath: this.initPath(),
|
|
userDataDir: userDataDir,
|
|
userDataDir: userDataDir,
|
|
args: [
|
|
args: [
|
|
'--start-maximized',
|
|
'--start-maximized',
|
|
'--no-sandbox',
|
|
'--no-sandbox',
|
|
'--disable-setuid-sandbox',
|
|
'--disable-setuid-sandbox',
|
|
- '--disable-blink-features=AutomationControlled',
|
|
|
|
|
|
+ '--disable-blink-features=AutomationControlled'
|
|
]
|
|
]
|
|
});
|
|
});
|
|
|
|
|
|
@@ -1014,6 +1054,7 @@
|
|
|
|
|
|
let num = Math.ceil(scrollHeight / cHeight);
|
|
let num = Math.ceil(scrollHeight / cHeight);
|
|
let start = -1;
|
|
let start = -1;
|
|
|
|
+ let scrollTime = this.initMs();
|
|
let scrollInt = setInterval(async() => {
|
|
let scrollInt = setInterval(async() => {
|
|
start ++;
|
|
start ++;
|
|
await page.evaluate((start) => {
|
|
await page.evaluate((start) => {
|
|
@@ -1030,7 +1071,7 @@
|
|
resolve(true);
|
|
resolve(true);
|
|
this.loading = false;
|
|
this.loading = false;
|
|
}
|
|
}
|
|
- }, 300);
|
|
|
|
|
|
+ }, scrollTime);
|
|
|
|
|
|
}catch(e){
|
|
}catch(e){
|
|
urlInfo.status = '5';
|
|
urlInfo.status = '5';
|
|
@@ -1100,7 +1141,10 @@
|
|
// userDataDir: os.tmpdir() + separator + 'chrome-data-capture',
|
|
// userDataDir: os.tmpdir() + separator + 'chrome-data-capture',
|
|
// });
|
|
// });
|
|
const page = await browser.newPage();
|
|
const page = await browser.newPage();
|
|
- await page.goto(urlInfo.url, {waitUntil : 'networkidle2'});
|
|
|
|
|
|
+
|
|
|
|
+ let waitUntil = 'networkidle2';
|
|
|
|
+ waitUntil = this.initDevelop().waitUntil;
|
|
|
|
+ await page.goto(urlInfo.url, {waitUntil : waitUntil});
|
|
|
|
|
|
if(urlInfo.title){
|
|
if(urlInfo.title){
|
|
if (fs.existsSync(this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title)) {
|
|
if (fs.existsSync(this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title)) {
|
|
@@ -1124,6 +1168,10 @@
|
|
|
|
|
|
let num = Math.ceil(scrollHeight / cHeight);
|
|
let num = Math.ceil(scrollHeight / cHeight);
|
|
let start = -1;
|
|
let start = -1;
|
|
|
|
+ let scrollTime = this.initMs();
|
|
|
|
+ if(scrollTime < 800 && this.menuIndex == '6'){
|
|
|
|
+ scrollTime = 800;
|
|
|
|
+ }
|
|
let scrollInt = setInterval(async() => {
|
|
let scrollInt = setInterval(async() => {
|
|
start ++;
|
|
start ++;
|
|
if(this.settingArr.indexOf('detailImg') > -1){
|
|
if(this.settingArr.indexOf('detailImg') > -1){
|
|
@@ -1156,7 +1204,7 @@
|
|
urlInfo.status = '4';
|
|
urlInfo.status = '4';
|
|
resolve(true);
|
|
resolve(true);
|
|
}
|
|
}
|
|
- }, 500);
|
|
|
|
|
|
+ }, scrollTime);
|
|
|
|
|
|
}catch(e){
|
|
}catch(e){
|
|
urlInfo.status = '5';
|
|
urlInfo.status = '5';
|
|
@@ -1560,19 +1608,9 @@
|
|
}
|
|
}
|
|
}
|
|
}
|
|
});
|
|
});
|
|
-
|
|
|
|
- await page.goto(urlInfo.url, {waitUntil : 'networkidle0'});
|
|
|
|
-
|
|
|
|
- if(urlInfo.title){
|
|
|
|
- if (fs.existsSync(this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title)) {
|
|
|
|
- urlInfo.newPath = this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title;
|
|
|
|
- } else {
|
|
|
|
- fs.mkdirSync(this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title);
|
|
|
|
- urlInfo.newPath = this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title;
|
|
|
|
- }
|
|
|
|
- }else{
|
|
|
|
- await this.getTitle(page, urlInfo); // 生成页面标题对应的文件夹
|
|
|
|
- }
|
|
|
|
|
|
+ let waitUntil = 'networkidle2';
|
|
|
|
+ waitUntil = this.initDevelop().waitUntil;
|
|
|
|
+ await page.goto(urlInfo.url, {waitUntil : waitUntil});
|
|
|
|
|
|
if(this.settingArr.indexOf('video') > -1){ // 用户选择下载视频的时候才会触发
|
|
if(this.settingArr.indexOf('video') > -1){ // 用户选择下载视频的时候才会触发
|
|
//鼠标放在主图第一张,生成视频
|
|
//鼠标放在主图第一张,生成视频
|
|
@@ -1606,10 +1644,23 @@
|
|
let scrollHeight = document.body.scrollHeight;
|
|
let scrollHeight = document.body.scrollHeight;
|
|
return {'scrollHeight': scrollHeight, 'cHeight': cHeight}
|
|
return {'scrollHeight': scrollHeight, 'cHeight': cHeight}
|
|
});
|
|
});
|
|
|
|
+
|
|
|
|
+ if(urlInfo.title){
|
|
|
|
+ if (fs.existsSync(this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title)) {
|
|
|
|
+ urlInfo.newPath = this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title;
|
|
|
|
+ } else {
|
|
|
|
+ fs.mkdirSync(this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title);
|
|
|
|
+ urlInfo.newPath = this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title;
|
|
|
|
+ }
|
|
|
|
+ }else{
|
|
|
|
+ await this.getTitle(page, urlInfo); // 生成页面标题对应的文件夹
|
|
|
|
+ }
|
|
|
|
+
|
|
let scrollHeight = pageInfo.scrollHeight;
|
|
let scrollHeight = pageInfo.scrollHeight;
|
|
let cHeight = pageInfo.cHeight;
|
|
let cHeight = pageInfo.cHeight;
|
|
let num = Math.ceil(scrollHeight / cHeight);
|
|
let num = Math.ceil(scrollHeight / cHeight);
|
|
let start = -1;
|
|
let start = -1;
|
|
|
|
+ let scrollTime = this.initMs();
|
|
let scrollInt = setInterval(async() => {
|
|
let scrollInt = setInterval(async() => {
|
|
start ++;
|
|
start ++;
|
|
if(this.settingArr.indexOf('commentImg') > -1 && this.settingArr.indexOf('detailImg') < 0){ // 选择了评论图没选择详情图
|
|
if(this.settingArr.indexOf('commentImg') > -1 && this.settingArr.indexOf('detailImg') < 0){ // 选择了评论图没选择详情图
|
|
@@ -1783,8 +1834,12 @@
|
|
return outObj;
|
|
return outObj;
|
|
}, authority, this.execNum);
|
|
}, authority, this.execNum);
|
|
|
|
|
|
- if(imgInfo.video.length == 0 && responseVideo.length > 0){
|
|
|
|
- imgInfo.video = responseVideo;
|
|
|
|
|
|
+ if(responseVideo.length > 0){
|
|
|
|
+ for(let l=0; l<responseVideo.length; l++){
|
|
|
|
+ if(imgInfo.video.indexOf(responseVideo[l]) == -1){
|
|
|
|
+ imgInfo.video.push(responseVideo[l]);
|
|
|
|
+ }
|
|
|
|
+ }
|
|
}
|
|
}
|
|
|
|
|
|
if(this.settingArr.indexOf('mainImg') > -1){
|
|
if(this.settingArr.indexOf('mainImg') > -1){
|
|
@@ -1916,7 +1971,7 @@
|
|
resolve(true);
|
|
resolve(true);
|
|
this.loading = false;
|
|
this.loading = false;
|
|
}
|
|
}
|
|
- }, 500);
|
|
|
|
|
|
+ }, scrollTime);
|
|
}catch(e){
|
|
}catch(e){
|
|
reject(e);
|
|
reject(e);
|
|
this.showError(e);
|
|
this.showError(e);
|
|
@@ -1988,7 +2043,7 @@
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// 视频
|
|
// 视频
|
|
- let arr5 = document.querySelectorAll('video.lib-video');
|
|
|
|
|
|
+ let arr5 = document.querySelectorAll('video');
|
|
for(let i=0; i< arr5.length; i++){
|
|
for(let i=0; i< arr5.length; i++){
|
|
if(outObj.video.indexOf(arr5[i].src) == -1){
|
|
if(outObj.video.indexOf(arr5[i].src) == -1){
|
|
if(!authority && i < execNum){
|
|
if(!authority && i < execNum){
|