|
@@ -71,7 +71,7 @@
|
|
|
<div>
|
|
|
<el-button-group>
|
|
|
<el-button type="primary" size="mini" icon="el-icon-circle-plus-outline"
|
|
|
- @click="addVisible = true;">添加链接</el-button>
|
|
|
+ @click="addVisible = true; add();">添加链接</el-button>
|
|
|
<el-button type="primary" size="mini" icon="el-icon-upload"
|
|
|
@click="pickLink()">导入链接</el-button>
|
|
|
<el-button type="primary" size="mini" icon="el-icon-delete"
|
|
@@ -253,11 +253,10 @@
|
|
|
<div>
|
|
|
<el-form label-position="right" label-width="80px" :rules="rules" :model="formData" ref="formData">
|
|
|
<el-form-item label="目录名称" prop="title">
|
|
|
- <el-input v-model="formData.title" placeholder="为空则默认使用网页标题前50个字符"></el-input>
|
|
|
+ <el-input id="aaa" v-model="formData.title" placeholder="为空则默认使用网页标题前50个字符"></el-input>
|
|
|
</el-form-item>
|
|
|
<el-form-item label="网页链接" prop="url">
|
|
|
- <el-input type="textarea" :rows="10" v-if="menuIndex < 10" :placeholder="'请输入网址链接(例:' + exampleUrl[menuIndex-1] + ')'" v-model="formData.url"></el-input>
|
|
|
- <el-input type="textarea" :rows="10" v-else :placeholder="'请输入网址链接'" v-model="formData.url"></el-input>
|
|
|
+ <el-input id="bbb" type="textarea" :rows="10" :placeholder="'请输入网址链接(例:' + exampleUrl[menuIndex-1] + ')'" v-model="formData.url"></el-input>
|
|
|
</el-form-item>
|
|
|
</el-form>
|
|
|
</div>
|
|
@@ -339,6 +338,7 @@
|
|
|
const listNameArr = ['alibaba','jd','tmall','tb','red','aliguoji','acaigou','amazon', '' ,'common'];
|
|
|
// const headless = true;
|
|
|
// const waitUntil = 'networkidle2';
|
|
|
+ const { ipcRenderer } = require('electron');
|
|
|
|
|
|
let separator = '';
|
|
|
if (os.platform == 'linux') {
|
|
@@ -399,7 +399,8 @@
|
|
|
commentImg: false,
|
|
|
video: false,
|
|
|
},
|
|
|
- exampleUrl: ['https://www.1688.com', 'https://www.jd.com', 'https://www.tmall.com', 'https://www.taobao.com', 'https://www.xiaohongshu.com', 'https://www.alibaba.com'],
|
|
|
+ exampleUrl: ['https://www.1688.com', 'https://www.jd.com', 'https://www.tmall.com', 'https://www.taobao.com', 'https://www.xiaohongshu.com',
|
|
|
+ 'https://www.alibaba.com', 'https://b2b.baidu.com', 'https://www.amazon.com/', '', 'https://www.example.com'],
|
|
|
|
|
|
fileList: [],
|
|
|
|
|
@@ -429,6 +430,8 @@
|
|
|
loginBrowser: null, // 登录用的浏览器实例
|
|
|
skipLogin: false,
|
|
|
pauseFlag: true, //暂停中止标志
|
|
|
+
|
|
|
+ rightClick: false, // 右击事件绑定
|
|
|
};
|
|
|
},
|
|
|
computed: {
|
|
@@ -486,9 +489,28 @@
|
|
|
// 初始化开发者设置
|
|
|
this.$utils.setStorage('headless', 1);
|
|
|
this.$utils.setStorage('waitUntil', 'networkidle2');
|
|
|
-
|
|
|
+
|
|
|
},
|
|
|
methods: {
|
|
|
+ add(){
|
|
|
+ if(!this.rightClick){
|
|
|
+ setTimeout(() => {
|
|
|
+ document.getElementById('aaa').addEventListener('contextmenu', e => {
|
|
|
+ e.preventDefault();
|
|
|
+ // 发送事件到主进程,附带元素类型信息
|
|
|
+ ipcRenderer.send('show-context-menu', 'input');
|
|
|
+ });
|
|
|
+
|
|
|
+ document.getElementById('bbb').addEventListener('contextmenu', e => {
|
|
|
+ e.preventDefault();
|
|
|
+ // 发送事件到主进程,附带元素类型信息
|
|
|
+ ipcRenderer.send('show-context-menu', 'input');
|
|
|
+ });
|
|
|
+ }, 800)
|
|
|
+
|
|
|
+ this.rightClick = true;
|
|
|
+ }
|
|
|
+ },
|
|
|
// 实时获取浏览器路径
|
|
|
initPath(){
|
|
|
let chromePath = puppeteer.executablePath().replace('win32-1', 'win64-1');
|
|
@@ -1015,7 +1037,22 @@
|
|
|
// });
|
|
|
const page = await commonBrowser.newPage();
|
|
|
|
|
|
+ let titleFlag = true;
|
|
|
page.on('response', async(response) => {
|
|
|
+ if(titleFlag){ // 第一次生成页面标题
|
|
|
+ if(urlInfo.title){
|
|
|
+ if (fs.existsSync(this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title)) {
|
|
|
+ urlInfo.newPath = this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title;
|
|
|
+ } else {
|
|
|
+ fs.mkdirSync(this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title);
|
|
|
+ urlInfo.newPath = this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title;
|
|
|
+ }
|
|
|
+ }else{
|
|
|
+ await this.getTitle(page, urlInfo); // 生成页面标题对应的文件夹
|
|
|
+ }
|
|
|
+ titleFlag = false;
|
|
|
+ }
|
|
|
+
|
|
|
// 检查响应的 MIME 类型是否以 'image/' 开头
|
|
|
let flag = false;
|
|
|
if(response.headers()['content-encoding'] && response.headers()['content-encoding'].indexOf('gzip') > -1){
|
|
@@ -1051,17 +1088,6 @@
|
|
|
isBase: isBase
|
|
|
}
|
|
|
|
|
|
- if(urlInfo.title){
|
|
|
- if (fs.existsSync(this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title)) {
|
|
|
- urlInfo.newPath = this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title;
|
|
|
- } else {
|
|
|
- fs.mkdirSync(this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title);
|
|
|
- urlInfo.newPath = this.downloadDir + separator + pjson.softInfo.softName + separator + urlInfo.title;
|
|
|
- }
|
|
|
- }else{
|
|
|
- await this.getTitle(page, urlInfo); // 生成页面标题对应的文件夹
|
|
|
- }
|
|
|
-
|
|
|
let outputPath = urlInfo.newPath + '\\' + this.randomString(35) + '.' + imgInfo.imgType;
|
|
|
|
|
|
urlInfo.status = '3';
|
|
@@ -1295,6 +1321,12 @@
|
|
|
executablePath: this.initPath(),
|
|
|
args: ['--window-size=1280,800'],
|
|
|
userDataDir: os.tmpdir() + separator + 'chrome-data-capture-jd',
|
|
|
+ args: [
|
|
|
+ '--start-maximized',
|
|
|
+ '--no-sandbox',
|
|
|
+ '--disable-setuid-sandbox',
|
|
|
+ '--disable-blink-features=AutomationControlled'
|
|
|
+ ]
|
|
|
});
|
|
|
const page = await this.loginBrowser.newPage();
|
|
|
await page.setViewport({ width: 1280, height: 800 });
|
|
@@ -1306,9 +1338,15 @@
|
|
|
let userTags = document.querySelector('.nickname');
|
|
|
let userTags2 = document.querySelector('.nick'); //企业账号
|
|
|
|
|
|
- if(navTags && navTags.innerHTML.indexOf('请登录') > -1){
|
|
|
+ let userInfo2 = document.querySelector('.login_info'); //页面右侧的登录信息
|
|
|
+ let userTags3 = false;
|
|
|
+ if(userInfo2 && userInfo2.innerText.indexOf('退出') > -1){
|
|
|
+ userTags3 = true;
|
|
|
+ }
|
|
|
+
|
|
|
+ if(navTags && navTags.innerHTML.indexOf('请登录') > -1 && !userTags3){
|
|
|
return false;
|
|
|
- }else if(userTags || userTags2){
|
|
|
+ }else if(userTags || userTags2 || userTags3){
|
|
|
return true;
|
|
|
}else{
|
|
|
return false;
|
|
@@ -1482,111 +1520,151 @@
|
|
|
});
|
|
|
await page.goto(urlInfo.url, {waitUntil : 'networkidle2'});
|
|
|
|
|
|
- /**new**/
|
|
|
- //detailImg:详情图;skuImg:sku图片;commentImg: 评论图;video: 视频
|
|
|
- const imgInfo = await page.evaluate((authority, execNum) => {
|
|
|
- let outObj = {
|
|
|
- mainImg: [],
|
|
|
- skuImg: [],
|
|
|
- };
|
|
|
- //主图
|
|
|
- let arr1 = document.querySelectorAll('#spec-list img');
|
|
|
- for(let i=0; i< arr1.length; i++){
|
|
|
- let mainImgUrl = arr1[i].src;
|
|
|
- let reg = /\/n[0-9]+\/jfs\//;
|
|
|
- let reg2 = /\/n[0-9]+\/s([0-9]+)x([0-9]+)_jfs\//;
|
|
|
- let reg3 = /![a-z]+_[0-9]+x[0-9]+(.avif)?/;
|
|
|
- let replaceStr = '/n1/s800x800_jfs/';
|
|
|
- if(mainImgUrl.match(/\/n[0-9]+\/s50x66_jfs\//)){
|
|
|
- replaceStr = '/n1/s750x1000_jfs/';
|
|
|
- }
|
|
|
- mainImgUrl = mainImgUrl.replace(reg, replaceStr).replace(reg2, replaceStr).replace(reg3, '').replace('.avif', '');
|
|
|
-
|
|
|
- if(!authority && i < execNum){
|
|
|
- outObj.mainImg.push(mainImgUrl);
|
|
|
- }
|
|
|
- if(authority){
|
|
|
- outObj.mainImg.push(mainImgUrl);
|
|
|
- }
|
|
|
- }
|
|
|
- //sku图片
|
|
|
- let arr2 = document.querySelectorAll('#choose-attr-1 img');
|
|
|
- for(let i=0; i< arr2.length; i++){
|
|
|
- let skuImgUrl = arr2[i].src;
|
|
|
- let skuReg = /\/n[0-9]+\/s([0-9]+)x([0-9]+)_jfs\//;
|
|
|
- let replaceStr = '/n1/s800x800_jfs/';
|
|
|
- if(skuImgUrl.match(/\/n[0-9]+\/s60x80_jfs\//)){
|
|
|
- replaceStr = '/n1/s750x1000_jfs/';
|
|
|
- }
|
|
|
- skuImgUrl = skuImgUrl.replace(skuReg, replaceStr).replace('.avif', '');
|
|
|
- if(!authority && i < execNum){
|
|
|
- outObj.skuImg.push(skuImgUrl);
|
|
|
- }
|
|
|
- if(authority){
|
|
|
- outObj.skuImg.push(skuImgUrl);
|
|
|
- }
|
|
|
-
|
|
|
- }
|
|
|
-
|
|
|
- return outObj;
|
|
|
- }, authority, this.execNum);
|
|
|
|
|
|
- if(this.settingArr.indexOf('mainImg') > -1){
|
|
|
- // 主图下载
|
|
|
- for(let j = 0; j < imgInfo.mainImg.length; j++){
|
|
|
- let fileName = imgInfo.mainImg[j].split('/').pop();
|
|
|
- if(fileName){
|
|
|
- let queryIndex = fileName.indexOf('?');
|
|
|
- if (queryIndex !== -1) {
|
|
|
- fileName = fileName.substr(0, queryIndex);
|
|
|
- }
|
|
|
-
|
|
|
- let num = Number(j) + 1;
|
|
|
- let suffix = '';
|
|
|
- if(fileName.lastIndexOf('.') > -1){
|
|
|
- suffix = fileName.substr(fileName.lastIndexOf('.'));
|
|
|
- }
|
|
|
-
|
|
|
- if (!fs.existsSync(urlInfo.newPath + '\\主图')) {
|
|
|
- fs.mkdirSync(urlInfo.newPath + '\\主图');
|
|
|
+ let pageInfo = await page.evaluate(() => {
|
|
|
+ let cHeight = document.documentElement.clientHeight;
|
|
|
+ let scrollHeight = document.body.scrollHeight;
|
|
|
+ return {'scrollHeight': scrollHeight, 'cHeight': cHeight}
|
|
|
+ });
|
|
|
+
|
|
|
+ let scrollHeight = pageInfo.scrollHeight;
|
|
|
+ let cHeight = pageInfo.cHeight;
|
|
|
+ let num = Math.ceil(scrollHeight / cHeight);
|
|
|
+ let start = -1;
|
|
|
+ let scrollTime = this.initMs();
|
|
|
+ let scrollInt = setInterval(async() => {
|
|
|
+ start ++;
|
|
|
+ if(this.settingArr.indexOf('detailImg') > -1){ // 选择详情图
|
|
|
+ let scrollHeight2 = await page.evaluate((start) => {
|
|
|
+ let scrollHeight = document.body.scrollHeight;
|
|
|
+ let cHeight = document.documentElement.clientHeight;
|
|
|
+ let obj = document.querySelector('.detail-content') || document.querySelector('.detail-content-item');
|
|
|
+ if(obj && obj.getBoundingClientRect().top < 0){
|
|
|
+ return -1;
|
|
|
}
|
|
|
-
|
|
|
- let outputPath = urlInfo.newPath + '\\主图\\主图' + num + suffix;
|
|
|
- await this.downloadImage(imgInfo.mainImg[j], outputPath, urlInfo);
|
|
|
+ window.scrollTo({
|
|
|
+ top: cHeight * start,
|
|
|
+ behavior: "smooth"
|
|
|
+ });
|
|
|
+ return scrollHeight;
|
|
|
+ }, start);
|
|
|
+
|
|
|
+ if(scrollHeight2 > 0){
|
|
|
+ num = Math.ceil(scrollHeight2 / cHeight);
|
|
|
+ }else{
|
|
|
+ num = 0;
|
|
|
}
|
|
|
+ }else{
|
|
|
+ num = 0;
|
|
|
}
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
- // sku图片下载
|
|
|
- if(this.settingArr.indexOf('skuImg') > -1){
|
|
|
- for(let j = 0; j < imgInfo.skuImg.length; j++){
|
|
|
- let fileName = imgInfo.skuImg[j].split('/').pop();
|
|
|
- if(fileName){
|
|
|
- let queryIndex = fileName.indexOf('?');
|
|
|
- if (queryIndex !== -1) {
|
|
|
- fileName = fileName.substr(0, queryIndex);
|
|
|
- }
|
|
|
+ if(start > num || start > 20){ // 防止页面过长,滚动20次自动停止
|
|
|
+ clearInterval(scrollInt);
|
|
|
+ //detailImg:详情图;skuImg:sku图片;commentImg: 评论图;video: 视频
|
|
|
+ const imgInfo = await page.evaluate((authority, execNum) => {
|
|
|
+ let outObj = {
|
|
|
+ mainImg: [],
|
|
|
+ skuImg: [],
|
|
|
+ };
|
|
|
+ //主图
|
|
|
+ let arr1 = document.querySelectorAll('#spec-list img');
|
|
|
+ for(let i=0; i< arr1.length; i++){
|
|
|
+ let mainImgUrl = arr1[i].src;
|
|
|
+ let reg = /\/n[0-9]+\/jfs\//;
|
|
|
+ let reg2 = /\/n[0-9]+\/s([0-9]+)x([0-9]+)_jfs\//;
|
|
|
+ let reg3 = /![a-z]+_[0-9]+x[0-9]+(.avif)?/;
|
|
|
+ let replaceStr = '/n1/s800x800_jfs/';
|
|
|
+ if(mainImgUrl.match(/\/n[0-9]+\/s50x66_jfs\//)){
|
|
|
+ replaceStr = '/n1/s750x1000_jfs/';
|
|
|
+ }
|
|
|
+ mainImgUrl = mainImgUrl.replace(reg, replaceStr).replace(reg2, replaceStr).replace(reg3, '').replace('.avif', '');
|
|
|
|
|
|
- let num = Number(j) + 1;
|
|
|
- let suffix = '';
|
|
|
- if(fileName.lastIndexOf('.') > -1){
|
|
|
- suffix = fileName.substr(fileName.lastIndexOf('.'));
|
|
|
+ if(!authority && i < execNum){
|
|
|
+ outObj.mainImg.push(mainImgUrl);
|
|
|
+ }
|
|
|
+ if(authority){
|
|
|
+ outObj.mainImg.push(mainImgUrl);
|
|
|
+ }
|
|
|
}
|
|
|
-
|
|
|
- if (!fs.existsSync(urlInfo.newPath + '\\sku图')) {
|
|
|
- fs.mkdirSync(urlInfo.newPath + '\\sku图');
|
|
|
+ //sku图片
|
|
|
+ let arr2 = document.querySelectorAll('#choose-attr-1 img');
|
|
|
+ for(let i=0; i< arr2.length; i++){
|
|
|
+ let skuImgUrl = arr2[i].src;
|
|
|
+ let skuReg = /\/n[0-9]+\/s([0-9]+)x([0-9]+)_jfs\//;
|
|
|
+ let replaceStr = '/n1/s800x800_jfs/';
|
|
|
+ if(skuImgUrl.match(/\/n[0-9]+\/s60x80_jfs\//)){
|
|
|
+ replaceStr = '/n1/s750x1000_jfs/';
|
|
|
+ }
|
|
|
+ skuImgUrl = skuImgUrl.replace(skuReg, replaceStr).replace('.avif', '');
|
|
|
+ if(!authority && i < execNum){
|
|
|
+ outObj.skuImg.push(skuImgUrl);
|
|
|
+ }
|
|
|
+ if(authority){
|
|
|
+ outObj.skuImg.push(skuImgUrl);
|
|
|
+ }
|
|
|
+
|
|
|
}
|
|
|
|
|
|
- let outputPath = urlInfo.newPath + '\\sku图\\sku图' + num + suffix;
|
|
|
- await this.downloadImage(imgInfo.skuImg[j], outputPath, urlInfo);
|
|
|
+ return outObj;
|
|
|
+ }, authority, this.execNum);
|
|
|
+
|
|
|
+ if(this.settingArr.indexOf('mainImg') > -1){
|
|
|
+ // 主图下载
|
|
|
+ for(let j = 0; j < imgInfo.mainImg.length; j++){
|
|
|
+ let fileName = imgInfo.mainImg[j].split('/').pop();
|
|
|
+ if(fileName){
|
|
|
+ let queryIndex = fileName.indexOf('?');
|
|
|
+ if (queryIndex !== -1) {
|
|
|
+ fileName = fileName.substr(0, queryIndex);
|
|
|
+ }
|
|
|
+
|
|
|
+ let num = Number(j) + 1;
|
|
|
+ let suffix = '';
|
|
|
+ if(fileName.lastIndexOf('.') > -1){
|
|
|
+ suffix = fileName.substr(fileName.lastIndexOf('.'));
|
|
|
+ }
|
|
|
+
|
|
|
+ if (!fs.existsSync(urlInfo.newPath + '\\主图')) {
|
|
|
+ fs.mkdirSync(urlInfo.newPath + '\\主图');
|
|
|
+ }
|
|
|
+
|
|
|
+ let outputPath = urlInfo.newPath + '\\主图\\主图' + num + suffix;
|
|
|
+ await this.downloadImage(imgInfo.mainImg[j], outputPath, urlInfo);
|
|
|
+ }
|
|
|
+ }
|
|
|
}
|
|
|
+
|
|
|
+ // sku图片下载
|
|
|
+ if(this.settingArr.indexOf('skuImg') > -1){
|
|
|
+ for(let j = 0; j < imgInfo.skuImg.length; j++){
|
|
|
+ let fileName = imgInfo.skuImg[j].split('/').pop();
|
|
|
+ if(fileName){
|
|
|
+ let queryIndex = fileName.indexOf('?');
|
|
|
+ if (queryIndex !== -1) {
|
|
|
+ fileName = fileName.substr(0, queryIndex);
|
|
|
+ }
|
|
|
+
|
|
|
+ let num = Number(j) + 1;
|
|
|
+ let suffix = '';
|
|
|
+ if(fileName.lastIndexOf('.') > -1){
|
|
|
+ suffix = fileName.substr(fileName.lastIndexOf('.'));
|
|
|
+ }
|
|
|
+
|
|
|
+ if (!fs.existsSync(urlInfo.newPath + '\\sku图')) {
|
|
|
+ fs.mkdirSync(urlInfo.newPath + '\\sku图');
|
|
|
+ }
|
|
|
+
|
|
|
+ let outputPath = urlInfo.newPath + '\\sku图\\sku图' + num + suffix;
|
|
|
+ await this.downloadImage(imgInfo.skuImg[j], outputPath, urlInfo);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ /**end**/
|
|
|
+ await page.close();
|
|
|
+ urlInfo.status = '4';
|
|
|
+ resolve(true);
|
|
|
+ this.loading = false;
|
|
|
}
|
|
|
- }
|
|
|
- /**end**/
|
|
|
- await page.close();
|
|
|
- urlInfo.status = '4';
|
|
|
- resolve(true);
|
|
|
+ }, scrollTime);
|
|
|
}catch(e){
|
|
|
reject(e);
|
|
|
this.showError(e);
|
|
@@ -3020,11 +3098,16 @@
|
|
|
async getTitle(page, urlInfo){
|
|
|
// 已页面标题作为新建文件夹,保留前50个字
|
|
|
let title = await page.title();
|
|
|
+ if(!title){
|
|
|
+ title = '无标题-'+new Date().getTime();
|
|
|
+ }
|
|
|
if(title){
|
|
|
title = title.substring(0, 50).trim();
|
|
|
if(this.containsAnyChar(title, ['\\', '/', ':', '*', '?', '"', '<', '>', '|'])){ //判断是否含有特殊字符
|
|
|
title = title.replace(/[\\|/|:|*|?|"|<|>||]/g, "");
|
|
|
}
|
|
|
+
|
|
|
+ console.log(title);
|
|
|
if (fs.existsSync(this.downloadDir + separator + pjson.softInfo.softName + separator + title)) {
|
|
|
urlInfo.newPath = this.downloadDir + separator + pjson.softInfo.softName + separator + title;
|
|
|
} else {
|