|
|
@@ -92,7 +92,7 @@
|
|
|
<el-button size="small" type="danger" @click="addLinks()" :loading="addLoading">开始提取</el-button>
|
|
|
<el-button size="small" type="danger" @click="pauseLinks()">停止提取</el-button>
|
|
|
<el-button size="small" type="danger" @click="clearList()">清除结果</el-button>
|
|
|
- <el-button size="small" type="danger" @click="exportLinks()">合并导出全部</el-button>
|
|
|
+ <el-button size="small" type="danger" @click="exportLinks()" :loading="allLoading">合并导出全部</el-button>
|
|
|
</div>
|
|
|
</div>
|
|
|
|
|
|
@@ -108,7 +108,7 @@
|
|
|
<vxe-table style="width: 100%;" ref="xTable" show-overflow class="img-table" max-height="100%" empty-text="没有更多数据了！" :loading="tabLoading" :row-config="{isHover: true}"
|
|
|
:loading-config="{icon: 'vxe-icon-indicator roll', text: '加载中...'}" :data="this[listStr+'List']" :scroll-y="{enabled: true}">
|
|
|
<vxe-column field="title" title="名称"></vxe-column>
|
|
|
- <vxe-column field="page" title="页数">
|
|
|
+ <vxe-column field="page" title="页数" width="160">
|
|
|
<template #default="{ row }">
|
|
|
第{{row.currentPage}}页，共{{row.totalPage}}页
|
|
|
</template>
|
|
|
@@ -118,7 +118,7 @@
|
|
|
<span v-if="row.urls">{{row.urls.length}}</span>
|
|
|
</template>
|
|
|
</vxe-column>
|
|
|
- <vxe-column field="status" title="状态" width="150">
|
|
|
+ <vxe-column field="status" title="状态" width="160">
|
|
|
<template #default="{ row }">
|
|
|
<template v-if="row.status == '1'">
|
|
|
<i class="el-icon-info" style="font-size: 16px; color: #999;"></i>
|
|
|
@@ -130,9 +130,10 @@
|
|
|
</template>
|
|
|
</template>
|
|
|
</vxe-column>
|
|
|
- <vxe-column title="操作" width="150">
|
|
|
+ <vxe-column title="操作" width="160">
|
|
|
<template #default="{ row, rowIndex }">
|
|
|
- <el-button :loading="row.loading" type="danger" size="mini" plain @click="exportPageLinks(row)">导出</el-button>
|
|
|
+ <el-button type="danger" size="mini" plain @click="delRow(rowIndex)">删除</el-button>
|
|
|
+ <el-button :loading="row.loading" size="mini" plain @click="exportPageLinks(row)">导出</el-button>
|
|
|
</template>
|
|
|
</vxe-column>
|
|
|
</vxe-table>
|
|
|
@@ -223,10 +224,7 @@
|
|
|
loginBrowser: null, // 登录用的浏览器实例
|
|
|
pauseFlag: true, //暂停中止标志
|
|
|
addLoading: false,
|
|
|
-
|
|
|
- videoBrowser: null,
|
|
|
- parseLoading: false,
|
|
|
- mergeLoading: false
|
|
|
+ allLoading: false,
|
|
|
|
|
|
|
|
|
};
|
|
|
@@ -254,13 +252,6 @@
|
|
|
if (!fs.existsSync(this.downloadDir + separator + pjson.softInfo.softName)) {
|
|
|
fs.mkdirSync(this.downloadDir + separator + pjson.softInfo.softName);
|
|
|
}
|
|
|
- if (!fs.existsSync(os.tmpdir() + separator + 'chrome-data-capture')) {
|
|
|
- fs.mkdirSync(os.tmpdir() + separator + 'chrome-data-capture');
|
|
|
- }
|
|
|
-
|
|
|
- if (!fs.existsSync(os.tmpdir() + separator + 'chrome-data-capture-jd')) {
|
|
|
- fs.mkdirSync(os.tmpdir() + separator + 'chrome-data-capture-jd');
|
|
|
- }
|
|
|
|
|
|
if (!fs.existsSync(os.tmpdir() + separator + 'chrome-data-capture-local')) {
|
|
|
fs.mkdirSync(os.tmpdir() + separator + 'chrome-data-capture-local');
|
|
|
@@ -332,48 +323,42 @@
|
|
|
},
|
|
|
// 实时获取浏览器路径
|
|
|
initPath(){
|
|
|
- let chromeType = this.$utils.getStorage('chromeType');
|
|
|
- let chromePath = puppeteer.executablePath().replace('win32-1', 'win64-1');
|
|
|
- if(chromeType == 1){ // 电脑自带浏览器
|
|
|
- chromePath = this.$utils.getStorage('chromePath');
|
|
|
- }else{
|
|
|
- chromePath = puppeteer.executablePath().replace('win32-1', 'win64-1');
|
|
|
- let versionType = this.$utils.getStorage('versionType');
|
|
|
- if(versionType && versionType == 1){
|
|
|
- chromePath = chromePath.replace('chrome-win', 'chrome7');
|
|
|
- }
|
|
|
- }
|
|
|
+ let chromePath = this.$utils.getStorage('chromePath');
|
|
|
return chromePath;
|
|
|
},
|
|
|
// 实时获取浏览器数据缓存
|
|
|
initDataDir(tag){
|
|
|
- let chromeType = this.$utils.getStorage('chromeType');
|
|
|
- let userDataDir = os.tmpdir() + separator + 'chrome-data-capture';
|
|
|
- if(chromeType == 1){ // 电脑自带浏览器
|
|
|
- userDataDir = os.tmpdir() + separator + 'chrome-data-capture-local';
|
|
|
- }else{
|
|
|
- userDataDir = os.tmpdir() + separator + 'chrome-data-capture';
|
|
|
- if(tag == 'jd'){
|
|
|
- userDataDir = os.tmpdir() + separator + 'chrome-data-capture-jd';
|
|
|
- }
|
|
|
- }
|
|
|
+ // let chromeType = this.$utils.getStorage('chromeType');
|
|
|
+ let userDataDir = os.tmpdir() + separator + 'chrome-data-capture-local';
|
|
|
+ // if(chromeType == 1){ // 电脑自带浏览器
|
|
|
+ // userDataDir = os.tmpdir() + separator + 'chrome-data-capture-local';
|
|
|
+ // }else{
|
|
|
+ // userDataDir = os.tmpdir() + separator + 'chrome-data-capture';
|
|
|
+ // if(tag == 'jd'){
|
|
|
+ // userDataDir = os.tmpdir() + separator + 'chrome-data-capture-jd';
|
|
|
+ // }
|
|
|
+ // }
|
|
|
return userDataDir;
|
|
|
},
|
|
|
// 任务下载间隔时间
|
|
|
initGap(){
|
|
|
let gap = this.$utils.getStorage('gap');
|
|
|
+ let gapRandom = this.$utils.getStorage('gapRandom');
|
|
|
if(gap){
|
|
|
- return Number(gap);
|
|
|
+ let g = Number(gap);
|
|
|
+ let r = 0;
|
|
|
+ if(gapRandom){
|
|
|
+ r = Number(gapRandom) + 1;
|
|
|
+ }
|
|
|
+ let n = Math.floor(Math.random() * r) + g;
|
|
|
+ return n;
|
|
|
}else{
|
|
|
- return 1;
|
|
|
+ return 5;
|
|
|
}
|
|
|
},
|
|
|
// 通用的延迟函数
|
|
|
- async delay(t) {
|
|
|
+ async delay() {
|
|
|
let s = this.initGap();
|
|
|
- if(t){
|
|
|
- s = t;
|
|
|
- }
|
|
|
if(s > 0){
|
|
|
return new Promise(resolve => setTimeout(resolve, Number(s)*1000));
|
|
|
}else{
|
|
|
@@ -414,16 +399,43 @@
|
|
|
},
|
|
|
// 清除缓存的后续操作
|
|
|
async clearCache(){
|
|
|
+ this.jdStatus = 3;
|
|
|
+ this.tbStatus = 3;
|
|
|
+ this.redStatus = 3;
|
|
|
+ this.alibabaStatus = 3;
|
|
|
+
|
|
|
if(this.loginBrowser){
|
|
|
await this.loginBrowser.close();
|
|
|
this.loginBrowser = null;
|
|
|
}
|
|
|
|
|
|
- if(this.videoBrowser){
|
|
|
- await this.videoBrowser.close();
|
|
|
- this.videoBrowser = null;
|
|
|
+ for(let i = 0; i < listNameArr.length; i++){
|
|
|
+ if(listNameArr[i]){
|
|
|
+ if(this[listNameArr[i]+'Browser']){
|
|
|
+ await this[listNameArr[i]+'Browser'].close();
|
|
|
+ this[listNameArr[i]+'Browser'] = null;
|
|
|
+ }
|
|
|
+ }
|
|
|
}
|
|
|
},
|
|
|
+ // 删除一行数据
|
|
|
+ delRow(rowIndex){
|
|
|
+ let index = Number(this.menuIndex) - 1;
|
|
|
+ let type = listNameArr[index];
|
|
|
+
|
|
|
+ this.$confirm('确认删除此行数据吗?', '提示', {
|
|
|
+ confirmButtonText: '确定',
|
|
|
+ cancelButtonText: '取消',
|
|
|
+ type: 'warning'
|
|
|
+ }).then(() => {
|
|
|
+ this[type+'List'].splice(rowIndex, 1);
|
|
|
+ if(this[type+'List'].length == 0) {
|
|
|
+ this.clearList();
|
|
|
+ }
|
|
|
+ }).catch(() => {
|
|
|
+
|
|
|
+ });
|
|
|
+ },
|
|
|
// 清除结果
|
|
|
clearList(){
|
|
|
let index = Number(this.menuIndex) - 1;
|
|
|
@@ -441,11 +453,16 @@
|
|
|
await this[browserName].close();
|
|
|
this[browserName] = null;
|
|
|
}
|
|
|
+
|
|
|
+ this.$notify.info({
|
|
|
+ title: '提示',
|
|
|
+ message: '提取任务已停止执行'
|
|
|
+ });
|
|
|
},
|
|
|
//导出全部链接
|
|
|
exportLinks(){
|
|
|
let allRows = {
|
|
|
- title: '淘宝商品链接汇总',
|
|
|
+ title: '商品链接汇总',
|
|
|
urls: [],
|
|
|
loading: false,
|
|
|
status: '1'
|
|
|
@@ -459,11 +476,7 @@
|
|
|
break;
|
|
|
case '3': // 天猫
|
|
|
case '4': // 淘宝
|
|
|
- let allUrls = [];
|
|
|
- for(let i = 0; i < this.tbList.length; i++){
|
|
|
- allUrls = allUrls.concat(this.tbList[i].urls);
|
|
|
- }
|
|
|
- allRows.urls = allUrls;
|
|
|
+ allRows.title = '淘宝商品链接汇总';
|
|
|
break;
|
|
|
case '5': // 小红书
|
|
|
allRows.title = '小红书笔记链接汇总';
|
|
|
@@ -472,8 +485,19 @@
|
|
|
allRows.title = '网页链接汇总';
|
|
|
break;
|
|
|
}
|
|
|
-
|
|
|
- this.exportPageLinks(allRows);
|
|
|
+ let index = Number(this.menuIndex) - 1;
|
|
|
+ let allUrls = [];
|
|
|
+ if(this[listNameArr[index]+'List'].length > 0){
|
|
|
+ this.allLoading = true;
|
|
|
+ for(let i = 0; i < this[listNameArr[index]+'List'].length; i++){
|
|
|
+ allUrls = allUrls.concat(this[listNameArr[index]+'List'][i].urls);
|
|
|
+ }
|
|
|
+ allRows.urls = allUrls;
|
|
|
+ this.exportPageLinks(allRows);
|
|
|
+ setTimeout(() => {
|
|
|
+ this.allLoading = false;
|
|
|
+ }, 2000)
|
|
|
+ }
|
|
|
},
|
|
|
// 导出单页链接
|
|
|
exportPageLinks(row){
|
|
|
@@ -519,6 +543,13 @@
|
|
|
},
|
|
|
//提取链接
|
|
|
async addLinks(){
|
|
|
+ if(!this.formatUrl.trim()){
|
|
|
+ this.$notify.error({
|
|
|
+ title: '提示',
|
|
|
+ message: '请输入需要提取的网址'
|
|
|
+ });
|
|
|
+ return false;
|
|
|
+ }
|
|
|
this.addLoading = true;
|
|
|
setTimeout(() => {
|
|
|
this.addLoading = false;
|
|
|
@@ -542,6 +573,7 @@
|
|
|
});
|
|
|
if(this.alibabaStatus == 3){
|
|
|
this.loading = false;
|
|
|
+ this.addLoading = false;
|
|
|
return false;
|
|
|
}
|
|
|
}
|
|
|
@@ -559,6 +591,7 @@
|
|
|
});
|
|
|
if(this.jdStatus == 3){
|
|
|
this.loading = false;
|
|
|
+ this.addLoading = false;
|
|
|
return false;
|
|
|
}
|
|
|
}
|
|
|
@@ -577,6 +610,7 @@
|
|
|
}
|
|
|
if(this.tbStatus == 3){
|
|
|
this.loading = false;
|
|
|
+ this.addLoading = false;
|
|
|
return false;
|
|
|
}
|
|
|
}
|
|
|
@@ -593,6 +627,7 @@
|
|
|
});
|
|
|
if(this.redStatus == 3){
|
|
|
this.loading = false;
|
|
|
+ this.addLoading = false;
|
|
|
return false;
|
|
|
}
|
|
|
}
|
|
|
@@ -616,43 +651,37 @@
|
|
|
redSize = '--window-size=1920,800'; //给浏览器一个初始大小，在无头模式下，页面会自适用缩放
|
|
|
}
|
|
|
|
|
|
- try{
|
|
|
- this[browserName] = await puppeteer.launch({
|
|
|
- headless: headless,
|
|
|
- executablePath: this.initPath(),
|
|
|
- userDataDir: userDataDir,
|
|
|
- args: [
|
|
|
- '--start-maximized',
|
|
|
- '--no-sandbox',
|
|
|
- '--disable-setuid-sandbox',
|
|
|
- '--disable-blink-features=AutomationControlled',
|
|
|
- redSize
|
|
|
- ]
|
|
|
- });
|
|
|
-
|
|
|
- switch(this.menuIndex){
|
|
|
- case '1': // 阿里巴巴
|
|
|
- this.alibabaScanUrl(this.formatUrl);
|
|
|
- break;
|
|
|
- case '2': // 京东
|
|
|
- this.jdScanUrl(this.formatUrl);
|
|
|
- break;
|
|
|
- case '3': // 天猫
|
|
|
- case '4': // 淘宝
|
|
|
- this.tbScanUrl(this.formatUrl);
|
|
|
- break;
|
|
|
- case '5': // 小红书
|
|
|
- this.redScanUrl(this.formatUrl);
|
|
|
- break;
|
|
|
- case '10': // 普通网址
|
|
|
- this.commonScanUrl(this.formatUrl);
|
|
|
- break;
|
|
|
- }
|
|
|
- }catch(e){
|
|
|
- console.log(e, '测试');
|
|
|
- }
|
|
|
-
|
|
|
+ this[browserName] = await puppeteer.launch({
|
|
|
+ headless: headless,
|
|
|
+ executablePath: this.initPath(),
|
|
|
+ userDataDir: userDataDir,
|
|
|
+ args: [
|
|
|
+ '--start-maximized',
|
|
|
+ '--no-sandbox',
|
|
|
+ '--disable-setuid-sandbox',
|
|
|
+ '--disable-blink-features=AutomationControlled',
|
|
|
+ redSize
|
|
|
+ ]
|
|
|
+ });
|
|
|
|
|
|
+ switch(this.menuIndex){
|
|
|
+ case '1': // 阿里巴巴
|
|
|
+ this.alibabaScanUrl(this.formatUrl);
|
|
|
+ break;
|
|
|
+ case '2': // 京东
|
|
|
+ this.jdScanUrl(this.formatUrl);
|
|
|
+ break;
|
|
|
+ case '3': // 天猫
|
|
|
+ case '4': // 淘宝
|
|
|
+ this.tbScanUrl(this.formatUrl);
|
|
|
+ break;
|
|
|
+ case '5': // 小红书
|
|
|
+ this.redScanUrl(this.formatUrl);
|
|
|
+ break;
|
|
|
+ case '10': // 普通网址
|
|
|
+ this.commonScanUrl(this.formatUrl);
|
|
|
+ break;
|
|
|
+ }
|
|
|
},
|
|
|
|
|
|
// 天猫/淘宝扫描网址
|
|
|
@@ -667,13 +696,7 @@
|
|
|
waitUntil = this.initDevelop().waitUntil;
|
|
|
await page.goto(url, {waitUntil : waitUntil});
|
|
|
|
|
|
- let searchTitle = await page.title() || '淘宝商品搜索';
|
|
|
- if(searchTitle){
|
|
|
- searchTitle = searchTitle.substring(0, 50);
|
|
|
- if(this.containsAnyChar(searchTitle)){ //判断是否含有特殊字符
|
|
|
- searchTitle = searchTitle.replace(/[\\/:*?"<>|#%&\s]/g, "");
|
|
|
- }
|
|
|
- }
|
|
|
+ let searchTitle = await this.getTitle(page, '淘宝商品搜索');
|
|
|
let outObj = {};
|
|
|
let pageHandle = await page.$$('[class^=next-pagination-display]');
|
|
|
let currentPage = 1;
|
|
|
@@ -748,7 +771,7 @@
|
|
|
}, authority, this.execNum, itemTitle);
|
|
|
|
|
|
this.tbList.push(imgInfo);
|
|
|
- await this.delay(5);
|
|
|
+ await this.delay();
|
|
|
}
|
|
|
|
|
|
await this.tbBrowser.close();
|
|
|
@@ -767,20 +790,14 @@
|
|
|
try{
|
|
|
let authority = this.$refs.headerRef.authority.isAuthority;
|
|
|
let page = await this.alibabaBrowser.newPage();
|
|
|
+ let gapTime = Number(this.initGap()) * 1000;
|
|
|
|
|
|
let responseVideo = [];
|
|
|
let waitUntil = 'networkidle2';
|
|
|
waitUntil = this.initDevelop().waitUntil;
|
|
|
await page.goto(url, {waitUntil : waitUntil});
|
|
|
|
|
|
- let searchTitle = await page.title() || '阿里巴巴商品搜索';
|
|
|
- if(searchTitle){
|
|
|
- searchTitle = searchTitle.substring(0, 50);
|
|
|
- if(this.containsAnyChar(searchTitle)){ //判断是否含有特殊字符
|
|
|
- searchTitle = searchTitle.replace(/[\\/:*?"<>|#%&\s]/g, "");
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
+ let searchTitle = await this.getTitle(page, '阿里巴巴商品搜索');
|
|
|
//找页码信息
|
|
|
let outObj = {};
|
|
|
let pageHandle = await page.$$('[class^=pagination-container] span');
|
|
|
@@ -791,77 +808,7 @@
|
|
|
currentPage = await page.$eval('[class^=pagination-container] .fui-current', el => el.innerText);
|
|
|
}
|
|
|
|
|
|
- for(let i = Number(currentPage); i < Number(totalPage)+1; i++){
|
|
|
- await new Promise(resolve => setTimeout(resolve, 1500)).then(async() => {
|
|
|
- await page.evaluate(() => {
|
|
|
- window.scrollTo({
|
|
|
- top: 10000,
|
|
|
- behavior: "smooth"
|
|
|
- });
|
|
|
- });
|
|
|
- });
|
|
|
-
|
|
|
- await new Promise(resolve => setTimeout(resolve, 5000)).then(async() => {
|
|
|
- let itemTitle = searchTitle + '-' + i;
|
|
|
- const imgInfo = await page.evaluate((authority, execNum, itemTitle) => {
|
|
|
- let outObj = {
|
|
|
- title: itemTitle,
|
|
|
- currentPage: 1,
|
|
|
- totalPage: 1,
|
|
|
- urls: [],
|
|
|
- status: '1',
|
|
|
- loading: false
|
|
|
- };
|
|
|
- let total = document.querySelector('[class^=pagination-container] span');
|
|
|
- if(total){
|
|
|
- outObj.currentPage = document.querySelector('[class^=pagination-container] .fui-current').innerText || 1;
|
|
|
- outObj.totalPage = total.innerText || 1;
|
|
|
- }
|
|
|
- // 广告链接
|
|
|
- // document.querySelectorAll('[class*=space-common-offerlist] a[class*=pc-ad]');
|
|
|
- // 正常
|
|
|
- let arr1 = document.querySelectorAll('[class*=space-common-offerlist] a[class*=search-offer-item]');
|
|
|
- for(let i=0; i< arr1.length; i++){
|
|
|
- let href = arr1[i].href;
|
|
|
- let idData = arr1[i].dataset.renderkey || '';
|
|
|
- let length = idData.split('_').length - 1;
|
|
|
- let id = idData.split('_')[length];
|
|
|
- let title = arr1[i].querySelector('[class^=title-text]') ? arr1[i].querySelector('[class^=title-text]').innerText : '阿里巴巴商品'+new Date().getTime();
|
|
|
- let priceText = arr1[i].querySelector('[class^=price-item]') ? arr1[i].querySelector('[class^=price-item]').innerText : '';
|
|
|
- let price = priceText.replace('¥', '').replaceAll('\n', '');
|
|
|
- let realDom = arr1[i].querySelector('.offer-price-row .offer-desc-item');
|
|
|
- let realSales = realDom ? realDom.innerText : '';
|
|
|
- let shopNameText = arr1[i].querySelector('[class^=offer-shop-row]') ? arr1[i].querySelector('[class^=offer-shop-row]').innerText : '';
|
|
|
- let shopUrl = arr1[i].querySelector('[class^=offer-shop-row] a') ? arr1[i].querySelector('[class^=offer-shop-row] a').href : '';
|
|
|
- let mainPic = arr1[i].querySelector('img[class^=main-img]') ? arr1[i].querySelector('img[class^=main-img]').src : '';
|
|
|
-
|
|
|
- let info = {
|
|
|
- id: id,
|
|
|
- title: title,
|
|
|
- href: href,
|
|
|
- price: price,
|
|
|
- realSales: realSales,
|
|
|
- shopNameText: shopNameText,
|
|
|
- shopUrl: shopUrl,
|
|
|
- mainPic: mainPic
|
|
|
- }
|
|
|
- outObj.urls.push(info);
|
|
|
- }
|
|
|
- let nextHandle = document.querySelector('.pagination-container .fui-next');
|
|
|
- if(nextHandle){
|
|
|
- nextHandle.click({force: true});
|
|
|
- }
|
|
|
- return outObj;
|
|
|
- }, authority, this.execNum, itemTitle);
|
|
|
- this.alibabaList.push(imgInfo);
|
|
|
- });
|
|
|
-
|
|
|
- }
|
|
|
-
|
|
|
- await this.alibabaBrowser.close();
|
|
|
- this.addLoading = false;
|
|
|
-
|
|
|
-
|
|
|
+ await this.initScroll('alibaba', page, Number(currentPage), Number(totalPage), searchTitle);
|
|
|
}catch(e){
|
|
|
this.addLoading = false;
|
|
|
this.showError(e);
|
|
|
@@ -875,20 +822,14 @@
|
|
|
try{
|
|
|
let authority = this.$refs.headerRef.authority.isAuthority;
|
|
|
let page = await this.jdBrowser.newPage();
|
|
|
+ let gapTime = Number(this.initGap()) * 1000;
|
|
|
|
|
|
let responseVideo = [];
|
|
|
let waitUntil = 'networkidle2';
|
|
|
waitUntil = this.initDevelop().waitUntil;
|
|
|
await page.goto(url, {waitUntil : waitUntil});
|
|
|
|
|
|
- let searchTitle = await page.title() || '京东商品搜索';
|
|
|
- if(searchTitle){
|
|
|
- searchTitle = searchTitle.substring(0, 50);
|
|
|
- if(this.containsAnyChar(searchTitle)){ //判断是否含有特殊字符
|
|
|
- searchTitle = searchTitle.replace(/[\\/:*?"<>|#%&\s]/g, "");
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
+ let searchTitle = await this.getTitle(page, '京东商品搜索');
|
|
|
//找页码信息
|
|
|
let outObj = {};
|
|
|
let pageHandle = await page.$$('[class*=quick-result]');
|
|
|
@@ -907,89 +848,7 @@
|
|
|
totalPage = totalInfo.replace('共', '').replace('页', '');
|
|
|
}
|
|
|
}
|
|
|
-
|
|
|
- for(let i = Number(currentPage); i < Number(totalPage)+1; i++){
|
|
|
- await new Promise(resolve => setTimeout(resolve, 1500)).then(async() => {
|
|
|
- await page.evaluate(() => {
|
|
|
- window.scrollTo({
|
|
|
- top: 8000,
|
|
|
- behavior: "smooth"
|
|
|
- });
|
|
|
- });
|
|
|
- });
|
|
|
-
|
|
|
- await new Promise(resolve => setTimeout(resolve, 5000)).then(async() => {
|
|
|
- let itemTitle = searchTitle + '-' + i;
|
|
|
- const imgInfo = await page.evaluate((authority, execNum, itemTitle) => {
|
|
|
- let outObj = {
|
|
|
- title: itemTitle,
|
|
|
- currentPage: 1,
|
|
|
- totalPage: 1,
|
|
|
- urls: [],
|
|
|
- status: '1',
|
|
|
- loading: false
|
|
|
- };
|
|
|
-
|
|
|
- let total = document.querySelector('[class*=quick-result]');
|
|
|
- if(total){
|
|
|
- let pageContent = total.innerText.replace('\n', '').replace('\n', '');
|
|
|
- outObj.currentPage =pageContent.split('/')[0] || 1;
|
|
|
- outObj.totalPage = pageContent.split('/')[1] || 1;
|
|
|
- }else{
|
|
|
- let currentDom = document.querySelector('[class*=pagination_center] [class*=active]');
|
|
|
- if(currentDom){
|
|
|
- outObj.currentPage = currentDom.innerText;
|
|
|
- }
|
|
|
- let totalInfo = document.querySelector('[class*=pagination_total]');
|
|
|
- if(totalInfo){
|
|
|
- utObj.totalPage = totalInfo.innerText.replace('共', '').replace('页', '');
|
|
|
- }
|
|
|
- }
|
|
|
- // 正常
|
|
|
- let arr1 = document.querySelectorAll('[class*=goodsContainer]>div');
|
|
|
- for(let i=0; i< arr1.length; i++){
|
|
|
- let sku = arr1[i].dataset.sku || '';
|
|
|
- let title = arr1[i].querySelector('[class*=goods_title]') ? arr1[i].querySelector('[class*=goods_title]').innerText : '京东商品'+new Date().getTime();
|
|
|
- let priceText = arr1[i].querySelector('[class*=price]') ? arr1[i].querySelector('[class*=price]').innerText : '';
|
|
|
- let price = priceText.replace('¥', '').replaceAll('\n', '');
|
|
|
- let realDom = arr1[i].querySelector('[class*=goods_volume]');
|
|
|
- let realSales = realDom ? realDom.innerText : '';
|
|
|
- let shopNameText = arr1[i].querySelector('[class*=shopFloor]') ? arr1[i].querySelector('[class*=shopFloor]').innerText : '';
|
|
|
- let shopUrl = arr1[i].querySelector('[class*=shopFloor] a') ? arr1[i].querySelector('[class*=shopFloor] a').href : '';
|
|
|
- let mainPic = arr1[i].querySelector('img') ? arr1[i].querySelector('img').src : '';
|
|
|
-
|
|
|
- let info = {
|
|
|
- title: title,
|
|
|
- id: sku,
|
|
|
- href: "https://item.jd.com/" + sku + ".html",
|
|
|
- price: price,
|
|
|
- realSales: realSales,
|
|
|
- shopNameText: shopNameText,
|
|
|
- shopUrl: shopUrl,
|
|
|
- mainPic: mainPic
|
|
|
- }
|
|
|
- outObj.urls.push(info);
|
|
|
- }
|
|
|
- let nextHandle = document.querySelectorAll('[class*=quick-result] [class*=quick-num]');
|
|
|
- if(nextHandle && nextHandle.length > 0){
|
|
|
- nextHandle[1].click({force: true});
|
|
|
- }else{
|
|
|
- nextHandle = document.querySelector('[class*=pagination_center] [class*=pagination_next]');
|
|
|
- if(nextHandle && nextHandle.length > 0){
|
|
|
- nextHandle.click({force: true});
|
|
|
- }
|
|
|
- }
|
|
|
- return outObj;
|
|
|
- }, authority, this.execNum, itemTitle);
|
|
|
- this.jdList.push(imgInfo);
|
|
|
- });
|
|
|
-
|
|
|
- }
|
|
|
-
|
|
|
- await this.jdBrowser.close();
|
|
|
- this.addLoading = false;
|
|
|
-
|
|
|
-
|
|
|
+ await this.initScroll('jd', page, Number(currentPage), Number(totalPage), searchTitle);
|
|
|
}catch(e){
|
|
|
this.addLoading = false;
|
|
|
this.showError(e);
|
|
|
@@ -1009,14 +868,7 @@
|
|
|
waitUntil = this.initDevelop().waitUntil;
|
|
|
await page.goto(url, {waitUntil : waitUntil});
|
|
|
|
|
|
- let searchTitle = await page.title() || '小红书笔记';
|
|
|
- if(searchTitle){
|
|
|
- searchTitle = searchTitle.substring(0, 50);
|
|
|
- if(this.containsAnyChar(searchTitle)){ //判断是否含有特殊字符
|
|
|
- searchTitle = searchTitle.replace(/[\\/:*?"<>|#%&\s]/g, "");
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
+ let searchTitle = await this.getTitle(page, '小红书笔记');
|
|
|
let pageInfo = await page.evaluate(() => {
|
|
|
let cHeight = document.documentElement.clientHeight;
|
|
|
let scrollHeight = document.body.scrollHeight;
|
|
|
@@ -1119,7 +971,6 @@
|
|
|
})();
|
|
|
},
|
|
|
|
|
|
-
|
|
|
// 普通扫描网址
|
|
|
commonScanUrl(url){
|
|
|
(async () => {
|
|
|
@@ -1132,7 +983,7 @@
|
|
|
waitUntil = this.initDevelop().waitUntil;
|
|
|
await page.goto(url, {waitUntil : waitUntil});
|
|
|
|
|
|
- let searchTitle = await page.title() || '网址链接';
|
|
|
+ let searchTitle = await this.getTitle(page, '网址链接');
|
|
|
if(searchTitle){
|
|
|
searchTitle = searchTitle.substring(0, 50);
|
|
|
if(this.containsAnyChar(searchTitle)){ //判断是否含有特殊字符
|
|
|
@@ -1225,6 +1076,199 @@
|
|
|
})();
|
|
|
},
|
|
|
|
|
|
+ // 滚动到页面底部或者滚动次数固定
|
|
|
+ async initScroll(typeStr, page, currentPage, totalPage, searchTitle){
|
|
|
+ let pageInfo = await page.evaluate(() => {
|
|
|
+ let cHeight = document.documentElement.clientHeight;
|
|
|
+ let scrollHeight = document.body.scrollHeight;
|
|
|
+ return {'scrollHeight': scrollHeight, 'cHeight': cHeight}
|
|
|
+ });
|
|
|
+
|
|
|
+ let scrollHeight = pageInfo.scrollHeight;
|
|
|
+ let cHeight = pageInfo.cHeight;
|
|
|
+ let num = Math.ceil(scrollHeight / cHeight);
|
|
|
+ let start = -1;
|
|
|
+ let scrollTime = this.initMs();
|
|
|
+ let scrollInt = setInterval(async() => {
|
|
|
+ if(this[typeStr+'Browser'] && !this[typeStr+'Browser'].isConnected()){
|
|
|
+ clearInterval(scrollInt);
|
|
|
+ await this[typeStr+'Browser'].close();
|
|
|
+ this[typeStr+'Browser'] = null;
|
|
|
+ }
|
|
|
+ start ++;
|
|
|
+ let scrollHeight = await page.evaluate((start) => {
|
|
|
+ let scrollHeight = document.body.scrollHeight;
|
|
|
+ let cHeight = document.documentElement.clientHeight;
|
|
|
+ window.scrollTo({
|
|
|
+ top: cHeight * start,
|
|
|
+ behavior: "smooth"
|
|
|
+ });
|
|
|
+ return scrollHeight;
|
|
|
+ }, start);
|
|
|
+
|
|
|
+ if(scrollHeight > 0){
|
|
|
+ num = Math.ceil(scrollHeight / cHeight);
|
|
|
+ }else{
|
|
|
+ num = 0;
|
|
|
+ }
|
|
|
+
|
|
|
+ if(start > num || start > 20){ //防止页面过长，滚动20次自动停止
|
|
|
+ clearInterval(scrollInt);
|
|
|
+ const imgInfo = await page.evaluate((searchTitle, typeStr) => {
|
|
|
+ let outObj = {
|
|
|
+ title: searchTitle,
|
|
|
+ currentPage: 1,
|
|
|
+ totalPage: 1,
|
|
|
+ urls: [],
|
|
|
+ status: '1',
|
|
|
+ loading: false
|
|
|
+ };
|
|
|
+ if(typeStr == 'alibaba'){ //阿里巴巴处理
|
|
|
+ let total = document.querySelector('[class^=pagination-container] span');
|
|
|
+ if(total){
|
|
|
+ outObj.currentPage = document.querySelector('[class^=pagination-container] .fui-current').innerText || 1;
|
|
|
+ outObj.totalPage = total.innerText || 1;
|
|
|
+ outObj.title = searchTitle + '-' + outObj.currentPage;
|
|
|
+ }
|
|
|
+
|
|
|
+ // 正常
|
|
|
+ let arr1 = document.querySelectorAll('[class*=page-offerlist] a[class*=search-offer-item]');
|
|
|
+ for(let i=0; i< arr1.length; i++){
|
|
|
+ let href = arr1[i].href;
|
|
|
+ let idData = arr1[i].dataset.renderkey || '';
|
|
|
+ let length = idData.split('_').length - 1;
|
|
|
+ let id = idData.split('_')[length];
|
|
|
+ let title = arr1[i].querySelector('[class^=title-text]') ? arr1[i].querySelector('[class^=title-text]').innerText : '阿里巴巴商品'+new Date().getTime();
|
|
|
+ let priceText = arr1[i].querySelector('[class^=price-item]') ? arr1[i].querySelector('[class^=price-item]').innerText : '';
|
|
|
+ let price = priceText.replace('¥', '').replaceAll('\n', '');
|
|
|
+ let realDom = arr1[i].querySelector('.offer-price-row .offer-desc-item');
|
|
|
+ let realSales = realDom ? realDom.innerText : '';
|
|
|
+ let shopNameText = arr1[i].querySelector('[class^=offer-shop-row]') ? arr1[i].querySelector('[class^=offer-shop-row]').innerText : '';
|
|
|
+ let shopUrl = arr1[i].querySelector('[class^=offer-shop-row] a') ? arr1[i].querySelector('[class^=offer-shop-row] a').href : '';
|
|
|
+ let mainPic = arr1[i].querySelector('img[class^=main-img]') ? arr1[i].querySelector('img[class^=main-img]').src : '';
|
|
|
+
|
|
|
+ let info = {
|
|
|
+ id: id,
|
|
|
+ title: title,
|
|
|
+ href: href,
|
|
|
+ price: price,
|
|
|
+ realSales: realSales,
|
|
|
+ shopNameText: shopNameText,
|
|
|
+ shopUrl: shopUrl,
|
|
|
+ mainPic: mainPic
|
|
|
+ }
|
|
|
+ outObj.urls.push(info);
|
|
|
+ }
|
|
|
+
|
|
|
+ // 广告链接
|
|
|
+ let arr2 = document.querySelectorAll('[class*=page-offerlist] div[class*=search-offer-wrapper]');
|
|
|
+ document.querySelectorAll('[class*=page-offerlist] a[class*=pc-ad]');
|
|
|
+ for(let j=0; j< arr2.length; j++){
|
|
|
+ let href = arr2[j].href;
|
|
|
+ let idData = arr2[j].dataset.renderkey || '';
|
|
|
+ let length = idData.split('_').length - 1;
|
|
|
+ let id = idData.split('_')[length];
|
|
|
+ let title = arr2[j].querySelector('[class^=title-text]') ? arr2[j].querySelector('[class^=title-text]').innerText : '阿里巴巴商品'+new Date().getTime();
|
|
|
+ let priceText = arr2[j].querySelector('[class^=price-item]') ? arr2[j].querySelector('[class^=price-item]').innerText : '';
|
|
|
+ let price = priceText.replace('¥', '').replaceAll('\n', '');
|
|
|
+ let realDom = arr2[j].querySelector('.offer-price-row .offer-desc-item');
|
|
|
+ let realSales = realDom ? realDom.innerText : '';
|
|
|
+ let shopNameText = arr2[j].querySelector('[class^=offer-shop-row]') ? arr2[j].querySelector('[class^=offer-shop-row]').innerText : '';
|
|
|
+ let shopUrl = arr2[j].querySelector('[class^=offer-shop-row] a') ? arr2[j].querySelector('[class^=offer-shop-row] a').href : '';
|
|
|
+ let mainPic = arr2[j].querySelector('img[class^=main-img]') ? arr2[j].querySelector('img[class^=main-img]').src : '';
|
|
|
+
|
|
|
+ let info = {
|
|
|
+ id: id,
|
|
|
+ title: title,
|
|
|
+ href: href,
|
|
|
+ price: price,
|
|
|
+ realSales: realSales,
|
|
|
+ shopNameText: shopNameText,
|
|
|
+ shopUrl: shopUrl,
|
|
|
+ mainPic: mainPic
|
|
|
+ }
|
|
|
+ outObj.urls.push(info);
|
|
|
+ }
|
|
|
+
|
|
|
+ let nextHandle = document.querySelector('.pagination-container .fui-next');
|
|
|
+ if(nextHandle){
|
|
|
+ nextHandle.click({force: true});
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ if(typeStr == 'jd'){ //京东处理
|
|
|
+ let total = document.querySelector('[class*=quick-result]');
|
|
|
+ if(total){
|
|
|
+ let pageContent = total.innerText.replace('\n', '').replace('\n', '');
|
|
|
+ outObj.currentPage =pageContent.split('/')[0] || 1;
|
|
|
+ outObj.totalPage = pageContent.split('/')[1] || 1;
|
|
|
+ outObj.title = searchTitle + '-' + outObj.currentPage;
|
|
|
+ }else{
|
|
|
+ let currentDom = document.querySelector('[class*=pagination_center] [class*=active]');
|
|
|
+ if(currentDom){
|
|
|
+ outObj.currentPage = currentDom.innerText;
|
|
|
+ outObj.title = searchTitle + '-' + outObj.currentPage;
|
|
|
+ }
|
|
|
+ let totalInfo = document.querySelector('[class*=pagination_total]');
|
|
|
+ if(totalInfo){
|
|
|
+ utObj.totalPage = totalInfo.innerText.replace('共', '').replace('页', '');
|
|
|
+ }
|
|
|
+ }
|
|
|
+ // 正常
|
|
|
+ let arr1 = document.querySelectorAll('[class*=goodsContainer]>div');
|
|
|
+ for(let i=0; i< arr1.length; i++){
|
|
|
+ let sku = arr1[i].dataset.sku || '';
|
|
|
+ let title = arr1[i].querySelector('[class*=goods_title]') ? arr1[i].querySelector('[class*=goods_title]').innerText : '京东商品'+new Date().getTime();
|
|
|
+ let priceText = arr1[i].querySelector('[class*=price]') ? arr1[i].querySelector('[class*=price]').innerText : '';
|
|
|
+ let price = priceText.replace('¥', '').replaceAll('\n', '');
|
|
|
+ let realDom = arr1[i].querySelector('[class*=goods_volume]');
|
|
|
+ let realSales = realDom ? realDom.innerText : '';
|
|
|
+ let shopNameText = arr1[i].querySelector('[class*=shopFloor]') ? arr1[i].querySelector('[class*=shopFloor]').innerText : '';
|
|
|
+ let shopUrl = arr1[i].querySelector('[class*=shopFloor] a') ? arr1[i].querySelector('[class*=shopFloor] a').href : '';
|
|
|
+ let mainPic = arr1[i].querySelector('img') ? arr1[i].querySelector('img').src : '';
|
|
|
+
|
|
|
+ let info = {
|
|
|
+ title: title,
|
|
|
+ id: sku,
|
|
|
+ href: "https://item.jd.com/" + sku + ".html",
|
|
|
+ price: price,
|
|
|
+ realSales: realSales,
|
|
|
+ shopNameText: shopNameText,
|
|
|
+ shopUrl: shopUrl,
|
|
|
+ mainPic: mainPic
|
|
|
+ }
|
|
|
+ outObj.urls.push(info);
|
|
|
+ }
|
|
|
+ let nextHandle = document.querySelectorAll('[class*=quick-result] [class*=quick-num]');
|
|
|
+ if(nextHandle && nextHandle.length > 0){
|
|
|
+ nextHandle[1].click({force: true});
|
|
|
+ }else{
|
|
|
+ nextHandle = document.querySelector('[class*=pagination_center] [class*=pagination_next]');
|
|
|
+ if(nextHandle && nextHandle.length > 0){
|
|
|
+ nextHandle.click({force: true});
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return outObj;
|
|
|
+ }, searchTitle, typeStr);
|
|
|
+
|
|
|
+ this[typeStr+'List'].push(imgInfo);
|
|
|
+ currentPage = Number(imgInfo.currentPage);
|
|
|
+ totalPage = Number(imgInfo.totalPage);
|
|
|
+ if(currentPage < totalPage){
|
|
|
+ currentPage += 1;
|
|
|
+ await this.initScroll(typeStr, page, currentPage, totalPage, searchTitle);
|
|
|
+ }else{
|
|
|
+ await this[typeStr+'Browser'].close();
|
|
|
+ this[typeStr+'Browser'] = null;
|
|
|
+ this.addLoading = false;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }, scrollTime);
|
|
|
+ },
|
|
|
+
|
|
|
+
|
|
|
// 检查天猫淘宝登录状态
|
|
|
checkLogin(){
|
|
|
this.checkLoading = true;
|
|
|
@@ -1445,7 +1489,6 @@
|
|
|
});
|
|
|
},
|
|
|
|
|
|
-
|
|
|
// 去登录
|
|
|
loginUrl(url){
|
|
|
(async () => {
|
|
|
@@ -1454,12 +1497,6 @@
|
|
|
this.loginBrowser = null;
|
|
|
}
|
|
|
try{
|
|
|
- if (!fs.existsSync(os.tmpdir() + separator + 'chrome-data-capture')) {
|
|
|
- fs.mkdirSync(os.tmpdir() + separator + 'chrome-data-capture');
|
|
|
- }
|
|
|
- if (!fs.existsSync(os.tmpdir() + separator + 'chrome-data-capture-jd')) {
|
|
|
- fs.mkdirSync(os.tmpdir() + separator + 'chrome-data-capture-jd');
|
|
|
- }
|
|
|
if (!fs.existsSync(os.tmpdir() + separator + 'chrome-data-capture-local')) {
|
|
|
fs.mkdirSync(os.tmpdir() + separator + 'chrome-data-capture-local');
|
|
|
}
|
|
|
@@ -1495,431 +1532,22 @@
|
|
|
})();
|
|
|
},
|
|
|
|
|
|
- // 开始解析
|
|
|
- async startParsing(){
|
|
|
- if(this.formatUrl.trim()){
|
|
|
- let formatUrl = this.formatUrl.trim();
|
|
|
- if(formatUrl.indexOf('http://') > -1){
|
|
|
- formatUrl = formatUrl.replace('http://', 'https://');
|
|
|
- }
|
|
|
-
|
|
|
- if(formatUrl.indexOf('https://') == -1){
|
|
|
- this.$message.error('错了哦，请输入正确的视频地址(https://开头)');
|
|
|
- return false;
|
|
|
- }
|
|
|
- let arr = formatUrl.split('https://');
|
|
|
- formatUrl = 'https://' + arr[1];
|
|
|
- this.downloadUrl = formatUrl;
|
|
|
-
|
|
|
- const regex = /https:\/\/.*?.douyin.com/;
|
|
|
- const res = regex.exec(formatUrl);
|
|
|
- this.bUrl = true;
|
|
|
- if(res && res.length > 0){ //抖音视频解析，使用puputter
|
|
|
- this.selectIndex = -1;
|
|
|
- this.videoList = [];
|
|
|
- let reg2 = /[?&]modal_id=(\w+)/;
|
|
|
- let res2 = reg2.exec(formatUrl);
|
|
|
- if(res2){
|
|
|
- let modal_id = res2[1];
|
|
|
- formatUrl = res[0] + '/video/'+modal_id;
|
|
|
- this.downloadUrl = formatUrl;
|
|
|
- }
|
|
|
- this.douyinParsing(formatUrl);
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- this.parseLoading = true;
|
|
|
- this.tabLoading = true;
|
|
|
- this.selectIndex = -1;
|
|
|
- this.videoList = [];
|
|
|
- let userAgent = [];
|
|
|
- let setingAgent = '';
|
|
|
- if(formatUrl.indexOf('weibo.com/') > -1 && setingAgent){
|
|
|
- userAgent = [
|
|
|
- '--add-header',
|
|
|
- 'User-Agent:'+setingAgent,
|
|
|
- '--add-header',
|
|
|
- "Referer:https://weibo.com/"
|
|
|
- ];
|
|
|
- }
|
|
|
- let params = [
|
|
|
- '--no-playlist',
|
|
|
- '--dump-json',
|
|
|
- ...userAgent,
|
|
|
- formatUrl
|
|
|
- ];
|
|
|
- electronApi.spawnExec(['dlp.exe', ...params]).then(res => {
|
|
|
- this.parseLoading = false;
|
|
|
- this.tabLoading = false;
|
|
|
- let info = res.stdout ? res.stdout.toString() : '{formats: []}';
|
|
|
- this.videoInfo = JSON.parse(info);
|
|
|
- this.videoList = this.videoInfo.formats || [];
|
|
|
- const bregex = /https:\/\/.*?.bilibili.com/;
|
|
|
- const bres = bregex.exec(formatUrl);
|
|
|
- if(bres && bres.length > 0){ //b站的视频
|
|
|
- this.bUrl = false;
|
|
|
- }else{
|
|
|
- this.bUrl = true;
|
|
|
- }
|
|
|
- this.biliAudioList = [];
|
|
|
- this.biliAudioIndex = -1;
|
|
|
- this.biliVideoList = [];
|
|
|
- this.biliVideoIndex = -1;
|
|
|
- this.videoList.map(item => {
|
|
|
- item.title = this.videoInfo.title,
|
|
|
- item.status = '1';
|
|
|
- if(!this.bUrl){ // b站的视频
|
|
|
- if(item.resolution.indexOf('audio') > -1 || item.audio_ext != 'none'){
|
|
|
- this.biliAudioList.push(item);
|
|
|
- }else{
|
|
|
- this.biliVideoList.push(item);
|
|
|
- }
|
|
|
- }
|
|
|
- })
|
|
|
- }).catch(err =>{
|
|
|
- this.parseLoading = false;
|
|
|
- this.tabLoading = false;
|
|
|
- // console.log('err1',err.stderr.toString());
|
|
|
- let errStr = err.stderr.toString();
|
|
|
- this.bUrl = true;
|
|
|
- if(errStr.indexOf('Unsupported URL') > -1){
|
|
|
- this.videoParsing(formatUrl);
|
|
|
- return false;
|
|
|
- }else if(formatUrl.indexOf('xiaohongshu.com/') > -1 && errStr.indexOf('No video formats') > -1){
|
|
|
- this.$notify.error({
|
|
|
- title: '设置中登录小红书账号下载！',
|
|
|
- message: errStr
|
|
|
- });
|
|
|
- this.videoParsing(formatUrl);
|
|
|
- return false;
|
|
|
- }else{
|
|
|
- this.$notify.error({
|
|
|
- title: '网址解析失败！',
|
|
|
- message: errStr
|
|
|
- });
|
|
|
- }
|
|
|
- })
|
|
|
- }
|
|
|
- },
|
|
|
- // 复制链接地址
|
|
|
- copyText(text){
|
|
|
- navigator.clipboard.writeText(text).then(() => {
|
|
|
- this.$message({message: '视频下载链接已成功复制到剪贴板', type: 'success'});
|
|
|
- }).catch(err => {
|
|
|
- this.$message.error('无法复制文本', err.toString());
|
|
|
- });
|
|
|
- },
|
|
|
- // 点击复制
|
|
|
- copy(index){
|
|
|
- this.commonUrl = '';
|
|
|
- this.thunderUrl = [];
|
|
|
- let tag = this.videoList[index].tag;
|
|
|
- let urlList = this.videoList[index].urlList;
|
|
|
- let title = this.videoList[index].title;
|
|
|
- let text = urlList[0];
|
|
|
- if(urlList.length <= 0){
|
|
|
- this.$message.error('无法复制下载地址');
|
|
|
- return false;
|
|
|
- }
|
|
|
- if(tag == 'douyin'){
|
|
|
- this.dyModal = true;
|
|
|
- this.dyIndex = index;
|
|
|
-
|
|
|
- urlList.map(uitem => {
|
|
|
- if(uitem.startsWith('https://www.douyin.com/')){
|
|
|
- this.commonUrl = uitem;
|
|
|
- }
|
|
|
- if(!title){
|
|
|
- title = '抖音视频';
|
|
|
- }
|
|
|
- // console.log(`AA`+uitem +`&filename=`+title+`.mp4ZZ`);
|
|
|
- let urlBuffer = Buffer.from(`AA`+uitem +`&filename=`+title+`.mp4ZZ`).toString('base64');
|
|
|
- this.thunderUrl.push('thunder://'+urlBuffer);
|
|
|
- })
|
|
|
- }else{
|
|
|
- this.copyText(text);
|
|
|
- }
|
|
|
- },
|
|
|
-
|
|
|
- // 解析抖音视频
|
|
|
- async douyinParsing(url){
|
|
|
- if(this.loginBrowser){
|
|
|
- await this.loginBrowser.close();
|
|
|
- this.loginBrowser = null;
|
|
|
- }
|
|
|
- if(this.videoBrowser){
|
|
|
- await this.videoBrowser.close();
|
|
|
- this.videoBrowser = null;
|
|
|
- }
|
|
|
- if (!fs.existsSync(os.tmpdir() + separator + 'chrome-data-capture-video')) {
|
|
|
- fs.mkdirSync(os.tmpdir() + separator + 'chrome-data-capture-video');
|
|
|
- }
|
|
|
-
|
|
|
- this.parseLoading = true;
|
|
|
- this.tabLoading = true;
|
|
|
-
|
|
|
- setTimeout(()=> {
|
|
|
- this.parseLoading = false;
|
|
|
- this.tabLoading = false;
|
|
|
- }, 20000)
|
|
|
-
|
|
|
- let userDataDir = os.tmpdir() + separator + 'chrome-data-capture-video';
|
|
|
- // 运行不同平台的浏览器
|
|
|
- puppeteer.use(StealthPlugin());
|
|
|
- let headless = true;
|
|
|
- headless = this.initDevelop().headless;
|
|
|
- this.videoBrowser = await puppeteer.launch({
|
|
|
- headless: headless,
|
|
|
- executablePath: this.initPath(),
|
|
|
- userDataDir: userDataDir,
|
|
|
- args: [
|
|
|
- '--start-maximized',
|
|
|
- '--no-sandbox',
|
|
|
- '--disable-setuid-sandbox',
|
|
|
- '--disable-blink-features=AutomationControlled',
|
|
|
- ]
|
|
|
- });
|
|
|
-
|
|
|
- await new Promise((resolve,reject) =>{
|
|
|
- (async () => {
|
|
|
- try{
|
|
|
- let authority = this.$refs.headerRef.authority.isAuthority;
|
|
|
- const page = await this.videoBrowser.newPage();
|
|
|
-
|
|
|
- let responseVideo = [];
|
|
|
- let responseUrl = [];
|
|
|
- let responseObj = {};
|
|
|
- let vtitle = '视频';
|
|
|
- page.on('response', async(response) => {
|
|
|
- // 检查响应的 MIME 类型是否以 'video/' 开头
|
|
|
- if (response.headers()['content-type'] && response.headers()['content-type'].startsWith('video/')) {
|
|
|
- if(responseVideo.indexOf(response.url()) < 0 && !response.url().startsWith('blob:https://')){
|
|
|
- responseVideo.push(response.url());
|
|
|
- vtitle = await page.title();
|
|
|
- if(vtitle){
|
|
|
- vtitle = vtitle.substring(0, 50);
|
|
|
- if(this.containsAnyChar(vtitle)){ //判断是否含有特殊字符
|
|
|
- vtitle = vtitle.replace(/[\\/:*?"<>|#%&\s]/g, "");
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- if (response.headers()['content-type'] && response.headers()['content-type'].startsWith('application/json')) {
|
|
|
- if(response.url().indexOf('/aweme/detail/') > -1){
|
|
|
- if(responseUrl.indexOf(response.url()) < 0){
|
|
|
- responseUrl.push(response.url());
|
|
|
- }else{
|
|
|
- return false;
|
|
|
- }
|
|
|
-
|
|
|
- let jsonText = await response.text();
|
|
|
- if(jsonText && typeof jsonText == 'string'){
|
|
|
- responseObj = JSON.parse(jsonText);
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- });
|
|
|
- let waitUntil = 'networkidle2';
|
|
|
- waitUntil = this.initDevelop().waitUntil;
|
|
|
- await page.goto(url, {waitUntil : waitUntil});
|
|
|
- await page.waitForTimeout(1000);
|
|
|
- await this.videoBrowser.close();
|
|
|
-
|
|
|
- if(responseObj['aweme_detail']){ // 返回的接口数据中有aweme_detail参数 才会解析接口数据
|
|
|
- let arr = ['aweme_detail']; //'video' ,'play_addr', 'url_list', '2'];
|
|
|
- for(let i = 0; i < arr.length; i++){
|
|
|
- responseObj = responseObj[arr[i]];
|
|
|
- }
|
|
|
-
|
|
|
- if(responseObj && responseObj.preview_title){
|
|
|
- responseObj.preview_title = responseObj.preview_title.substring(0, 50);
|
|
|
- if(this.containsAnyChar(responseObj.preview_title)){ //判断是否含有特殊字符
|
|
|
- responseObj.preview_title = responseObj.preview_title.replace(/[\\/:*?"<>|#%&\s]/g, "");
|
|
|
- }
|
|
|
- }
|
|
|
- if(responseObj && responseObj.video && responseObj.video.play_addr){
|
|
|
- let vinfo = {
|
|
|
- title: responseObj.preview_title,
|
|
|
- tag: 'douyin',
|
|
|
- format_id: 'default',
|
|
|
- ext: 'mp4',
|
|
|
- resolution: responseObj.video.play_addr.width + 'x' + responseObj.video.play_addr.height,
|
|
|
- fps: '-',
|
|
|
- filesize: responseObj.video.play_addr.data_size,
|
|
|
- vcodec: '-',
|
|
|
- acodec: '-',
|
|
|
- urlList: responseObj.video.play_addr.url_list,
|
|
|
- status: '1',
|
|
|
- loading: false
|
|
|
- }
|
|
|
- this.videoList.push(vinfo);
|
|
|
- }
|
|
|
- if(responseObj && responseObj.video && responseObj.video.play_addr_265){
|
|
|
- let vinfo = {
|
|
|
- title: responseObj.preview_title,
|
|
|
- tag: 'douyin',
|
|
|
- format_id: 'play_addr_265',
|
|
|
- ext: 'mp4',
|
|
|
- resolution: responseObj.video.play_addr_265.width + 'x' + responseObj.video.play_addr_265.height,
|
|
|
- fps: '-',
|
|
|
- filesize: responseObj.video.play_addr_265.data_size,
|
|
|
- vcodec: '-',
|
|
|
- acodec: '-',
|
|
|
- urlList: responseObj.video.play_addr_265.url_list,
|
|
|
- status: '1',
|
|
|
- loading: false
|
|
|
- }
|
|
|
- this.videoList.push(vinfo);
|
|
|
- }
|
|
|
- if(responseObj && responseObj.video && responseObj.video.play_addr_h264){
|
|
|
- let vinfo = {
|
|
|
- title: responseObj.preview_title,
|
|
|
- tag: 'douyin',
|
|
|
- format_id: 'play_addr_h264',
|
|
|
- ext: 'mp4',
|
|
|
- resolution: responseObj.video.play_addr_h264.width + 'x' + responseObj.video.play_addr_h264.height,
|
|
|
- fps: '-',
|
|
|
- filesize: responseObj.video.play_addr_h264.data_size,
|
|
|
- vcodec: '-',
|
|
|
- acodec: '-',
|
|
|
- urlList: responseObj.video.play_addr_h264.url_list,
|
|
|
- status: '1',
|
|
|
- loading: false
|
|
|
- }
|
|
|
- this.videoList.push(vinfo);
|
|
|
- }
|
|
|
- }else if(responseVideo.length > 0){ //网页解析到视频地址
|
|
|
- let vinfo = {
|
|
|
- title: vtitle,
|
|
|
- tag: 'douyin',
|
|
|
- format_id: 'video',
|
|
|
- ext: 'mp4',
|
|
|
- resolution: '-',
|
|
|
- fps: '-',
|
|
|
- filesize: '',
|
|
|
- vcodec: '-',
|
|
|
- acodec: '-',
|
|
|
- urlList: responseVideo,
|
|
|
- status: '1',
|
|
|
- loading: false
|
|
|
- }
|
|
|
- this.videoList.push(vinfo);
|
|
|
- }
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
- this.parseLoading = false;
|
|
|
- this.tabLoading = false;
|
|
|
- }catch(e){
|
|
|
- this.parseLoading = false;
|
|
|
- this.tabLoading = false;
|
|
|
- reject(e);
|
|
|
- this.showError(e);
|
|
|
- }
|
|
|
- })();
|
|
|
- });
|
|
|
-
|
|
|
- },
|
|
|
-
|
|
|
- // 下载网址链接的图片
|
|
|
- async downloadImage(imageUrl, outputPath, urlInfo) {
|
|
|
- let _this = this;
|
|
|
- let received_bytes = 0;
|
|
|
- let total_bytes = 0;
|
|
|
- try {
|
|
|
- let req = request({
|
|
|
- method: 'GET', uri: imageUrl, strictSSL: false
|
|
|
- });
|
|
|
- let out = fs.createWriteStream(outputPath);
|
|
|
- req.pipe(out);
|
|
|
-
|
|
|
- return new Promise((resolve, reject) => {
|
|
|
- req.on('response', (data) => {
|
|
|
- total_bytes = parseInt(data.headers['content-length']);
|
|
|
- const status = data.statusCode;
|
|
|
- if(this.menuIndex != '1'){
|
|
|
- if (status < 200 || status >= 300) {
|
|
|
- this.$notify.error({
|
|
|
- title: '网络资源访问异常！- 1',
|
|
|
- message: imageUrl.slice(0,50)
|
|
|
- });
|
|
|
- }else if(isNaN(total_bytes)){
|
|
|
- this.$notify.error({
|
|
|
- title: '网络资源访问异常！- 2',
|
|
|
- message: imageUrl.slice(0,50)
|
|
|
- });
|
|
|
- }else{
|
|
|
- // console.log('下载中...')
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- if(status == 403 || isNaN(total_bytes)){
|
|
|
- fs.unlinkSync(outputPath);
|
|
|
- urlInfo.status = '4';
|
|
|
- }
|
|
|
-
|
|
|
- });
|
|
|
-
|
|
|
- req.on('data', (chunk) => {
|
|
|
- received_bytes += chunk.length;
|
|
|
- if(urlInfo.filesize){
|
|
|
- let size = Number(urlInfo.filesize);
|
|
|
- let percent = Number(received_bytes / size * 100);
|
|
|
- urlInfo.percent = percent.toFixed(2);
|
|
|
- if(percent > 100){
|
|
|
- urlInfo.percent = 100;
|
|
|
- }
|
|
|
- this.$forceUpdate();
|
|
|
- }
|
|
|
- });
|
|
|
-
|
|
|
- req.on('end', ()=> {
|
|
|
- urlInfo.status = '3';
|
|
|
- //console.log('下载完成', outputPath)
|
|
|
- resolve(true);
|
|
|
- });
|
|
|
- });
|
|
|
- } catch (error) {
|
|
|
- urlInfo.status = '4';
|
|
|
- console.error(imageUrl, `Failed to download image: ${error.message}`);
|
|
|
- throw error;
|
|
|
- }
|
|
|
- },
|
|
|
-
|
|
|
// 获取页面标题 - 生成对应的文件夹
|
|
|
- async getTitle(page, urlInfo){
|
|
|
+ async getTitle(page, str){
|
|
|
// 已页面标题作为新建文件夹,保留前50个字
|
|
|
let title = await page.title();
|
|
|
if(title){
|
|
|
- title = title.substring(0, 50);
|
|
|
- if(this.containsAnyChar(title)){ //判断是否含有特殊字符
|
|
|
- title = title.replace(/[\\/:*?"<>|#%&\s]/g, "");
|
|
|
- }
|
|
|
- if (fs.existsSync(this.downloadDir + separator + pjson.softInfo.softName + separator + title)) {
|
|
|
- urlInfo.newPath = this.downloadDir + separator + pjson.softInfo.softName + separator + title;
|
|
|
- } else {
|
|
|
- fs.mkdirSync(this.downloadDir + separator + pjson.softInfo.softName + separator + title);
|
|
|
- urlInfo.newPath = this.downloadDir + separator + pjson.softInfo.softName + separator + title;
|
|
|
+ title = title.substring(0, 50).trim();
|
|
|
+ if(this.containsAnyChar(title, ['\\', '/', ':', '*', '?', '"', '<', '>', '|', '.'])){ //判断是否含有特殊字符
|
|
|
+ title = title.replace(/[\\|/|:|*|?|.|"|<|>||]/g, "");
|
|
|
}
|
|
|
+ return title;
|
|
|
+ }else{
|
|
|
+ let t = str ? str : '网页无标题-' + new Date().getTime();
|
|
|
+ return t;
|
|
|
}
|
|
|
},
|
|
|
|
|
|
- // 下载base64位的图片
|
|
|
- async downloadBaseImage(base64String, outputPath, urlInfo) {
|
|
|
- const buffer = Buffer.from(base64String, 'base64');
|
|
|
- const writeStream = fs.createWriteStream(outputPath);
|
|
|
- writeStream.write(buffer);
|
|
|
- writeStream.end();
|
|
|
- writeStream.on('finish', () => {
|
|
|
- urlInfo.num += 1;
|
|
|
- });
|
|
|
- // 监听错误事件
|
|
|
- writeStream.on('error', (err) => {
|
|
|
- console.error('base64位写入文件时出错:', err);
|
|
|
- });
|
|
|
- },
|
|
|
-
|
|
|
// 错误提示
|
|
|
showError(e){
|
|
|
let str = '';
|
