save

node/user/process-user-img.js

@@ -0,0 +1,82 @@
+import fs from 'fs'
+import request from 'request'
+import {nanoid} from 'nanoid'
+
+let fileName = './user.json'
+let saveFileStr = fs.readFileSync(fileName, "utf8");
+let inputData = JSON.parse(saveFileStr);
+
+
+const downloadImage = async (src, dest, callback) => {
+  console.log('下载:', src, dest, Date.now())
+  return new Promise(resolve => {
+    request.head(src, (err, res, body) => {
+      if (err) {
+        console.log(err);
+        return;
+      }
+      src &&
+      request(src)
+        .pipe(fs.createWriteStream(dest))
+        .on("close", () => {
+          setTimeout(resolve, 1000)
+          callback && callback(null, dest);
+        });
+    });
+  })
+};
+
+let saveFilePath = './user-imgs/'
+
+async function sleep(val) {
+  return new Promise(resolve => {
+    setTimeout(resolve, val)
+  })
+}
+
+async function test(list) {
+  for (let j = 0; j < list.length; j++) {
+    let a = list[j]
+    let coverUrl = a.url_list[a.url_list.length - 1]
+    a.url_list = [coverUrl]
+    if (coverUrl.includes('http')) {
+      let name = nanoid() + '.png'
+      await downloadImage(coverUrl, saveFilePath + name, () => {
+        // console.log('close', name)
+        a.url_list = [name]
+        fs.writeFileSync(fileName, JSON.stringify(inputData, null, 2));
+      })
+    }
+  }
+}
+
+async function test2(list) {
+  for (let j = 0; j < list.length; j++) {
+    let coverUrl = list[list.length - 1]
+    if (coverUrl.includes('http')) {
+      let name = nanoid() + '.png'
+      await downloadImage(coverUrl, saveFilePath + name, () => {
+        // console.log('close', name)
+        fs.writeFileSync(fileName, JSON.stringify(inputData, null, 2));
+      })
+      return [name]
+    }
+    return []
+  }
+}
+
+for (let i = 0; i < inputData.slice(0, 1111).length; i++) {
+  let v = inputData[i]
+  await test(v.cover_url)
+  await test(v.white_cover_url)
+  delete v.cover_and_head_image_info
+  fs.writeFileSync(fileName, JSON.stringify(inputData, null, 2));
+  if (v.share_info && v.share_info.share_image_url && v.share_info.share_image_url.url_list) {
+    let r = await test2(v.share_info.share_image_url.url_list)
+    if (r.length) {
+      v.share_info.share_image_url.url_list = r
+      fs.writeFileSync(fileName, JSON.stringify(inputData, null, 2));
+    }
+  }
+}
+

node/user/process-user.js

@@ -0,0 +1,63 @@
+import fs from 'fs'
+
+let fileName = './data.json'
+let saveFileStr = fs.readFileSync(fileName, "utf8");
+let inputData = JSON.parse(saveFileStr);
+
+
+inputData.map(v => {
+  Object.keys(v).map(k => {
+    if (![
+      'avatar_168x168',
+      'avatar_300x300',
+      'aweme_count',
+      'birthday_hide_level',
+      'can_show_group_card',
+      'commerce_user_level',
+      'province',
+      'city',
+      'country',
+      'district',
+      'favoriting_count',
+      'follow_status',
+      'follower_count',
+      'follower_request_status',
+      'follower_status',
+      'following_count',
+      'forward_count',
+      'public_collects_count',
+      'share_info',
+      'mplatform_followers_count',
+      'nickname',
+      'max_follower_count',
+      'gender',
+      'cover_colour',
+      'cover_url',
+      'commerce_info',
+      'commerce_user_info',
+      'short_id',
+      'signature',
+      'total_favorited',
+      'uid',
+      'unique_id',
+      'user_age',
+      'white_cover_url',
+      'card_entries',
+      'cover_and_head_image_info',
+      'ip_location',
+    ].includes(k)) {
+      delete v[k]
+    }
+  })
+})
+
+// saveFileData = saveFileData.concat(inputData.aweme_list)
+// fs.writeFileSync(
+//   "./ouput.json",
+//   JSON.stringify(inputData.aweme_list, null, 2)
+// );
+fs.writeFileSync(
+  // fileName,
+  './data.json',
+  JSON.stringify(inputData, null, 2)
+);