From 7ebd58ed3364291b4b3aef3da273674085c7bfaf Mon Sep 17 00:00:00 2001 From: sal-uva <10960315+sal-uva@users.noreply.github.com> Date: Thu, 10 Sep 2026 22:58:12 +0000 Subject: [PATCH] chore: sync map_item for tiktok from 4CAT c6ac6d473be8987bc853d0aaa0f6debdc7e95dfd --- modules/tiktok.js | 170 ++++++++++++++++++++++++++-------------------- 1 file changed, 95 insertions(+), 75 deletions(-) diff --git a/modules/tiktok.js b/modules/tiktok.js index 586c9e1..11370bd 100644 --- a/modules/tiktok.js +++ b/modules/tiktok.js @@ -107,94 +107,114 @@ export function capture(response, source_platform_url, source_url) { // === auto-generated by 4cat map_item sync — BLOCK REPLACED AUTOMATICALLY === // (regenerated from datasources/tiktok/search_tiktok.py) -export function map_item(post) { +function getValidThumbnail(options) { + const now = Math.floor(Date.now() / 1000); + const valid = options.filter(url => { + try { + const parsed = new URL(url); + const expiresStr = parsed.searchParams.get('x-expires'); + const expires = expiresStr ? parseInt(expiresStr, 10) : now; + return expires >= now; + } catch (e) { + return false; + } + }); + return valid.length ? valid.pop() : ""; +} + +export function map_item(item) { + const post = item; // Zeeschuimer metadata - const metadata = post["__import_meta"] ?? {}; + const metadata = py_get(post, '__import_meta', {}); - const challenges = (post["challenges"] ?? []).map(ch => ch.title); + // Challenges titles + const challenges = (py_get(post, 'challenges', [])).map(ch => ch.title); - const hashtags = (post["textExtra"] ?? []).filter(extra => "hashtagName" in extra && extra.hashtagName).map(extra => extra.hashtagName); + // Hashtags extraction + const hashtags = (py_get(post, 'textExtra', [])).filter(extra => 'hashtagName' in extra && extra.hashtagName).map(extra => extra.hashtagName); - const labels = Array.isArray(post["diversificationLabels"]) ? post["diversificationLabels"].join(",") : ""; + // Diversification labels + const divLabels = py_get(post, 'diversificationLabels'); + const labels = Array.isArray(divLabels) ? divLabels.join(',') : ''; - let user_nickname = ""; - let user_fullname = ""; - let user_thumbnail = ""; - if (post["author"] && typeof post["author"] === "object") { + // Author handling + const authorField = py_get(post, 'author'); + let user_nickname = ''; + let user_fullname = ''; + let user_thumbnail = ''; + if (authorField && typeof authorField === 'object' && !Array.isArray(authorField)) { // from intercepted API response - user_nickname = post["author"]["uniqueId"] ?? ""; - user_fullname = post["author"]["nickname"] ?? ""; - user_thumbnail = post["author"]["avatarThumb"] ?? ""; - } else if (post["author"]) { + user_nickname = authorField.uniqueId ?? ''; + user_fullname = authorField.nickname ?? ''; + user_thumbnail = py_get(authorField, 'avatarThumb', ''); + } else if (authorField) { // from embedded JSON object - user_nickname = post["author"] ?? ""; - user_fullname = post["nickname"] ?? ""; - user_thumbnail = ""; + user_nickname = authorField; + user_fullname = py_get(post, 'nickname', ''); + user_thumbnail = ''; } - // Determine the best thumbnail URL that hasn't expired yet - const thumbnail_options = []; - if (post["video"]?.shareCover) { - const shareCover = post["video"]["shareCover"]; - if (Array.isArray(shareCover) && shareCover.length) { - thumbnail_options.push(shareCover[shareCover.length - 1]); - } else if (typeof shareCover === "string") { - thumbnail_options.push(shareCover); - } + // Thumbnail selection + const video = py_get(post, 'video', {}); + const thumbnailOptions = []; + const shareCover = py_get(video, 'shareCover'); + if (Array.isArray(shareCover) && shareCover.length) { + thumbnailOptions.push(shareCover[shareCover.length - 1]); } - if (post["video"]?.cover) { - thumbnail_options.push(post["video"]["cover"]); + const cover = py_get(video, 'cover'); + if (cover) { + thumbnailOptions.push(cover); } - const now = Math.floor(Date.now() / 1000); - const validThumbnails = thumbnail_options.filter(url => { - if (!url) return false; - try { - const expiresStr = new URL(url).searchParams.get("x-expires"); - const expires = expiresStr ? parseInt(expiresStr, 10) : now; - return expires >= now; - } catch (e) { - return false; - } - }); - const thumbnail_url = validThumbnails.length ? validThumbnails[validThumbnails.length - 1] : ""; + const thumbnail_url = getValidThumbnail(thumbnailOptions); + + // Stats and other fields + const authorStats = py_get(post, 'authorStats', {}); + const stats = py_get(post, 'stats', {}); + const music = py_get(post, 'music', {}); + const effectStickers = py_get(post, 'effectStickers', []); + const warnInfo = py_get(post, 'warnInfo', []); + const stickersOnItem = py_get(post, 'stickersOnItem', []); + + const timestampSec = parseInt(post.createTime, 10); return new MappedItem({ - "collected_from_url": metadata["source_platform_url"] ? normalize_url_encoding(metadata["source_platform_url"]) : "", - "id": post["id"], - "thread_id": post["id"], - "author": user_nickname, - "author_full": user_fullname, - "author_followers": post["authorStats"]?.followerCount ?? "", - "author_likes": post["authorStats"]?.diggCount ?? "", - "author_videos": post["authorStats"]?.videoCount ?? "", - "author_avatar": user_thumbnail, - "body": post["desc"], - "stickers": (post["stickersOnItem"] ?? []).map(s => s.stickerText.join(" ")).join("\n"), - "timestamp": formatUtcTimestamp(parseInt(post["createTime"], 10)), - "unix_timestamp": parseInt(post["createTime"], 10), - "is_duet": (post["duetInfo"]?.duetFromId && post["duetInfo"]["duetFromId"] !== "0") ? "yes" : "no", - "is_ad": post["isAd"] ? "yes" : "no", - "is_paid_partnership": post["adAuthorization"] ? "yes" : "no", - "is_sensitive": post["maskType"] === 3 ? "yes" : "no", - "is_photosensitive": post["maskType"] === 4 ? "yes" : "no", - "music_name": post["music"]?.title ?? "", - "music_id": post["music"]?.id ?? "", - "music_url": post["music"]?.playUrl ?? "", - "music_thumbnail": post["music"]?.coverLarge ?? "", - "music_author": post["music"]?.authorName ?? "", - "video_url": post["video"]?.downloadAddr ?? "", - "tiktok_url": `https://www.tiktok.com/@${user_nickname}/video/${post["id"]}`, - "thumbnail_url": thumbnail_url, - "likes": post["stats"]?.diggCount, - "comments": post["stats"]?.commentCount, - "shares": post["stats"]?.shareCount, - "plays": post["stats"]?.playCount, - "hashtags": hashtags.join(","), - "challenges": challenges.join(","), - "diversification_labels": labels, - "location_created": post["locationCreated"] ?? "", - "effects": (post["effectStickers"] ?? []).map(e => e.name).join(","), - "warning": (post["warnInfo"] ?? []).map(w => w.text).join(",") + collected_from_url: metadata && py_get(metadata, 'source_platform_url') ? normalize_url_encoding(py_get(metadata, 'source_platform_url')) : "", + id: post.id, + thread_id: post.id, + author: user_nickname, + author_full: user_fullname, + author_followers: py_get(authorStats, 'followerCount', ''), + author_likes: py_get(authorStats, 'diggCount', ''), + author_videos: py_get(authorStats, 'videoCount', ''), + author_avatar: user_thumbnail, + body: post.desc, + stickers: stickersOnItem.map(s => (s.stickerText || []).join(' ')).join('\n'), + timestamp: formatUtcTimestamp(timestampSec), + unix_timestamp: timestampSec, + is_image_post: py_get(post, 'imagePost') ? "yes" : "no", + is_duet: (function(){ const duetInfo = py_get(post, 'duetInfo', {}); return duetInfo && duetInfo.duetFromId && duetInfo.duetFromId !== "0" ? "yes" : "no"; })(), + is_ad: py_get(post, 'isAd', false) ? "yes" : "no", + is_paid_partnership: py_get(post, 'adAuthorization') ? "yes" : "no", + is_sensitive: py_get(post, 'maskType') === 3 ? "yes" : "no", + is_photosensitive: py_get(post, 'maskType') === 4 ? "yes" : "no", + music_name: music.title, + music_id: music.id, + music_url: py_get(music, 'playUrl', ''), + music_thumbnail: py_get(music, 'coverLarge', ''), + music_author: py_get(music, 'authorName', ''), + video_url: py_get(video, 'downloadAddr', ''), + tiktok_url: `https://www.tiktok.com/@${user_nickname}/video/${post.id}`, + thumbnail_url: thumbnail_url, + likes: py_get(stats, 'diggCount'), + comments: py_get(stats, 'commentCount'), + shares: py_get(stats, 'shareCount'), + plays: py_get(stats, 'playCount'), + hashtags: hashtags.join(','), + challenges: challenges.join(','), + diversification_labels: labels, + location_created: py_get(post, 'locationCreated', ''), + effects: effectStickers.map(e => e.name).join(','), + warning: warnInfo.map(w => w.text).join(',') }); } // === end auto-generated ===