Refactor aggregateDaily to further reduce memory usage

src/aggregateDaily.ts

@@ -1,4 +1,4 @@
-import {aggregateSpeedscopeData} from "./repositories/profileRepository.js";
+import {aggregateSpeedscopeData, AggregationResult} from "./repositories/profileRepository.js";
 import {AggregatedProfileType} from "../generated/prisma/enums.js";
 import {gzipSync} from "node:zlib";
 import {prisma} from "./prisma.js";
@@ -8,40 +8,60 @@ import config from "./config/config.js";
 const end = new Date();
 const start = new Date(end.getTime() - (24 * 60 * 60 * 1000)); // 1 day ago
 
-const aggregatedProfiles: AggregatedProfile[] = await prisma.aggregatedProfile.findMany({
-  where: {
-    startTime: {
-      gte: start,
+const aggregateData = async ( start: Date, end: Date ) => {
+  const aggregatedProfiles: AggregatedProfile[] = await prisma.aggregatedProfile.findMany({
+    where: {
+      startTime: {
+        gte: start,
+      },
+      type: AggregatedProfileType.HOURLY,
     },
-    type: AggregatedProfileType.HOURLY,
-  },
-  orderBy: {
-    startTime: 'asc',
-  },
-});
+    orderBy: {
+      startTime: 'asc',
+    },
+  });
 
-if (!aggregatedProfiles || aggregatedProfiles.length === 0) {
-  console.log('No profiles found in the last day.');
-  process.exit(0);
-}
+  if (!aggregatedProfiles || aggregatedProfiles.length === 0) {
+    console.log('No profiles found in the last day.');
+    process.exit(0);
+  }
 
-const aggregatedData = aggregateSpeedscopeData(
-    aggregatedProfiles,
-    `Daily aggregation (${start.toISOString()} to ${end.toISOString()})`
-);
+  const aggregatedData = aggregateSpeedscopeData(
+      aggregatedProfiles,
+      `Daily aggregation (${start.toISOString()} to ${end.toISOString()})`
+  );
 
-console.log('Converting data to JSON...');
+  const profileCount = aggregatedProfiles
+    .map((p) => p.profileCount)
+    .reduce((a, b) => a + b, 0);
 
-const profileJson = JSON.stringify(aggregatedData.file);
-delete aggregatedData.file;
+  return { aggregatedData: aggregatedData, profileCount: profileCount };
+};
 
-const frameTimingJson = JSON.stringify(aggregatedData.frameTimings, (k, v) => {
-  if (v instanceof Map) {
-    return Array.from(v.entries());
-  }
-  return v;
-});
-delete aggregatedData.frameTimings;
+const compressFrameTimings = ( aggregatedData: AggregationResult ) => {
+  console.log('Converting frame timings to JSON...');
+  const frameTimingJson = JSON.stringify(aggregatedData.frameTimings, (k, v) => {
+    if (v instanceof Map) {
+      return Array.from(v.entries());
+    }
+    return v;
+  });
+  delete aggregatedData.frameTimings;
+  console.log('Compressing frame timings...');
+  return gzipSync(frameTimingJson);
+};
+
+const compressProfile = ( aggregatedData: AggregationResult ) => {
+  console.log('Converting profile to JSON...');
+  const profileJson = JSON.stringify(aggregatedData.file);
+  delete aggregatedData.file;
+  console.log('Compressing profile...');
+  return gzipSync(profileJson);
+};
+
+const { aggregatedData, profileCount } = await aggregateData( start, end );
+const compressedProfile = compressProfile( aggregatedData );
+const compressedFrameTimings = compressFrameTimings( aggregatedData );
 
 console.log('Writing data to DB...');
 
@@ -50,11 +70,9 @@ await prisma.aggregatedProfile.create({
     startTime: start,
     endTime: end,
     type: AggregatedProfileType.DAILY,
-    profileCount: aggregatedProfiles
-      .map((p) => p.profileCount)
-      .reduce((a, b) => a + b, 0),
-    speedscopeData: gzipSync(profileJson),
-    frameTimingData: gzipSync(frameTimingJson),
+    profileCount: profileCount,
+    speedscopeData: compressedProfile,
+    frameTimingData: compressedFrameTimings,
   }
 });