#!/bin/bash
# 5090-1 comfyui-h3 프로파일: jobs_served 에 대한 메모리·성능 추이
# ★리사이클을 끄지 않는다 — 자연 세션(1→30잡)을 그대로 훑어 heap 누적 유무만 본다.
OUT=~/ai/experiments/dockaiq-recycle-threshold/samples.tsv
[ -s "$OUT" ] || printf 'ts\tpid\trss_mb\tetime_s\tgpu_mb\tuptime_txt\tjobs_served\n' > "$OUT"
for i in $(seq 1 60); do
  P=$(ssh -o BatchMode=yes 5090-1 'ps -eo pid,rss,etimes,args --sort=-rss | grep -i "[c]omfy" | head -1 | awk "{print \$1\"\t\"\$2\"\t\"\$3}"; nvidia-smi --query-compute-apps=pid,used_memory --format=csv,noheader | head -1 | awk -F", " "{print \$2}" | tr -d " MiB"' 2>/dev/null | tr '\n' '\t')
  S=$(dockaiq status 2>/dev/null | grep -A1 "5090-1  :" | grep -o "uptime [0-9]*[a-z]*.*jobs served" | head -1)
  U=$(echo "$S" | grep -o "uptime [0-9]*[a-z]*")
  J=$(echo "$S" | grep -o "· [0-9]* jobs served" | grep -o "[0-9]*")
  RSSMB=$(echo "$P" | awk -F'\t' '{printf "%.0f", $2/1024}')
  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$(date +%H:%M:%S)" "$(echo "$P"|cut -f1)" "$RSSMB" "$(echo "$P"|cut -f3)" "$(echo "$P"|cut -f4)" "$U" "$J" >> "$OUT"
  sleep 30
done