1
YoutubeLearnAStock/
2
├── yt_asr.sh
3
├── out/
4
│   ├── *.m4a
5
│   ├── *.zh.srt
6
│   ├── *.zh.txt
7
│   └── logs/
8
└── README.md

1
#!/usr/bin/env bash
2
# yt_asr.sh - Final version + urls.txt batch mode
3
# Supports:
4
#   1) URLs / local audio files as args
5
#   2) --urls-file urls.txt  (one URL per line)
6
# Robust for Ubuntu + pyenv + faster-whisper
7

8
set -euo pipefail
9

10
# -----------------------------
11
# Defaults
12
# -----------------------------
13
OUTDIR="${OUTDIR:-./out}"
14
ASR_LANG="${ASR_LANG:-zh}"          # zh / en / ja / ...
15
MODEL="${MODEL:-large-v3}"
16
DEVICE="${DEVICE:-cpu}"             # cpu | cuda
17
COMPUTE="${COMPUTE:-int8}"
18
BROWSER="${BROWSER:-chrome}"
19
USE_COOKIES="${USE_COOKIES:-1}"
20
USE_REMOTE_COMPONENTS="${USE_REMOTE_COMPONENTS:-1}"
21
KEEP_AUDIO="${KEEP_AUDIO:-1}"
22
FORMAT_SELECT="${FORMAT_SELECT:-140/bestaudio[ext=m4a]/bestaudio[ext=webm]/bestaudio/best}"
23
AUDIO_FORMAT="${AUDIO_FORMAT:-m4a}"
24
VAD_FILTER="${VAD_FILTER:-1}"
25

26
URLS_FILE=""
27

28
# -----------------------------
29
# Helpers
30
# -----------------------------
31
die() { echo "❌ $*" >&2; exit 1; }
32
log() { echo "👉 $*" >&2; }
33

34
need_cmd() { command -v "$1" >/dev/null 2>&1 || die "Missing command: $1"; }
35

36
ensure_python_pkg() {
37
  local mod="$1"
38
  local pkg="${2:-$1}"
39
  log "🔎 python: $(command -v python)"
40
  if python - <<PY >/dev/null 2>&1
41
import importlib.util, sys
42
sys.exit(0 if importlib.util.find_spec("$mod") else 1)
43
PY
44
  then
45
    log "✅ Python package OK: $mod"
46
  else
47
    log "⬇️  Installing Python package: $pkg"
48
    python -m pip install -U "$pkg"
49
  fi
50
}
51

52
is_url() { [[ "$1" =~ ^https?:// ]]; }
53

54
build_ytdlp_args() {
55
  local -a a=()
56
  [[ "$USE_COOKIES" == "1" ]] && a+=(--cookies-from-browser "$BROWSER")
57
  [[ "$USE_REMOTE_COMPONENTS" == "1" ]] && a+=(--remote-components ejs:github)
58
  printf '%s\0' "${a[@]}"
59
}
60

61
download_audio() {
62
  local input="$1" outdir="$2" logf="$3"
63
  mkdir -p "$outdir"
64

65
  local -a ytdlp=()
66
  while IFS= read -r -d '' x; do ytdlp+=("$x"); done < <(build_ytdlp_args)
67

68
  local tmpl="$outdir/%(title).200B [%(id)s].%(ext)s"
69
  log "⬇️  Downloading audio: $input"
70

71
  local filepath
72
  filepath="$(
73
    yt-dlp "${ytdlp[@]}" \
74
      -f "$FORMAT_SELECT" \
75
      -x --audio-format "$AUDIO_FORMAT" \
76
      -o "$tmpl" \
77
      --print after_move:filepath \
78
      "$input" \
79
      2>>"$logf"
80
  )" || die "yt-dlp failed: $input"
81

82
  filepath="$(printf '%s\n' "$filepath" | sed '/^[[:space:]]*$/d' | tail -n 1)"
83
  [[ -f "$filepath" ]] || die "Audio not found after download: $filepath"
84
  printf '%s\n' "$filepath"
85
}
86

87
transcribe_audio() {
88
  local audio="$1" lang="$2" model="$3" device="$4" compute="$5" vad="$6" logf="$7"
89
  [[ -f "$audio" ]] || die "Audio not found: $audio"
90

91
  local base="${audio%.*}"
92
  local srt="${base}.${lang}.srt"
93
  local txt="${base}.${lang}.txt"
94
  local json="${base}.${lang}.json"
95
  local tsv="${base}.${lang}.tsv"
96

97
  log "🧠 Transcribing: $audio"
98
  python - "$audio" "$lang" "$model" "$device" "$compute" "$vad" \
99
    "$srt" "$txt" "$json" "$tsv" >>"$logf" 2>&1 << 'PY'
100
import sys, json, os
101
from faster_whisper import WhisperModel
102

103
audio, lang, model_name, device, compute, vad, srt_p, txt_p, json_p, tsv_p = sys.argv[1:]
104
vad = vad == "1"
105

106
model = WhisperModel(model_name, device=device, compute_type=compute)
107
segments, info = model.transcribe(audio, language=lang, beam_size=5, vad_filter=vad)
108

109
def ts(t):
110
    h=int(t//3600); m=int((t%3600)//60); s=t%60
111
    return f"{h:02d}:{m:02d}:{s:06.3f}".replace(".", ",")
112

113
rows=[]
114
with open(srt_p,"w",encoding="utf-8") as srt, open(txt_p,"w",encoding="utf-8") as txt:
115
    i=1
116
    for seg in segments:
117
        text=(seg.text or "").strip()
118
        if not text: continue
119
        srt.write(f"{i}\n{ts(seg.start)} --> {ts(seg.end)}\n{text}\n\n")
120
        txt.write(text+"\n")
121
        rows.append({"i":i,"start":float(seg.start),"end":float(seg.end),"text":text})
122
        i+=1
123

124
with open(json_p,"w",encoding="utf-8") as f:
125
    json.dump({"audio":os.path.basename(audio),"lang":lang,"segments":rows},f,ensure_ascii=False,indent=2)
126

127
with open(tsv_p,"w",encoding="utf-8") as f:
128
    f.write("i\tstart\tend\ttext\n")
129
    for r in rows:
130
        f.write(f"{r['i']}\t{r['start']:.3f}\t{r['end']:.3f}\t{r['text']}\n")
131
PY
132
}
133

134
usage() {
135
  cat <<'USAGE'
136
Usage:
137
  ./yt_asr.sh [options] <url_or_audio> [more...]
138
  ./yt_asr.sh --urls-file urls.txt
139

140
Options:
141
  --urls-file FILE       Read URLs from file (one per line, # for comments)
142
  -o, --outdir DIR
143
  --lang LANG            zh / en / ja (default: zh)
144
  --model MODEL
145
  --device cpu|cuda
146
  --compute TYPE
147
  --browser NAME
148
  --no-cookies
149
  --no-remote-components
150
  --keep-audio | --no-keep-audio
151
  --no-vad
152
  -h, --help
153
USAGE
154
}
155

156
# -----------------------------
157
# Parse args
158
# -----------------------------
159
ARGS=()
160
while [[ $# -gt 0 ]]; do
161
  case "$1" in
162
    --urls-file) URLS_FILE="$2"; shift 2;;
163
    -o|--outdir) OUTDIR="$2"; shift 2;;
164
    --lang) ASR_LANG="$2"; shift 2;;
165
    --model) MODEL="$2"; shift 2;;
166
    --device) DEVICE="$2"; shift 2;;
167
    --compute) COMPUTE="$2"; shift 2;;
168
    --browser) BROWSER="$2"; shift 2;;
169
    --no-cookies) USE_COOKIES=0; shift;;
170
    --no-remote-components) USE_REMOTE_COMPONENTS=0; shift;;
171
    --keep-audio) KEEP_AUDIO=1; shift;;
172
    --no-keep-audio) KEEP_AUDIO=0; shift;;
173
    --no-vad) VAD_FILTER=0; shift;;
174
    -h|--help) usage; exit 0;;
175
    *) ARGS+=("$1"); shift;;
176
  esac
177
done
178

179
# -----------------------------
180
# Preflight
181
# -----------------------------
182
need_cmd yt-dlp
183
need_cmd ffmpeg
184
need_cmd python
185
ensure_python_pkg faster_whisper faster-whisper
186

187
mkdir -p "$OUTDIR" "$OUTDIR/logs"
188

189
# -----------------------------
190
# Collect inputs
191
# -----------------------------
192
ITEMS=()
193

194
if [[ -n "$URLS_FILE" ]]; then
195
  [[ -f "$URLS_FILE" ]] || die "urls file not found: $URLS_FILE"
196
  while IFS= read -r line; do
197
    line="$(echo "$line" | sed 's/#.*//g' | xargs)"
198
    [[ -z "$line" ]] && continue
199
    ITEMS+=("$line")
200
  done < "$URLS_FILE"
201
fi
202

203
ITEMS+=("${ARGS[@]}")
204
[[ ${#ITEMS[@]} -ge 1 ]] || { usage; exit 1; }
205

206
# -----------------------------
207
# Main loop
208
# -----------------------------
209
for item in "${ITEMS[@]}"; do
210
  echo "============================================================" >&2
211
  log "INPUT: $item"
212

213
  safe_id="$(echo "$item" | sed 's#[^A-Za-z0-9._-]#_#g' | cut -c1-80)"
214
  ts="$(date +%Y%m%d_%H%M%S)"
215
  logf="$OUTDIR/logs/${ts}_${safe_id}.log"
216
  : > "$logf"
217

218
  audio=""
219
  downloaded=0
220

221
  if is_url "$item"; then
222
    audio="$(download_audio "$item" "$OUTDIR" "$logf")"
223
    downloaded=1
224
  else
225
    [[ -f "$item" ]] || die "Not a URL or file: $item"
226
    audio="$item"
227
  fi
228

229
  transcribe_audio "$audio" "$ASR_LANG" "$MODEL" "$DEVICE" "$COMPUTE" "$VAD_FILTER" "$logf"
230

231
  if [[ "$downloaded" == "1" && "$KEEP_AUDIO" == "0" ]]; then
232
    rm -f -- "$audio"
233
  fi
234

235
  log "📝 Log saved: $logf"
236
done
237

238
log "🎉 All done."