AI Flow: Video

@@ -4,7 +4,7 @@ version = 4  [[package]] name = "adom-aiflow-video"-version = "0.6.1"+version = "0.7.0" dependencies = [  "clap",  "serde",

@@ -2,7 +2,7 @@ # Copied from adom/adom-aiflow template/flow-app. This copy is yours: change anything, never depend on aiflow. [package] name = "adom-aiflow-video"-version = "0.6.1"+version = "0.7.0" edition = "2021" license = "MIT" 

@@ -45,7 +45,7 @@ The AI voiceover is fine for a draft. A film people want to watch has a person i - **The description.** What the film shows, chapters, and a link back to the project's wiki page. - **The upload.** `youtube check` says which channel each of your adom-google accounts uploads to, and how to set one up if you have none. `youtube upload` always sends the film as unlisted, so only people with the link see it and going public is your call. It then opens the video's YouTube Studio page for you and uploads the film's own captions, because YouTube's automatic ones misspell names. When adom-google cannot reach your channel, the AI uploads through YouTube Studio in a browser it drives. -- **You in the thumbnail.** A pose shot on the same page: your camera with a "point here" dot where the product will sit, a countdown and three stills. The AI cuts you out and puts you in the thumbnail pointing at what you built.+- **You in the thumbnail, pointing at what you built.** A pose shot on the same page: your camera with the product's picture on it and a big "POINT AT THIS" marker, a countdown and three stills. The AI cuts you out of your background and puts you in the thumbnail pointing at the product. The helpers for that ship with the app (`thumbkit`).  ![The picker: three thumbnails and four titles, previewed the way YouTube shows them](docs/screenshots/title-thumbnail-picker.jpg) @@ -93,7 +93,7 @@ script <script.json>            scan <files...>      track <path>      clean [-- director --out <file.js>        guide                narrate --video <draft.mp4> brand [--video <final.mp4> --out <branded.mp4>] youtube check | title "..." | thumbnail <file> | upload --file f.mp4 --title T --description-file d.txt --thumbnail t.jpg | open <link> | captions <link> --file f.srt-narrate pitch --titles "A" "B" --thumbs a.jpg b.jpg+narrate pitch --titles "A" "B" --thumbs a.jpg b.jpg      narrate pose --thumbs product.jpg      thumbkit --out thumbkit/ ```  Every command answers `OK:` with `Hint:` lines. The skill is in [skills/adom-aiflow-video/SKILL.md](skills/adom-aiflow-video/SKILL.md); the walkthrough guide is in [docs/walkthrough.md](docs/walkthrough.md).@@ -104,4 +104,4 @@ Your film is yours: it carries no Adom logo. The Adom logo sting is only for off  ## Status -0.6.1. It grew out of the Ghost Hand films ([john/ghost-hand](https://wiki.adom.inc/john/ghost-hand)). Working today: the flow and its hints, clip recording, the disk estimate, script feedback, the text privacy scan, file tracking and cleanup, the walkthrough director, the narration page (voice, camera picture, drawing, playback check, AI notes), and the YouTube channel check, title and thumbnail feedback and upload. Still done by the AI by hand, with the skill as the guide: the cut itself, putting the voice, picture and drawings on the film, lining up a second recorder, composing thumbnails, and the frame-by-frame privacy check.+0.7.0. It grew out of the Ghost Hand films ([john/ghost-hand](https://wiki.adom.inc/john/ghost-hand)). Working today: the flow and its hints, clip recording, the disk estimate, script feedback, the text privacy scan, file tracking and cleanup, the walkthrough director, the narration page (voice, camera picture, drawing, playback check, AI notes), and the YouTube channel check, title and thumbnail feedback and upload. Still done by the AI by hand, with the skill as the guide: the cut itself, putting the voice, picture and drawings on the film, lining up a second recorder, composing thumbnails, and the frame-by-frame privacy check.

@@ -180,3 +180,9 @@ kbd { font: 500 11px var(--font-mono); border: 1px solid var(--border); border-r .pose-shots { display: flex; gap: 8px; margin: 0 0 12px; } .pose-shots img { width: 32%; border-radius: 8px; border: 1px solid var(--border); }  .esc-hint { align-self: center; color: var(--muted); font-size: 13px; }++.pose .target { position: absolute; width: 30%; transform: translate(-50%, -50%); cursor: grab; border: 5px solid #ffd43b; border-radius: 12px; box-shadow: 0 0 0 4px rgba(0,0,0,.45), 0 0 40px rgba(255,212,59,.8); animation: pulse 1s infinite; }+.pose .target img { display: block; width: 100%; aspect-ratio: 5/4; object-fit: cover; border-radius: 7px; opacity: .88; pointer-events: none; }+.pose .target b { position: absolute; left: 50%; bottom: 100%; transform: translate(-50%, -8px); white-space: nowrap; background: #ffd43b; color: #111; font: 800 clamp(16px, 1.8vw, 30px) var(--font-head); padding: 4px 14px; border-radius: 999px; }+.pose .target .arrow { position: absolute; right: 100%; top: 50%; width: 0; height: 0; margin: -22px 14px 0 0; border: 22px solid transparent; border-left: 34px solid #ffd43b; animation: nudge .7s ease-in-out infinite alternate; }+@keyframes nudge { from { transform: translateX(-14px); } to { transform: translateX(0); } }

@@ -580,7 +580,7 @@   $('pitchbox').onclick = showPitch; $('pitchbox').onkeydown = (e) => { if (e.key === 'Enter' || e.key === ' ') showPitch(); };    // ---------------- pose shot: a still of you from the camera, for the thumbnail -----------------  async function poseShot(say) {+  async function poseShot(say, product) {     if (mode !== 'idle') return toast('Finish what is running first.', 'info');     let st = camStream, why = '';     if (!st) {                                                   // try every camera: one may be held by another app@@ -590,8 +590,8 @@     }     if (!st) { say('Pose shot: no camera: ' + why); report(); return toast('No camera for the photo: ' + why + '.', 'error'); }     const own = st !== camStream;-    modal(`<h2>A photo of you for the thumbnail</h2><p>${esc(say || 'Point at the dot: that is where your product will be in the thumbnail. Drag the dot if you want it somewhere else. Big expression, big gesture.')}</p>-      <div class="pose" id="pose"><video id="pose-v" muted playsinline></video><i class="dot" id="pose-dot" style="left:18%;top:62%"></i><div class="count" id="pose-n"></div><div class="flash" id="pose-f"></div></div>+    modal(`<h2>A photo of you for the thumbnail</h2><p>${esc(say || (product ? 'Point at the picture on the right: that is your product, where it will sit in the thumbnail. Reach toward it and look at it. Drag the picture if you want it somewhere else. Big expression, big gesture.' : 'Point at the dot: that is where your product will be in the thumbnail. Drag the dot if you want it somewhere else. Big expression, big gesture.'))}</p>+      <div class="pose" id="pose"><video id="pose-v" muted playsinline></video>${product ? `<div class="target" id="pose-dot" style="left:76%;top:62%"><img src="pose-product?v=${product}" alt=""><b>POINT AT THIS</b><span class="arrow"></span></div>` : `<i class="dot" id="pose-dot" style="left:76%;top:62%"></i>`}<div class="count" id="pose-n"></div><div class="flash" id="pose-f"></div></div>       <div class="pose-shots" id="pose-shots"></div>       <div class="row"><button class="btn lg" id="pose-go">Take 3 photos</button><button class="btn ghost lg" id="pose-done">Done</button></div>`);     const pv = $('pose-v'); pv.srcObject = st; pv.play().catch(() => {});@@ -649,7 +649,7 @@       for (const kv of String(d.spec || '').split(/\s+/)) { const [k, val] = kv.split('='); if (!val) continue;         if (k === 'shape' && ASPECT[val]) PIP.shape = val; else if (k === 'corner') { PIP.x = /r/.test(val) ? 0.99 : 0.01; PIP.y = /b/.test(val) ? 0.75 : 0.01; } else if (['size', 'x', 'y', 'zoom', 'ox', 'oy'].includes(k) && isFinite(+val)) PIP[k] = +val; }       PIP.size = Math.max(SMIN, Math.min(SMAX, PIP.size)); PIP.zoom = Math.max(1, Math.min(2.5, PIP.zoom)); layout(true); },-    pose: (d) => poseShot(d.say),+    pose: (d) => poseShot(d.say, d.hasProduct ? (d.stamp || 1) : 0),     pitch: (d) => { pitch = { titles: d.titles || [], thumbs: d.thumbs || [], stamp: d.stamp || Date.now() }; pitchChoice = null; pitchBox(); if (mode === 'idle') showPitch(); else toast('Your AI has title and thumbnail ideas for you: see the box at the top right.', 'info'); },     seek: (d) => p.seekTo(Math.max(0, +d.seconds || 0)), play: () => v().play(), pause: () => v().pause(),     review: (d) => { const tk = takes[(+d.take || takes.length) - 1]; tk ? review(tk) : toast('No such take', 'error'); },

@@ -0,0 +1,26 @@+"""Compose a 1280 x 720 thumbnail: the narrator (cut out) pointing at the product picture, and a two-line headline.+  python3 compose.py --person pose-cut.png --product product.jpg --line1 "AI built this" --line2 "for my daughter." \+      --tip 1230,555 --out thumb.jpg [--font /path/Bold.ttf] [--scale 0.54]+--tip is the fingertip in the person photo (pixels). The person is placed so the fingertip lands on the product+picture's edge. No logo is added: the film is the user's own."""+import argparse, numpy as np+from PIL import Image, ImageDraw, ImageFont, ImageFilter+a=argparse.ArgumentParser(); [a.add_argument(x,**k) for x,k in [('--person',{}),('--product',{}),('--line1',{}),('--line2',{}),('--tip',{}),('--out',{}),('--font',{'default':''}),('--scale',{'type':float,'default':0.54})]]+o=a.parse_args(); W,H=1280,720+def glow(img,c,cx,cy,r,s):+    y,x=np.mgrid[0:H,0:W]; al=np.clip(1-np.sqrt((x-cx)**2+(y-cy)**2)/r,0,1)**2*s; arr=np.asarray(img,dtype=np.float32); return Image.fromarray((arr*(1-al[...,None])+np.array(c,dtype=np.float32)*al[...,None]).astype(np.uint8))+im=glow(glow(Image.new('RGB',(W,H),(13,17,23)),(0,150,170),-60,800,700,.5),(110,90,210),1340,820,640,.45).convert('RGBA')+tw,th,tx,ty=590,474,660,196+t=Image.open(o.product).convert('RGB'); r=max(tw/t.width,th/t.height); t=t.resize((int(t.width*r)+1,int(t.height*r)+1),Image.LANCZOS); t=t.crop(((t.width-tw)//2,(t.height-th)//2,(t.width-tw)//2+tw,(t.height-th)//2+th))+m=Image.new('L',(tw,th),0); ImageDraw.Draw(m).rounded_rectangle([0,0,tw-1,th-1],radius=20,fill=255)+g=Image.new('RGBA',(W,H),(0,0,0,0)); ImageDraw.Draw(g).rounded_rectangle([tx-4,ty-4,tx+tw+4,ty+th+4],radius=24,fill=(255,255,255,90)); im.alpha_composite(g.filter(ImageFilter.GaussianBlur(14))); im.paste(t,(tx,ty),m)+p=Image.open(o.person).convert('RGBA'); k=o.scale; p=p.resize((int(p.width*k),int(p.height*k)),Image.LANCZOS); tipx,tipy=[float(v) for v in o.tip.split(',')]+ox,oy=int(tx+12-tipx*k),H-p.height+sh=Image.new('RGBA',(W,H),(0,0,0,0)); b=Image.new('RGBA',p.size,(0,0,0,255)); b.putalpha(p.split()[3].point(lambda v:int(v*.55))); sh.paste(b,(ox,oy+10),b); im.alpha_composite(sh.filter(ImageFilter.GaussianBlur(18))); im.paste(p,(ox,oy),p)+def font(sz):+    for f in [o.font,'/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf','DejaVuSans-Bold.ttf']:+        try: return ImageFont.truetype(f,sz)+        except Exception: pass+    return ImageFont.load_default()+d=ImageDraw.Draw(im); f=font(84); d.text((44,30),o.line1,font=f,fill=(255,255,255,255)); d.text((44,30+82),o.line2,font=f,fill=(47,208,195,255))+im.convert('RGB').save(o.out,quality=92); print('OK:',o.out)

@@ -0,0 +1,19 @@+"""Cut a person out of a photo: python3 cutout.py <photo.jpg> <out.png> [model.onnx]+Needs onnxruntime, numpy and Pillow, and the u2net_human_seg model (about 170 MB):+  curl -L -o u2net_human_seg.onnx https://github.com/danielgatis/rembg/releases/download/v0.0.0/u2net_human_seg.onnx+Runs offline on the CPU in a few seconds."""+import sys, numpy as np, onnxruntime as ort+from PIL import Image, ImageFilter+def cut(src, dst, model='u2net_human_seg.onnx'):+    im=Image.open(src).convert('RGB'); W,H=im.size+    x=np.asarray(im.resize((320,320),Image.LANCZOS),dtype=np.float32); x/=x.max()+    x=(x-np.array([0.485,0.456,0.406],dtype=np.float32))/np.array([0.229,0.224,0.225],dtype=np.float32)+    x=x.transpose(2,0,1)[None].astype(np.float32)+    s=ort.InferenceSession(model,providers=['CPUExecutionProvider']); o=s.run(None,{s.get_inputs()[0].name:x})[0][0,0]+    o=(o-o.min())/(o.max()-o.min()+1e-8)+    m=Image.fromarray((o*255).astype(np.uint8)).resize((W,H),Image.BICUBIC).filter(ImageFilter.GaussianBlur(2))+    a=np.asarray(m,dtype=np.float32)/255; a=np.clip((a-0.35)/0.4,0,1)            # firm up the soft edge+    out=im.convert('RGBA'); out.putalpha(Image.fromarray((a*255).astype(np.uint8))); out.save(dst); return out+if __name__=='__main__':+    cut(sys.argv[1], sys.argv[2], *(sys.argv[3:4]))+    print('OK:', sys.argv[2])

⋯ 1 unchanged line ⋯

⋯ 1 unchanged line ⋯

⋯ 1 unchanged line ⋯

@@ -1,7 +1,7 @@ {   "slug": "adom-aiflow-video",   "type": "app",-  "version": "0.6.1",+  "version": "0.7.0",   "title": "AI Flow: Video",   "description": "Make a narrated walkthrough video of a board built in Adom Hydrogen, put it on the project's wiki page and help the human upload it to YouTube. A dynamic skill: a small Rust binary that suggests each step, records clips, ships a ready-made 3D walkthrough director and gives gentle feedback; the AI does the work. Open source (MIT): the full Rust source ships on this page; build it yourself with ./build.sh.",   "category": "Developer Tools",

@@ -1,7 +1,7 @@ {   "slug": "adom-aiflow-video",   "type": "app",-  "version": "0.6.1",+  "version": "0.7.0",   "title": "AI Flow: Video",   "description": "Make a narrated walkthrough video of a board built in Adom Hydrogen, put it on the project's wiki page and help the human upload it to YouTube. A dynamic skill: a small Rust binary that suggests each step, records clips, ships a ready-made 3D walkthrough director and gives gentle feedback; the AI does the work. Open source (MIT): the full Rust source ships on this page; build it yourself with ./build.sh.",   "category": "Developer Tools",

@@ -134,6 +134,6 @@ The thumbnail's few words are a promise, and the narration has to keep it, so se  ## A photo of the narrator for the thumbnail -A person pointing at the product beats a face in a circle. `narrate pose "<what to do>"` opens the camera on the narration page with a "point here" dot (they can drag it), counts down and takes three stills, saved as `pose-N.jpg` in the takes folder, mirrored the way they saw themselves so what they pointed at is where they point. Cut them out of the background (a person-segmentation model such as u2net_human_seg run with onnxruntime works offline), place them in the thumbnail pointing at the product, and show the result in the picker. If the page says another app is using the camera, tell them: the browser cannot take it from another program.+A person pointing at the product beats a face in a circle. `narrate pose "<what to do>" --thumbs <product picture>` opens the camera on the narration page with the product's own picture on the view, framed in yellow under a large "POINT AT THIS" marker (they can drag it; without --thumbs it is a plain dot, which people find hard to read, so always give the picture), counts down and takes three stills, saved as `pose-N.jpg` in the takes folder, mirrored the way they saw themselves so what they pointed at is where they point. Then `thumbkit --out thumbkit/` writes two helpers: `cutout.py` cuts them out of their background (a person-segmentation model run with onnxruntime, offline) and `compose.py` places them pointing at the product beside a two-line headline. Look at the result, adjust, and show two or three in the picker. Pick the still where they look at what they point to. If the page says another app is using the camera, tell them: the browser cannot take it from another program.  Only one open copy of the narration page acts on camera, microphone and recording commands: a hidden tab never does, and among visible ones it is the newest or the one the human last touched. `narrate status` shows `pageId`, `inControl` and `hidden` for the page that last reported.

@@ -62,6 +62,8 @@ enum Cmd {     Brand { #[arg(long)] video: Option<String>, #[arg(long)] out: Option<String> },     /// Write the ready-made 3D walkthrough director (JavaScript for the Adom 3D viewer): `director --out walkthrough-director.js`     Director { #[arg(long, default_value = "walkthrough-director.js")] out: String },+    /// Write the thumbnail helpers (cut a person out of a photo, compose them pointing at the product): `thumbkit --out thumbkit/`+    Thumbkit { #[arg(long, default_value = "thumbkit")] out: String },     /// Print the walkthrough guide: the steps file, how to record a take, and what we learned     Guide,     /// Let the human narrate with their computer's microphone: `narrate --video teleprompter.mp4` serves a page that plays it and records them in step@@ -198,6 +200,15 @@ fn main() {             }             _ => err("youtube takes: check, title, thumbnail, upload, open, captions", &[]),         },+        Cmd::Thumbkit { out } => {+            let _ = std::fs::create_dir_all(out);+            for (n, body) in [("cutout.py", include_str!("../assets/thumbkit/cutout.py")), ("compose.py", include_str!("../assets/thumbkit/compose.py"))] { let _ = std::fs::write(Path::new(out).join(n), body); }+            ok(&format!("thumbnail helpers written to {out}/ (cutout.py, compose.py)"), &[+                "1. `narrate pose --thumbs <product picture>` : the human points at the product on their camera view; three stills land as pose-N.jpg in the takes folder.".into(),+                "2. `python3 cutout.py pose-1.jpg pose-1-cut.png` cuts them out of their background (the script's first lines say how to fetch the model once).".into(),+                "3. `python3 compose.py --person pose-1-cut.png --product product.jpg --line1 \"...\" --line2 \"...\" --tip <x,y of the fingertip> --out thumb.jpg`, then look at it, adjust --scale and --tip, and show two or three in the picker (`narrate pitch`).".into(),+                "The scripts are a starting point: change the layout freely. They add no logo; the film is the human's own.".into()]);+        }         Cmd::Brand { video, out } => {             match (video, out) {                 (Some(v), Some(o)) => match brand::apply(v, o) { Ok((m, h)) => ok(&m, &h), Err((m, h)) => err(&m, &h) },@@ -288,7 +299,7 @@ fn main() {                         ("seek", Some(a)) => json!({"seconds": a.parse::<f64>().unwrap_or(0.0)}),                         ("mic", Some(a)) => json!({"name": a}),                         ("pip", Some(a)) => json!({"spec": a}),-                        ("pose", a) => json!({"say": a.clone().unwrap_or_default()}),+                        ("pose", a) => json!({"say": a.clone().unwrap_or_default(), "product": thumbs.first().map(|t| std::fs::canonicalize(t).map(|p| p.display().to_string()).unwrap_or_default())}),                         ("pitch", _) => {                             if titles.is_empty() || thumbs.is_empty() { err("pitch needs --titles \"A\" \"B\" ... and --thumbs a.jpg b.jpg ...", &["Bring about five titles and two or three thumbnails (1280 x 720). `youtube title` and `youtube thumbnail` give feedback on them first.".into()]); }                             let abs: Vec<String> = thumbs.iter().map(|t| std::fs::canonicalize(t).map(|p| p.display().to_string()).unwrap_or_else(|_| err(&format!("no picture at {t}"), &[]))).collect();

@@ -285,6 +285,7 @@ fn handle(mut s: TcpStream, video: PathBuf, out: PathBuf, sh: Arc<Mutex<Shared>>             respond(&mut s, "200 OK", "application/json", "", b"{\"ok\":true}")         }         // a still of the narrator from their camera, posed for the thumbnail: saved as pose-N.jpg in the takes folder+        ("GET", "/pose-product") => { let f = sh.lock().unwrap().state["poseProduct"].as_str().map(PathBuf::from); match f { Some(f) if f.is_file() => serve_file(&mut s, &f, None), _ => respond(&mut s, "404 Not Found", "text/plain", "", b"") } }         ("POST", "/pose") => {             let mut body = vec![0u8; len];             if rd.read_exact(&mut body).is_err() { return respond(&mut s, "400 Bad Request", "application/json", "", b"{\"error\":\"short body\"}"); }@@ -354,6 +355,7 @@ fn handle(mut s: TcpStream, video: PathBuf, out: PathBuf, sh: Arc<Mutex<Shared>>             data["leader"] = sh.lock().unwrap().state["leader"].clone();             let pages = sh.lock().unwrap().clients.len();             if verb == "note" { sh.lock().unwrap().state["note"] = data["text"].clone(); }+            if verb == "pose" { let mut g = sh.lock().unwrap(); if data["product"].is_string() && !data["product"].as_str().unwrap_or("").is_empty() { g.state["poseProduct"] = data["product"].clone(); } data["hasProduct"] = g.state["poseProduct"].is_string().into(); data["stamp"] = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0).into(); }             if verb == "pitch" { let mut g = sh.lock().unwrap(); let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0); data["stamp"] = stamp.into(); g.state["pitch"] = json!({"titles": data["titles"], "thumbs": data["thumbs"], "stamp": stamp}); let _ = std::fs::create_dir_all(&out); let _ = std::fs::write(out.join("pitch-ideas.json"), serde_json::to_string_pretty(&g.state["pitch"]).unwrap_or_default() + "\n"); g.state["pitchChoice"] = Value::Null; }             if ["toast", "click", "reveal", "cursor", "reload"].contains(&verb.as_str()) {                 data["type"] = verb.clone().into();