Add mobile compatibility concept, Pixel 6 demo video infrastructure, and CI/CD workflow (#1006)

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: thomasnordquist <7721625+thomasnordquist@users.noreply.github.com>

scripts/cutVideoSegmentsMobile.js

@@ -0,0 +1,107 @@
+#!/usr/bin/env node
+
+/**
+ * Cut mobile demo video into segments as GIFs based on scenes-mobile.json
+ * 
+ * This script reads scenes-mobile.json and uses ffmpeg to create GIF segments
+ * from the ui-test-mobile.mp4 video file.
+ */
+
+const fs = require('fs');
+const { spawn } = require('child_process');
+
+// Check required files exist
+if (!fs.existsSync('scenes-mobile.json')) {
+  console.error('scenes-mobile.json not found');
+  process.exit(1);
+}
+
+if (!fs.existsSync('ui-test-mobile.mp4')) {
+  console.error('ui-test-mobile.mp4 not found');
+  process.exit(1);
+}
+
+console.log('Cutting mobile video into GIF segments based on scenes-mobile.json...');
+
+const scenes = JSON.parse(fs.readFileSync('scenes-mobile.json', 'utf8'));
+
+const GIF_SCALE = process.env.GIF_SCALE || '412';
+
+console.log('Creating mobile GIF segments...');
+
+// Sanitize scene name to prevent path traversal and command injection
+function sanitizeName(name) {
+  // Remove any characters that aren't alphanumeric, dash, or underscore
+  return name.replace(/[^a-zA-Z0-9_-]/g, '-');
+}
+
+async function cutSegmentAsGif(scene, index) {
+  const safeName = sanitizeName(scene.name);
+  const segmentName = `segment-mobile-${String(index + 1).padStart(2, '0')}-${safeName}`;
+  const paletteFile = `${segmentName}-palette.png`;
+  const outputFile = `${segmentName}.gif`;
+  const startTime = scene.start / 1000; // Convert ms to seconds
+  const duration = scene.duration / 1000; // Convert ms to seconds
+  
+  console.log(`Creating ${outputFile} (start: ${startTime}s, duration: ${duration}s)`);
+  
+  // Step 1: Generate palette for this segment
+  await new Promise((resolve, reject) => {
+    const ffmpeg = spawn('ffmpeg', [
+      '-y',
+      '-ss', startTime.toString(),
+      '-t', duration.toString(),
+      '-i', 'ui-test-mobile.mp4',
+      '-vf', `fps=10,scale=${GIF_SCALE}:-1:flags=lanczos,palettegen`,
+      paletteFile
+    ]);
+    
+    ffmpeg.on('close', (code) => {
+      if (code === 0) {
+        resolve();
+      } else {
+        console.error(`Failed to create palette for ${outputFile}`);
+        reject(new Error(`ffmpeg palette generation exited with code ${code}`));
+      }
+    });
+  });
+  
+  // Step 2: Create GIF using the palette
+  await new Promise((resolve, reject) => {
+    const ffmpeg = spawn('ffmpeg', [
+      '-y',
+      '-ss', startTime.toString(),
+      '-t', duration.toString(),
+      '-i', 'ui-test-mobile.mp4',
+      '-i', paletteFile,
+      '-filter_complex', `fps=10,scale=${GIF_SCALE}:-1:flags=lanczos[x];[x][1:v]paletteuse`,
+      outputFile
+    ]);
+    
+    ffmpeg.on('close', (code) => {
+      // Clean up palette file
+      try {
+        fs.unlinkSync(paletteFile);
+      } catch (e) {
+        // Ignore cleanup errors
+      }
+      
+      if (code === 0) {
+        resolve();
+      } else {
+        console.error(`Failed to create ${outputFile}`);
+        reject(new Error(`ffmpeg GIF creation exited with code ${code}`));
+      }
+    });
+  });
+}
+
+(async () => {
+  for (let i = 0; i < scenes.length; i++) {
+    await cutSegmentAsGif(scenes[i], i);
+  }
+  console.log('Mobile video segments created successfully');
+})().catch(err => {
+  console.error(err);
+  process.exit(1);
+});

scripts/generateMarkdownSummaryMobile.js

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+const fs = require('fs');
+
+// Read scenes-mobile.json
+const scenes = JSON.parse(fs.readFileSync('scenes-mobile.json', 'utf8'));
+
+// Get base URL from command line arguments
+const baseUrl = process.argv[2];
+
+if (!baseUrl) {
+  console.error('Usage: node generateMarkdownSummaryMobile.js <base-url>');
+  process.exit(1);
+}
+
+// Sanitize scene name to prevent path traversal
+function sanitizeName(name) {
+  // Remove any characters that aren't alphanumeric, dash, or underscore
+  return name.replace(/[^a-zA-Z0-9_-]/g, '-');
+}
+
+// Generate markdown
+let markdown = '## 📱 Mobile Demo Video Generated\n\n';
+markdown += `### Full Mobile Video (Pixel 6 - 412x915)\n\n`;
+markdown += `[📥 Download Mobile Video (MP4)](${baseUrl}/ui-test-mobile.mp4) | [GIF](${baseUrl}/ui-test-mobile.gif)\n\n`;
+markdown += `---\n\n`;
+markdown += `### 📑 Mobile Video Segments\n\n`;
+markdown += `<details>\n`;
+markdown += `<summary>Click to expand mobile segments</summary>\n\n`;
+
+scenes.forEach((scene, index) => {
+  const safeName = sanitizeName(scene.name);
+  const segmentFile = `segment-mobile-${String(index + 1).padStart(2, '0')}-${safeName}.gif`;
+  const title = scene.title || scene.name;
+  const duration = (scene.duration / 1000).toFixed(1);
+  
+  markdown += `<details>\n`;
+  markdown += `<summary><strong>${index + 1}. ${title}</strong> (${duration}s)</summary>\n\n`;
+  markdown += `![${title}](${baseUrl}/${segmentFile})\n\n`;
+  markdown += `</details>\n\n`;
+});
+
+markdown += `</details>\n\n`;
+markdown += `_Mobile videos recorded at 412x915 (Pixel 6 viewport). Videos will expire in 90 days._`;
+
+console.log(markdown);

scripts/prepareVideoMobile.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Mobile demo video post-processing script
+# Converts raw mobile video to MP4 and GIF, then cuts into segments
+
+DIMENSIONS="412x915"
+GIF_SCALE="412"
+
+ffmpeg -s:v $DIMENSIONS -r 20 -f rawvideo -pix_fmt yuv420p -i qrawvideorgb24-mobile.yuv app2-mobile.mp4
+
+# The video starts with a few blank frames, we want to know when they stop
+ffprobe -f lavfi -i "movie=app2-mobile.mp4,blackdetect[out0]" -show_entries tags=lavfi.black_start,lavfi.black_end -of default=nw=1 -v quiet > ffmpeg_info_mobile
+END_OF_BLACK=`cat ffmpeg_info_mobile | grep end | head -n1 | cut -d'=' -f2`
+
+# Remove grey frames at the beginning (app start and splash screen)
+END_OF_BLACK=`awk "BEGIN {print $END_OF_BLACK+0.8; exit}"`
+
+# Trim black frames at start
+ffmpeg -s:v $DIMENSIONS -r 20 -f rawvideo -pix_fmt yuv420p -i qrawvideorgb24-mobile.yuv -ss $END_OF_BLACK app-mobile.mp4
+
+# Generate gif palette
+ffmpeg -y -s:v $DIMENSIONS -r 20 -f rawvideo -pix_fmt yuv420p -i qrawvideorgb24-mobile.yuv -vf "fps=10,scale=$GIF_SCALE:-1:flags=lanczos,palettegen" palette-mobile.png
+
+# Create gif
+ffmpeg -s:v $DIMENSIONS -r 20 -f rawvideo -pix_fmt yuv420p -i qrawvideorgb24-mobile.yuv -i palette-mobile.png -ss $END_OF_BLACK -filter_complex "fps=10,scale=$GIF_SCALE:-1:flags=lanczos[x];[x][1:v]paletteuse" app-mobile.gif
+
+# Clean up
+rm ffmpeg_info_mobile palette-mobile.png qrawvideorgb24-mobile.yuv app2-mobile.mp4
+
+mv app-mobile.mp4 ui-test-mobile.mp4
+mv app-mobile.gif ui-test-mobile.gif
+
+# Cut video into segments based on scenes-mobile.json
+echo "Cutting mobile video into segments..."
+if [ -f "scenes-mobile.json" ]; then
+  node ./scripts/cutVideoSegmentsMobile.js
+else
+  echo "Warning: scenes-mobile.json not found, skipping segment creation"
+fi

scripts/uiTestsMobile.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+function finish {
+  set +e
+  echo "Exiting, cleaning up.."
+
+  echo "Stopping TMUX session (record-mobile).."
+  tmux kill-session -t record-mobile || echo "Already stopped"
+
+  if [[ ! -z "$PID_MOSQUITTO" ]]; then
+    echo "Stopping mosquitto ($PID_MOSQUITTO).."
+    kill "$PID_MOSQUITTO" || echo "Already stopped"
+  fi
+
+  if [[ ! -z "$PID_VNC" ]]; then
+    echo "Stopping VNC ($PID_VNC).."
+    kill "$PID_VNC" || echo "Already stopped"
+  fi
+
+  if [[ ! -z "$PID_XVFB" ]]; then
+    echo "Stopping XVFB ($PID_XVFB).."
+    kill "$PID_XVFB" || echo "Already stopped"
+  fi
+
+  if [[ ! -z "$PID_SERVER" ]]; then
+    echo "Stopping MQTT Explorer server ($PID_SERVER).."
+    kill "$PID_SERVER" || echo "Already stopped"
+  fi
+}
+
+trap finish EXIT
+set -e
+
+# Mobile viewport dimensions (Pixel 6)
+DIMENSIONS="412x915"
+SCR=99
+
+# Start new window manager
+Xvfb :$SCR -screen 0 "$DIMENSIONS"x24 -ac &
+export PID_XVFB=$!
+sleep 2
+
+# Debug with VNC
+x11vnc -localhost -rfbport 5900 -passwd "bierbier" -display :$SCR & 
+export PID_VNC=$!
+
+# Start mqtt broker
+mosquitto &
+export PID_MOSQUITTO=$!
+sleep 2
+
+# Start MQTT Explorer in browser mode
+export MQTT_EXPLORER_USERNAME=admin
+export MQTT_EXPLORER_PASSWORD=password
+export MQTT_EXPLORER_SKIP_AUTH=true
+export DISPLAY=:$SCR
+node dist/src/server.js &
+export PID_SERVER=$!
+sleep 5
+
+# Delete old video
+rm -f ./app-mobile*.mp4
+rm -f ./qrawvideorgb24-mobile.yuv
+
+# Start recording in tmux
+tmux new-session -d -s record-mobile ffmpeg -f x11grab -draw_mouse 0 -video_size $DIMENSIONS -i :$SCR -r 20 -vcodec rawvideo -pix_fmt yuv420p qrawvideorgb24-mobile.yuv
+
+# Start tests
+export BROWSER_MODE_URL=http://localhost:3000
+DISPLAY=:$SCR node dist/src/spec/demoVideoMobile.js
+TEST_EXIT_CODE=$?
+echo "Test script exited with $TEST_EXIT_CODE"
+
+# Stop recording
+tmux send-keys -t record-mobile q
+
+# Ensure video is written
+sleep 5
+
+exit $TEST_EXIT_CODE