Segment demo video by scene with embedded GIF segments (#990)

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: thomasnordquist <7721625+thomasnordquist@users.noreply.github.com>

.github/workflows/tests.yml

@@ -80,12 +80,12 @@ jobs:
         run: yarn ui-test
       - name: Post-processing
         run: ./scripts/prepareVideo.sh
-      - name: Generate unique filename
+      - name: Generate unique base path
-        id: filename
+        id: basepath
         run: |
           TIMESTAMP=$(date +%Y%m%d-%H%M%S)
-          FILENAME="pr-${{ github.event.pull_request.number }}-${TIMESTAMP}.gif"
+          BASEPATH="pr-${{ github.event.pull_request.number }}-${TIMESTAMP}"
-          echo "filename=${FILENAME}" >> $GITHUB_OUTPUT
+          echo "basepath=${BASEPATH}" >> $GITHUB_OUTPUT
       - name: Install AWS CLI v2
         run: |
           apt-get update && apt-get install -y unzip
@@ -99,44 +99,76 @@ jobs:
           aws-access-key-id: ${{ vars.AWS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: 'eu-central-1'
-      - name: Upload to S3
+      - name: Upload full video to S3
-        id: upload
         env:
           AWS_BUCKET: ${{ vars.AWS_BUCKET }}
-          FILENAME: ${{ steps.filename.outputs.filename }}
+          BASEPATH: ${{ steps.basepath.outputs.basepath }}
         run: |
+          # Upload GIF
           aws s3api put-object \
             --bucket ${AWS_BUCKET} \
-            --key artifacts/${FILENAME} \
+            --key artifacts/${BASEPATH}/ui-test.gif \
             --body ./ui-test.gif \
             --content-type image/gif
-      - name: Generate file URL
+          
+          # Upload MP4
+          aws s3api put-object \
+            --bucket ${AWS_BUCKET} \
+            --key artifacts/${BASEPATH}/ui-test.mp4 \
+            --body ./ui-test.mp4 \
+            --content-type video/mp4
+      - name: Upload video segments to S3
+        env:
+          AWS_BUCKET: ${{ vars.AWS_BUCKET }}
+          BASEPATH: ${{ steps.basepath.outputs.basepath }}
+        run: |
+          # Upload all GIF segment files if they exist
+          shopt -s nullglob  # Make glob return empty list if no matches
+          for segment in segment-*.gif; do
+            echo "Uploading $segment..."
+            aws s3api put-object \
+              --bucket ${AWS_BUCKET} \
+              --key artifacts/${BASEPATH}/${segment} \
+              --body ./${segment} \
+              --content-type image/gif
+          done
+          shopt -u nullglob  # Restore default behavior
+      - name: Generate file URLs
         id: fileurl
         env:
           AWS_BUCKET: ${{ vars.AWS_BUCKET }}
-          FILENAME: ${{ steps.filename.outputs.filename }}
+          BASEPATH: ${{ steps.basepath.outputs.basepath }}
         run: |
-          FILE_URL="https://${AWS_BUCKET}.s3.eu-central-1.amazonaws.com/artifacts/${FILENAME}"
+          BASE_URL="https://${AWS_BUCKET}.s3.eu-central-1.amazonaws.com/artifacts/${BASEPATH}"
-          echo "file-url=${FILE_URL}" >> $GITHUB_OUTPUT
+          echo "base-url=${BASE_URL}" >> $GITHUB_OUTPUT
-          echo "Uploaded to: ${FILE_URL}"
+          echo "Uploaded to: ${BASE_URL}"
-      - name: Show URL
+      - name: Generate markdown summary
-        run: echo '${{ steps.fileurl.outputs.file-url }}'
+        id: markdown
-        id: artifact-upload-step
+        env:
-      - run: echo '<picture><img src="${{ steps.fileurl.outputs.file-url }}"></picture>' \
+          BASE_URL: ${{ steps.fileurl.outputs.base-url }}
-          >> $GITHUB_STEP_SUMMARY
+        run: |
+          MARKDOWN=$(node ./scripts/generateMarkdownSummary.js "${BASE_URL}")
+          echo "markdown<<EOF" >> $GITHUB_OUTPUT
+          echo "$MARKDOWN" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+      - name: Add to workflow summary
+        env:
+          MARKDOWN: ${{ steps.markdown.outputs.markdown }}
+        run: |
+          echo "$MARKDOWN" >> $GITHUB_STEP_SUMMARY
       - name: Post video to PR
         uses: actions/github-script@v7
         env:
-          VIDEO_URL: ${{ steps.fileurl.outputs.file-url }}
+          MARKDOWN: ${{ steps.markdown.outputs.markdown }}
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
-            const videoUrl = process.env.VIDEO_URL;
+            const markdown = process.env.MARKDOWN;
             github.rest.issues.createComment({
               owner: context.repo.owner,
               repo: context.repo.repo,
               issue_number: context.issue.number,
-              body: `## 🎬 Demo Video Generated\n\n![Demo Video](${videoUrl})\n\n_This video will expire in 90 days._`
+              body: markdown
             });
 
   test-browser:

.gitignore

@@ -23,4 +23,16 @@ browser-debug-screenshot.png
 app/.webpack-cache
 
 # Temporary files
-/tmp
+/tmp
+
+# Demo video artifacts
+scenes.json
+segment-*.mp4
+segment-*.gif
+ui-test.mp4
+ui-test.gif
+app.mp4
+app2.mp4
+app720.gif
+qrawvideorgb24.yuv
+intro.png

scripts/cutVideoSegments.sh

@@ -0,0 +1,106 @@
+#!/bin/bash
+set -e
+
+# Read scenes.json and cut video into segments as GIFs
+if [ ! -f "scenes.json" ]; then
+  echo "scenes.json not found"
+  exit 1
+fi
+
+if [ ! -f "ui-test.mp4" ]; then
+  echo "ui-test.mp4 not found"
+  exit 1
+fi
+
+echo "Cutting video into GIF segments based on scenes.json..."
+
+GIF_SCALE="1024"
+
+# Parse scenes.json and cut video segments as GIFs
+node -e "
+const fs = require('fs');
+const { spawn } = require('child_process');
+
+const scenes = JSON.parse(fs.readFileSync('scenes.json', 'utf8'));
+
+console.log('Creating GIF segments...');
+
+// Sanitize scene name to prevent path traversal and command injection
+function sanitizeName(name) {
+  // Remove any characters that aren't alphanumeric, dash, or underscore
+  return name.replace(/[^a-zA-Z0-9_-]/g, '-');
+}
+
+async function cutSegmentAsGif(scene, index) {
+  const safeName = sanitizeName(scene.name);
+  const segmentName = \`segment-\${String(index + 1).padStart(2, '0')}-\${safeName}\`;
+  const paletteFile = \`\${segmentName}-palette.png\`;
+  const outputFile = \`\${segmentName}.gif\`;
+  const startTime = scene.start / 1000; // Convert ms to seconds
+  const duration = scene.duration / 1000; // Convert ms to seconds
+  
+  console.log(\`Creating \${outputFile} (start: \${startTime}s, duration: \${duration}s)\`);
+  
+  // Step 1: Generate palette for this segment
+  await new Promise((resolve, reject) => {
+    const ffmpeg = spawn('ffmpeg', [
+      '-y',
+      '-ss', startTime.toString(),
+      '-t', duration.toString(),
+      '-i', 'ui-test.mp4',
+      '-vf', 'fps=10,scale=${process.env.GIF_SCALE || 1024}:-1:flags=lanczos,palettegen',
+      paletteFile
+    ]);
+    
+    ffmpeg.on('close', (code) => {
+      if (code === 0) {
+        resolve();
+      } else {
+        console.error(\`Failed to create palette for \${outputFile}\`);
+        reject(new Error(\`ffmpeg palette generation exited with code \${code}\`));
+      }
+    });
+  });
+  
+  // Step 2: Create GIF using the palette
+  await new Promise((resolve, reject) => {
+    const ffmpeg = spawn('ffmpeg', [
+      '-y',
+      '-ss', startTime.toString(),
+      '-t', duration.toString(),
+      '-i', 'ui-test.mp4',
+      '-i', paletteFile,
+      '-filter_complex', 'fps=10,scale=${process.env.GIF_SCALE || 1024}:-1:flags=lanczos[x];[x][1:v]paletteuse',
+      outputFile
+    ]);
+    
+    ffmpeg.on('close', (code) => {
+      // Clean up palette file
+      try {
+        fs.unlinkSync(paletteFile);
+      } catch (e) {
+        // Ignore cleanup errors
+      }
+      
+      if (code === 0) {
+        resolve();
+      } else {
+        console.error(\`Failed to create \${outputFile}\`);
+        reject(new Error(\`ffmpeg GIF creation exited with code \${code}\`));
+      }
+    });
+  });
+}
+
+(async () => {
+  for (let i = 0; i < scenes.length; i++) {
+    await cutSegmentAsGif(scenes[i], i);
+  }
+  console.log('All GIF segments created successfully');
+})().catch(err => {
+  console.error(err);
+  process.exit(1);
+});
+"
+
+echo "Video segments created successfully"

scripts/generateMarkdownSummary.js

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+const fs = require('fs');
+
+// Read scenes.json
+const scenes = JSON.parse(fs.readFileSync('scenes.json', 'utf8'));
+
+// Get base URL from command line arguments
+const baseUrl = process.argv[2];
+
+if (!baseUrl) {
+  console.error('Usage: node generateMarkdownSummary.js <base-url>');
+  process.exit(1);
+}
+
+// Sanitize scene name to prevent path traversal
+function sanitizeName(name) {
+  // Remove any characters that aren't alphanumeric, dash, or underscore
+  return name.replace(/[^a-zA-Z0-9_-]/g, '-');
+}
+
+// Generate markdown
+let markdown = '## 🎬 Demo Video Generated\n\n';
+markdown += `### Full Video\n\n`;
+markdown += `[📥 Download Full Video (MP4)](${baseUrl}/ui-test.mp4) | [GIF](${baseUrl}/ui-test.gif)\n\n`;
+markdown += `---\n\n`;
+markdown += `### 📑 Video Segments\n\n`;
+markdown += `<details>\n`;
+markdown += `<summary>Click to expand segments</summary>\n\n`;
+
+scenes.forEach((scene, index) => {
+  const safeName = sanitizeName(scene.name);
+  const segmentFile = `segment-${String(index + 1).padStart(2, '0')}-${safeName}.gif`;
+  const title = scene.title || scene.name;
+  const duration = (scene.duration / 1000).toFixed(1);
+  
+  markdown += `<details>\n`;
+  markdown += `<summary><strong>${index + 1}. ${title}</strong> (${duration}s)</summary>\n\n`;
+  markdown += `![${title}](${baseUrl}/${segmentFile})\n\n`;
+  markdown += `</details>\n\n`;
+});
+
+markdown += `</details>\n\n`;
+markdown += `_Videos will expire in 90 days._`;
+
+console.log(markdown);

scripts/prepareVideo.sh

@@ -25,3 +25,12 @@ rm ffmpeg_info palette*.png qrawvideorgb24.yuv
 
 mv app.mp4 ui-test.mp4
 mv app720.gif ui-test.gif
+
+# Cut video into segments based on scenes.json
+echo "Cutting video into segments..."
+if [ -f "scenes.json" ]; then
+  ./scripts/cutVideoSegments.sh
+else
+  echo "Warning: scenes.json not found, skipping segment creation"
+fi
+

scripts/testVideoSegmentation.js

@@ -0,0 +1,185 @@
+#!/usr/bin/env node
+
+/**
+ * Integration test for video segmentation workflow
+ * 
+ * This test validates:
+ * 1. SceneBuilder generates scenes with titles
+ * 2. Scenes are saved to scenes.json
+ * 3. cutVideoSegments script processes scenes correctly
+ * 4. generateMarkdownSummary creates proper output
+ */
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { execSync } = require('child_process');
+
+function testSceneGeneration() {
+  console.log('Testing scene generation...');
+  
+  // Check if compiled files exist
+  const distPath = path.join(__dirname, '../dist/src/spec/SceneBuilder.js');
+  if (!fs.existsSync(distPath)) {
+    console.log('  ⚠ Skipping test - TypeScript not compiled. Run "npx tsc" first.');
+    return null;
+  }
+  
+  // Import and test SceneBuilder
+  const SceneBuilder = require('../dist/src/spec/SceneBuilder').SceneBuilder;
+  const SCENE_TITLES = require('../dist/src/spec/SceneBuilder').SCENE_TITLES;
+  
+  // Verify SCENE_TITLES exist
+  const titleKeys = Object.keys(SCENE_TITLES);
+  if (titleKeys.length === 0) {
+    throw new Error('SCENE_TITLES is empty');
+  }
+  
+  console.log(`  ✓ Found ${titleKeys.length} scene titles`);
+  
+  // Create a sample scene
+  const builder = new SceneBuilder();
+  const testScenes = [
+    { name: 'connect', start: 0, stop: 1000, duration: 1000, title: SCENE_TITLES.connect },
+    { name: 'numeric_plots', start: 1000, stop: 2000, duration: 1000, title: SCENE_TITLES.numeric_plots },
+    { name: 'end', start: 2000, stop: 3000, duration: 1000, title: SCENE_TITLES.end },
+  ];
+  
+  // Manually populate scenes for testing
+  builder.scenes = testScenes;
+  
+  // Save to JSON
+  const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'test-workflow-'));
+  const scenesPath = path.join(tempDir, 'scenes.json');
+  fs.writeFileSync(scenesPath, JSON.stringify(builder.scenes, null, 2));
+  
+  console.log(`  ✓ Saved scenes to ${scenesPath}`);
+  
+  return { tempDir, scenesPath, scenes: builder.scenes };
+}
+
+function testCuttingLogic(tempDir, scenes) {
+  console.log('Testing cutting logic...');
+  
+  // Create dummy video file
+  const videoPath = path.join(tempDir, 'ui-test.mp4');
+  fs.writeFileSync(videoPath, 'dummy video content');
+  
+  // Process scenes like the cutting script would (now creates GIFs)
+  const expectedSegments = [];
+  scenes.forEach((scene, index) => {
+    const outputFile = `segment-${String(index + 1).padStart(2, '0')}-${scene.name}.gif`;
+    const startTime = scene.start / 1000;
+    const duration = scene.duration / 1000;
+    
+    expectedSegments.push({
+      filename: outputFile,
+      start: startTime,
+      duration: duration,
+      title: scene.title
+    });
+  });
+  
+  console.log(`  ✓ Processed ${expectedSegments.length} segments`);
+  
+  // Verify segment naming
+  expectedSegments.forEach(seg => {
+    if (!seg.filename.match(/^segment-\d{2}-.+\.gif$/)) {
+      throw new Error(`Invalid segment filename: ${seg.filename}`);
+    }
+  });
+  
+  console.log('  ✓ All segment filenames are valid');
+  
+  return expectedSegments;
+}
+
+function testMarkdownGeneration(tempDir) {
+  console.log('Testing markdown generation...');
+  
+  const baseUrl = 'https://example.com/test-pr-123';
+  const scenesPath = path.join(tempDir, 'scenes.json');
+  
+  // Run markdown generation script
+  const originalCwd = process.cwd();
+  process.chdir(tempDir);
+  
+  let markdown;
+  try {
+    markdown = execSync(
+      `node "${path.join(__dirname, 'generateMarkdownSummary.js')}" "${baseUrl}"`,
+      { encoding: 'utf8' }
+    );
+  } finally {
+    process.chdir(originalCwd);
+  }
+  
+  // Verify markdown content
+  if (!markdown.includes('## 🎬 Demo Video Generated')) {
+    throw new Error('Markdown missing title');
+  }
+  
+  if (!markdown.includes('### Full Video')) {
+    throw new Error('Markdown missing full video section');
+  }
+  
+  if (!markdown.includes('### 📑 Video Segments')) {
+    throw new Error('Markdown missing segments section');
+  }
+  
+  if (!markdown.includes('<details>')) {
+    throw new Error('Markdown missing collapsible sections');
+  }
+  
+  if (!markdown.includes('Connecting to MQTT Broker')) {
+    throw new Error('Markdown missing scene title');
+  }
+  
+  if (!markdown.includes(baseUrl)) {
+    throw new Error('Markdown missing base URL');
+  }
+  
+  console.log('  ✓ Markdown structure is valid');
+  console.log('  ✓ Markdown contains all required sections');
+  
+  return markdown;
+}
+
+function cleanup(tempDir) {
+  console.log('Cleaning up...');
+  fs.rmSync(tempDir, { recursive: true, force: true });
+  console.log('  ✓ Temporary files removed');
+}
+
+// Run tests
+try {
+  console.log('=== Video Segmentation Integration Test ===\n');
+  
+  const result = testSceneGeneration();
+  
+  if (!result) {
+    console.log('\n=== Test Skipped (TypeScript not compiled) ===\n');
+    console.log('Run "npx tsc" to compile TypeScript before running this test.');
+    process.exit(0);
+  }
+  
+  const { tempDir, scenesPath, scenes } = result;
+  const segments = testCuttingLogic(tempDir, scenes);
+  const markdown = testMarkdownGeneration(tempDir);
+  cleanup(tempDir);
+  
+  console.log('\n=== All Tests Passed ✓ ===\n');
+  
+  // Display sample output
+  console.log('Sample Markdown Output:');
+  console.log('─'.repeat(60));
+  console.log(markdown.split('\n').slice(0, 25).join('\n'));
+  console.log('...');
+  console.log('─'.repeat(60));
+  
+  process.exit(0);
+} catch (error) {
+  console.error('\n✗ Test failed:', error.message);
+  console.error(error.stack);
+  process.exit(1);
+}

src/spec/SceneBuilder.spec.ts

@@ -0,0 +1,50 @@
+import { expect } from 'chai'
+import { SceneBuilder, SCENE_TITLES } from './SceneBuilder'
+
+describe('SceneBuilder', () => {
+  it('should record scenes with titles', async () => {
+    const builder = new SceneBuilder()
+    
+    await builder.record('connect', async () => {
+      // Simulate some work
+      await new Promise(resolve => setTimeout(resolve, 100))
+    })
+    
+    expect(builder.scenes).to.have.length(1)
+    expect(builder.scenes[0].name).to.equal('connect')
+    expect(builder.scenes[0].title).to.equal('Connecting to MQTT Broker')
+    expect(builder.scenes[0].duration).to.be.greaterThan(90)
+  })
+  
+  it('should have titles for all scene types', () => {
+    const sceneNames = Object.keys(SCENE_TITLES)
+    expect(sceneNames.length).to.be.greaterThan(0)
+    
+    // Verify each scene has a non-empty title
+    sceneNames.forEach(name => {
+      expect(SCENE_TITLES[name as keyof typeof SCENE_TITLES]).to.be.a('string')
+      expect(SCENE_TITLES[name as keyof typeof SCENE_TITLES].length).to.be.greaterThan(0)
+    })
+  })
+  
+  it('should record multiple scenes in sequence', async () => {
+    const builder = new SceneBuilder()
+    
+    await builder.record('connect', async () => {
+      await new Promise(resolve => setTimeout(resolve, 50))
+    })
+    
+    await builder.record('numeric_plots', async () => {
+      await new Promise(resolve => setTimeout(resolve, 50))
+    })
+    
+    expect(builder.scenes).to.have.length(2)
+    expect(builder.scenes[0].name).to.equal('connect')
+    expect(builder.scenes[0].title).to.equal('Connecting to MQTT Broker')
+    expect(builder.scenes[1].name).to.equal('numeric_plots')
+    expect(builder.scenes[1].title).to.equal('Plot Topic History')
+    
+    // Second scene should start at or after first one ends
+    expect(builder.scenes[1].start).to.be.at.least(builder.scenes[0].stop)
+  })
+})

src/spec/SceneBuilder.ts

@@ -3,6 +3,7 @@ export interface Scene {
   start: number
   stop: number
   duration: number
+  title?: string
 }
 
 export type SceneNames =
@@ -22,6 +23,24 @@ export type SceneNames =
   | 'sparkplugb-decoding'
   | 'end'
 
+export const SCENE_TITLES: Record<SceneNames, string> = {
+  connect: 'Connecting to MQTT Broker',
+  topic_updates: 'Topic Updates',
+  numeric_plots: 'Plot Topic History',
+  'json-formatting': 'Formatted Messages',
+  diffs: 'Diff Capability',
+  publish_topic: 'Publish Topics',
+  json_formatting_publish: 'JSON Formatting Publish',
+  clipboard: 'Copy to Clipboard',
+  topic_filter: 'Search Topic Hierarchy',
+  delete_retained_topics: 'Delete Retained Topics',
+  settings: 'Settings',
+  customize_subscriptions: 'Customize Subscriptions',
+  keyboard_shortcuts: 'Keyboard Shortcuts',
+  'sparkplugb-decoding': 'SparkplugB Decoding',
+  end: 'The End',
+}
+
 export class SceneBuilder {
   public scenes: Array<Scene> = []
   public offset = Date.now()
@@ -36,6 +55,7 @@ export class SceneBuilder {
       start,
       stop,
       duration: stop - start,
+      title: SCENE_TITLES[name],
     })
   }
 }

tsconfig.json

@@ -28,6 +28,7 @@
     "src/spec/ui-tests-comprehensive.spec.ts",
     "src/spec/expandTopic.spec.ts",
     "src/spec/security-tests.spec.ts",
+    "src/spec/SceneBuilder.spec.ts",
     "scripts/*.ts"
   ],
   "exclude": ["node_modules"]