to be continued

TODO

@@ -15,4 +15,4 @@
 1. 视频设备延迟过高
 
 # 2022-10-11
-1. 解决视频设备延迟->丢帧机制？或者使用tmpfs
+1. 解决视频设备延迟->丢帧机制？或者使用tmpfs->RGB到YUV时，双for循环慢，平均在30-50ms

drivers/video/ppm.go

@@ -5,8 +5,8 @@ import (
 	"image"
 	"image/color"
 	"io"
+	"time"
 
-	"github.com/nfnt/resize"
 	"github.com/pion/mediadevices/pkg/frame"
 	"github.com/pion/mediadevices/pkg/io/video"
 	"github.com/pion/mediadevices/pkg/prop"
@@ -17,10 +17,15 @@ import (
 
 const DefaultFPS float32 = 60.0
 
+type Frame struct {
+	Time  time.Time
+	Image io.ReadCloser
+}
+
 type PPMStreamDriver struct {
 	Height, Width int
 	FPS           float32
-	PPMImage      <-chan io.ReadCloser
+	PPMImage      <-chan Frame
 	closed        <-chan struct{}
 	cancel        func()
 }
@@ -57,28 +62,45 @@ func (v *PPMStreamDriver) VideoRecord(p prop.Media) (video.Reader, error) {
 		image.Rect(0, 0, p.Width, p.Height),
 		image.YCbCrSubsampleRatio420,
 	)
+
+	Y, Cb, Cr := color.RGBToYCbCr(0, 0, 0)
+	for y := 0; y < canvas.Rect.Dy(); y++ {
+		for x := 0; x < canvas.Rect.Dx(); x++ {
+			canvas.Y[canvas.YOffset(x, y)] = Y
+			canvas.Cb[canvas.COffset(x, y)] = Cb
+			canvas.Cr[canvas.COffset(x, y)] = Cr
+		}
+	}
+
 	r := video.ReaderFunc(func() (img image.Image, release func(), err error) {
 		select {
 		case <-v.closed:
 			return nil, func() {}, io.EOF
 		case ppmF := <-v.PPMImage:
-			defer ppmF.Close()
+			defer ppmF.Image.Close()
 
-			img, _, err := image.Decode(ppmF)
+			img, _, err := image.Decode(ppmF.Image)
 			if err != nil {
 				return nil, func() {}, err
 			}
 
 			// resize image and draw it to canvas
-			resized := resize.Resize(uint(p.Width), uint(p.Height), img, resize.Bilinear)
-			for y := 0; y < resized.Bounds().Dy(); y++ {
-				for x := 0; x < resized.Bounds().Dx(); x++ {
-					r, g, b, _ := resized.At(x, y).RGBA()
+			//resized := resize.Resize(uint(p.Width), uint(p.Height), img, resize.Bilinear)
+			offsetX := (canvas.Rect.Dx() - img.Bounds().Dx()) / 2
+			offsetY := (canvas.Rect.Dy() - img.Bounds().Dy()) / 2
+
+			for y := 0; y < img.Bounds().Dy(); y++ {
+				for x := 0; x < img.Bounds().Dx(); x++ {
+					r, g, b, _ := img.At(x, y).RGBA()
+
 					Y, Cb, Cr := color.RGBToYCbCr(uint8(r), uint8(g), uint8(b))
 
-					canvas.Y[canvas.YOffset(x, y)] = Y
-					canvas.Cb[canvas.COffset(x, y)] = Cb
-					canvas.Cr[canvas.COffset(x, y)] = Cr
+					_x := offsetX + x
+					_y := offsetY + y
+
+					canvas.Y[canvas.YOffset(_x, _y)] = Y
+					canvas.Cb[canvas.COffset(_x, _y)] = Cb
+					canvas.Cr[canvas.COffset(_x, _y)] = Cr
 				}
 			}
 		default:

servers/qemuserver/options.go

@@ -43,7 +43,7 @@ func ExampleOptions() *Options {
 		Video: VideoOptions{
 			Height: 768,
 			Width:  1024,
-			FPS:    60,
+			FPS:    30,
 		},
 	}
 }

servers/qemuserver/server.go

@@ -2,7 +2,6 @@ package qemuserver
 
 import (
 	"fmt"
-	"io"
 	"net/url"
 	"os"
 	"time"
@@ -29,7 +28,7 @@ type Server struct {
 	qemu        *qemu.Domain
 	audioHeader chan *audio.WavHeader
 	pcm         chan []byte
-	ppm         chan io.ReadCloser
+	ppm         chan video.Frame
 }
 
 var DefaultServer *Server
@@ -39,7 +38,7 @@ func NewServer(o *Options) (*Server, error) {
 		options:     o,
 		audioHeader: make(chan *audio.WavHeader, 1),
 		pcm:         make(chan []byte),
-		ppm:         make(chan io.ReadCloser, 1), // to be configured
+		ppm:         make(chan video.Frame), // to be configured
 	}
 
 	u, err := url.Parse(o.QmpAddress)
@@ -120,17 +119,21 @@ func (s *Server) Run() error {
 		defer ticker.Stop()
 
 		for { // to be configured
+			now := time.Now()
 			ppm, err := s.qemu.ScreenDump()
 			if err != nil {
 				logrus.Error(err)
 				continue
 			}
+
 			select {
-			case s.ppm <- ppm:
-			case <-ticker.C:
-				continue
-			}
+			case s.ppm <- video.Frame{
+				Time:  now,
+				Image: ppm,
+			}:
 				<-ticker.C
+			case <-ticker.C:
+			}
 		}
 	}()
 
@@ -156,12 +159,14 @@ func (s *Server) Run() error {
 		ticker := time.NewTicker(waveHeader.GetLatnecy(s.options.Audio.BufferSize))
 		defer ticker.Stop()
 		for {
+
 			b := make([]byte, s.options.Audio.BufferSize) // to be configured
 			if _, err := f.Read(b[:]); err != nil {
 				logrus.Error(err)
 			}
 			select {
 			case s.pcm <- b:
+				<-ticker.C
 			case <-ticker.C:
 			}
 		}