From c64db4a7a5d2a5eef3d0684999b486760dcb88e8 Mon Sep 17 00:00:00 2001 From: TonyChyi Date: Wed, 12 Oct 2022 11:23:14 +0800 Subject: [PATCH] to be continued --- TODO | 2 +- drivers/video/ppm.go | 44 ++++++++++++++++++++++++++--------- servers/qemuserver/options.go | 2 +- servers/qemuserver/server.go | 17 +++++++++----- 4 files changed, 46 insertions(+), 19 deletions(-) diff --git a/TODO b/TODO index 3aa5427..8a47b5b 100644 --- a/TODO +++ b/TODO @@ -15,4 +15,4 @@ 1. 视频设备延迟过高 # 2022-10-11 -1. 解决视频设备延迟->丢帧机制？或者使用tmpfs \ No newline at end of file +1. 解决视频设备延迟->丢帧机制？或者使用tmpfs->RGB到YUV时，双for循环慢，平均在30-50ms \ No newline at end of file diff --git a/drivers/video/ppm.go b/drivers/video/ppm.go index 0555f96..36a5b58 100644 --- a/drivers/video/ppm.go +++ b/drivers/video/ppm.go @@ -5,8 +5,8 @@ import ( "image" "image/color" "io" + "time" - "github.com/nfnt/resize" "github.com/pion/mediadevices/pkg/frame" "github.com/pion/mediadevices/pkg/io/video" "github.com/pion/mediadevices/pkg/prop" @@ -17,10 +17,15 @@ import ( const DefaultFPS float32 = 60.0 +type Frame struct { + Time time.Time + Image io.ReadCloser +} + type PPMStreamDriver struct { Height, Width int FPS float32 - PPMImage <-chan io.ReadCloser + PPMImage <-chan Frame closed <-chan struct{} cancel func() } @@ -57,28 +62,45 @@ func (v *PPMStreamDriver) VideoRecord(p prop.Media) (video.Reader, error) { image.Rect(0, 0, p.Width, p.Height), image.YCbCrSubsampleRatio420, ) + + Y, Cb, Cr := color.RGBToYCbCr(0, 0, 0) + for y := 0; y < canvas.Rect.Dy(); y++ { + for x := 0; x < canvas.Rect.Dx(); x++ { + canvas.Y[canvas.YOffset(x, y)] = Y + canvas.Cb[canvas.COffset(x, y)] = Cb + canvas.Cr[canvas.COffset(x, y)] = Cr + } + } + r := video.ReaderFunc(func() (img image.Image, release func(), err error) { select { case <-v.closed: return nil, func() {}, io.EOF case ppmF := <-v.PPMImage: - defer ppmF.Close() + defer ppmF.Image.Close() - img, _, err := image.Decode(ppmF) + img, _, err := image.Decode(ppmF.Image) if err != nil { return nil, func() {}, err } // resize image and draw it to canvas - resized := resize.Resize(uint(p.Width), uint(p.Height), img, resize.Bilinear) - for y := 0; y < resized.Bounds().Dy(); y++ { - for x := 0; x < resized.Bounds().Dx(); x++ { - r, g, b, _ := resized.At(x, y).RGBA() + //resized := resize.Resize(uint(p.Width), uint(p.Height), img, resize.Bilinear) + offsetX := (canvas.Rect.Dx() - img.Bounds().Dx()) / 2 + offsetY := (canvas.Rect.Dy() - img.Bounds().Dy()) / 2 + + for y := 0; y < img.Bounds().Dy(); y++ { + for x := 0; x < img.Bounds().Dx(); x++ { + r, g, b, _ := img.At(x, y).RGBA() + Y, Cb, Cr := color.RGBToYCbCr(uint8(r), uint8(g), uint8(b)) - canvas.Y[canvas.YOffset(x, y)] = Y - canvas.Cb[canvas.COffset(x, y)] = Cb - canvas.Cr[canvas.COffset(x, y)] = Cr + _x := offsetX + x + _y := offsetY + y + + canvas.Y[canvas.YOffset(_x, _y)] = Y + canvas.Cb[canvas.COffset(_x, _y)] = Cb + canvas.Cr[canvas.COffset(_x, _y)] = Cr } } default: diff --git a/servers/qemuserver/options.go b/servers/qemuserver/options.go index e713449..9ea7c10 100644 --- a/servers/qemuserver/options.go +++ b/servers/qemuserver/options.go @@ -43,7 +43,7 @@ func ExampleOptions() *Options { Video: VideoOptions{ Height: 768, Width: 1024, - FPS: 60, + FPS: 30, }, } } diff --git a/servers/qemuserver/server.go b/servers/qemuserver/server.go index 543250e..1e6cfb3 100644 --- a/servers/qemuserver/server.go +++ b/servers/qemuserver/server.go @@ -2,7 +2,6 @@ package qemuserver import ( "fmt" - "io" "net/url" "os" "time" @@ -29,7 +28,7 @@ type Server struct { qemu *qemu.Domain audioHeader chan *audio.WavHeader pcm chan []byte - ppm chan io.ReadCloser + ppm chan video.Frame } var DefaultServer *Server @@ -39,7 +38,7 @@ func NewServer(o *Options) (*Server, error) { options: o, audioHeader: make(chan *audio.WavHeader, 1), pcm: make(chan []byte), - ppm: make(chan io.ReadCloser, 1), // to be configured + ppm: make(chan video.Frame), // to be configured } u, err := url.Parse(o.QmpAddress) @@ -120,17 +119,21 @@ func (s *Server) Run() error { defer ticker.Stop() for { // to be configured + now := time.Now() ppm, err := s.qemu.ScreenDump() if err != nil { logrus.Error(err) continue } + select { - case s.ppm <- ppm: + case s.ppm <- video.Frame{ + Time: now, + Image: ppm, + }: + <-ticker.C case <-ticker.C: - continue } - <-ticker.C } }() @@ -156,12 +159,14 @@ func (s *Server) Run() error { ticker := time.NewTicker(waveHeader.GetLatnecy(s.options.Audio.BufferSize)) defer ticker.Stop() for { + b := make([]byte, s.options.Audio.BufferSize) // to be configured if _, err := f.Read(b[:]); err != nil { logrus.Error(err) } select { case s.pcm <- b: + <-ticker.C case <-ticker.C: } }