update

TODO

@@ -4,3 +4,18 @@
 3. 整体重构！
 4. 综合调试
 
+# 2022-9-30
+1. 视频没有图像
+2. qemu只在声卡初始化后才开始抓取声音
+
+# 2022-10-3
+1. 虚拟设备应当一直运行，无阻塞
+
+# 2022-10-10
+1. 视频设备延迟过高
+
+# 2022-10-11
+1. 解决视频设备延迟->丢帧机制？或者使用tmpfs->RGB到YUV时，双for循环慢，平均在30-50ms
+
+# 2022-10-18
+1. 视频稳定在15ms左右，但大分辨率仍然慢

cmd/config/config.go

@@ -47,7 +47,7 @@ func NewConfig() *Config {
 }
 
 func genconf(c *cli.Context) error {
-	f, err := os.OpenFile(c.String("config"), os.O_CREATE|os.O_WRONLY, 0644)
+	f, err := os.OpenFile(c.String("config"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
 	if err != nil {
 		return err
 	}

drivers/audio/wavfifo.go

@@ -1,4 +1,4 @@
-package audiodriver
+package audio
 
 import (
 	"context"
@@ -18,22 +18,17 @@ const (
 )
 
 // BufferSize for pcm bytes
-const BufferSize = 512
 const BitsPerByte = 8
 
-type WavFIFODriver struct {
+type PCMStreamDriver struct {
-	PCM        <-chan [BufferSize]byte
+	PCM               <-chan []byte
+	BufferSizeByBytes uint16
 	WaveHeader        *WavHeader
 	closed            <-chan struct{}
 	cancel            func()
 }
 
-func New() *WavFIFODriver {
+func (w *PCMStreamDriver) Open() error {
-	return &WavFIFODriver{}
-}
-
-func (w *WavFIFODriver) Open() error {
-	defer logrus.Debug("device opened")
 	ctx, cancel := context.WithCancel(context.Background())
 	w.closed = ctx.Done()
 	w.cancel = cancel
@@ -41,12 +36,12 @@ func (w *WavFIFODriver) Open() error {
 	return nil
 }
 
-func (w *WavFIFODriver) Close() error {
+func (w *PCMStreamDriver) Close() error {
 	w.cancel()
 	return nil
 }
 
-func (w *WavFIFODriver) Properties() []prop.Media {
+func (w *PCMStreamDriver) Properties() []prop.Media {
 	logrus.Debugf("wave header: %v", w.WaveHeader)
 	return []prop.Media{
 		{
@@ -54,7 +49,7 @@ func (w *WavFIFODriver) Properties() []prop.Media {
 				SampleRate:    int(w.WaveHeader.SampleRate),
 				ChannelCount:  int(w.WaveHeader.NumChannels),
 				SampleSize:    int(w.WaveHeader.BitsPerSample),
-				Latency:       w.WaveHeader.GetLatnecy(),
+				Latency:       w.WaveHeader.GetLatnecy(w.BufferSizeByBytes),
 				IsFloat:       false, // just 8bit or 16bit with qemu
 				IsBigEndian:   false, // qemu should be little endian
 				IsInterleaved: true,
@@ -63,25 +58,25 @@ func (w *WavFIFODriver) Properties() []prop.Media {
 	}
 }
 
-func (w *WavFIFODriver) AudioRecord(p prop.Media) (audio.Reader, error) {
+func (w *PCMStreamDriver) AudioRecord(p prop.Media) (audio.Reader, error) {
 	logrus.Debug(p)
-
+	chunkInfo := wave.ChunkInfo{
-	reader := func() (wave.Audio, func(), error) {
+		Len:          int(w.BufferSizeByBytes) / int(p.SampleSize/BitsPerByte),
-		a := wave.NewInt16Interleaved(wave.ChunkInfo{
-			Len:          BufferSize / int(p.SampleSize/BitsPerByte),
 		Channels:     p.ChannelCount,
 		SamplingRate: p.SampleRate,
-		})
+	}
+
+	reader := func() (wave.Audio, func(), error) {
+		a := wave.NewInt16Interleaved(chunkInfo)
+		ticker := time.NewTicker(p.Latency)
+		defer ticker.Stop()
 		select {
 		case <-w.closed:
 			return nil, func() {}, io.EOF
-		case pcmData, ok := <-w.PCM:
+		case pcmData := <-w.PCM:
-			logrus.Debug("got %d bytes pcm data", len(pcmData))
-			if !ok {
-				return nil, func() {}, io.ErrClosedPipe
-			}
 			copy(a.Data, bytesTo16BitSamples(pcmData[:]))
-		case <-time.After(p.Latency):
+		case <-ticker.C:
+			// no stuck
 		}
 		return a, func() {}, nil
 	}
@@ -91,7 +86,7 @@ func (w *WavFIFODriver) AudioRecord(p prop.Media) (audio.Reader, error) {
 func bytesTo16BitSamples(b []byte) []int16 {
 	samples := make([]int16, 0)
 	for i := 0; i < len(b); i += 2 {
-		sample := binary.LittleEndian.Uint16(b[i : i+1])
+		sample := binary.LittleEndian.Uint16(b[i : i+2])
 		samples = append(samples, int16(sample))
 	}
 	return samples

drivers/audio/wavheader.go

@@ -1,4 +1,4 @@
-package audiodriver
+package audio
 
 import (
 	"bytes"
@@ -11,7 +11,7 @@ import (
 
 // Skip riff header and `fmt ` just 16 bytes
 const (
-	FmtHeaderOffset        = 0x0c
+	FmtHeaderOffset        = 0x0c // 12
 	FmtHeaderIDSize        = 4
 	FmtHeaderChunkSizeSize = 4
 	FmtHeaderSizeDefault   = 16
@@ -49,8 +49,8 @@ func DefaultHeader() *WavHeader {
 
 func (w *WavHeader) Parse(f io.Reader) error {
 	// skip headers
-	var headers [FmtHeaderOffset]byte
+	var _headers [FmtHeaderOffset]byte
-	if _, err := f.Read(headers[:]); err != nil {
+	if _, err := f.Read(_headers[:]); err != nil {
 		return err
 	}
 
@@ -58,7 +58,7 @@ func (w *WavHeader) Parse(f io.Reader) error {
 	if _, err := f.Read(id[:]); err != nil {
 		return err
 	}
-	if bytes.Equal(id[:], []byte{'f', 'm', 't', '0'}[:]) {
+	if bytes.Equal(id[:], []byte("fmt ")) {
 		return errors.New("bad header")
 	}
 	w.ID = id
@@ -113,6 +113,12 @@ func (w *WavHeader) String() string {
 	return string(b)
 }
 
-func (w *WavHeader) GetLatnecy() time.Duration {
+func (w *WavHeader) GetLatnecy(bufferSizeByBytes uint16) time.Duration {
-	return time.Millisecond * time.Duration(BufferSize) / time.Duration(w.SampleRate) / time.Duration(w.NumChannels)
+	bytesPerSample := w.BitsPerSample / BitsPerByte
+	bufferLength := bufferSizeByBytes / bytesPerSample
+
+	return time.Second *
+		time.Duration(bufferLength) /
+		time.Duration(w.NumChannels) /
+		time.Duration(w.SampleRate)
 }

drivers/video/ppm.go

@@ -0,0 +1,102 @@
+package video
+
+import (
+	"context"
+	"image"
+	"image/draw"
+	"io"
+	"time"
+
+	"github.com/pion/mediadevices/pkg/frame"
+	"github.com/pion/mediadevices/pkg/io/video"
+	"github.com/pion/mediadevices/pkg/prop"
+	"github.com/sirupsen/logrus"
+
+	_ "github.com/jbuchbinder/gopnm"
+)
+
+const DefaultFPS float32 = 60.0
+
+type Frame struct {
+	Time  time.Time
+	Image io.ReadCloser
+}
+
+type PPMStreamDriver struct {
+	Height, Width int
+	FPS           float32
+	PPMImage      <-chan Frame
+	closed        <-chan struct{}
+	cancel        func()
+}
+
+func (v *PPMStreamDriver) Open() error {
+	defer logrus.Debug("device opened")
+	ctx, cancel := context.WithCancel(context.Background())
+	v.closed = ctx.Done()
+	v.cancel = cancel
+	return nil
+}
+
+func (v *PPMStreamDriver) Close() error {
+	v.cancel()
+	return nil
+}
+
+func (v *PPMStreamDriver) Properties() []prop.Media {
+	return []prop.Media{
+		{
+			Video: prop.Video{
+				Width:       v.Width,
+				Height:      v.Height,
+				FrameRate:   v.FPS,
+				FrameFormat: frame.FormatYUYV,
+			},
+		},
+	}
+}
+
+func (v *PPMStreamDriver) VideoRecord(p prop.Media) (video.Reader, error) {
+	logrus.Debug(p)
+
+	canvas := image.NewRGBA(image.Rect(0, 0, p.Width, p.Height))
+	var (
+		prevHeight, prevWidth int
+	)
+
+	r := video.ReaderFunc(func() (img image.Image, release func(), err error) {
+		select {
+		case <-v.closed:
+			return nil, func() {}, io.EOF
+		case ppmF := <-v.PPMImage:
+			defer ppmF.Image.Close()
+
+			// skip timeouted frame
+			if time.Since(ppmF.Time) > time.Second/time.Duration(p.FrameRate) {
+				return canvas, func() {}, nil
+			}
+
+			img, _, err := image.Decode(ppmF.Image)
+			if err != nil {
+				return nil, func() {}, err
+			}
+
+			// screen geometroy change
+			if img.Bounds().Dx() != prevWidth || img.Bounds().Dy() != prevHeight {
+				draw.Draw(canvas, canvas.Rect, image.Black, image.Black.Bounds().Min, draw.Over)
+				prevWidth = img.Bounds().Dx()
+				prevHeight = img.Bounds().Dy()
+			}
+
+			offsetX := (canvas.Rect.Dx() - img.Bounds().Dx()) / 2
+			offsetY := (canvas.Rect.Dy() - img.Bounds().Dy()) / 2
+
+			draw.Draw(canvas, image.Rect(
+				offsetX, offsetY, offsetX+img.Bounds().Dx(), offsetY+img.Bounds().Dy(),
+			), img, img.Bounds().Min, draw.Over)
+		default:
+		}
+		return canvas, func() {}, nil
+	})
+	return r, nil
+}

go.mod

@@ -3,6 +3,7 @@ module git.sense-t.eu.org/ACE/ace
 go 1.19
 
 require (
+	github.com/jbuchbinder/gopnm v0.0.0-20220507095634-e31f54490ce0
 	github.com/pion/mediadevices v0.3.11
 	github.com/urfave/cli/v2 v2.11.2
 )
@@ -50,8 +51,8 @@ require (
 require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/digitalocean/go-qemu v0.0.0-20220804221245-2002801203aa
-	github.com/gin-contrib/static v0.0.1
 	github.com/gin-gonic/gin v1.8.1
+	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
 	github.com/pion/webrtc/v3 v3.1.43
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sirupsen/logrus v1.9.0

go.sum

@@ -16,27 +16,20 @@ github.com/gen2brain/malgo v0.10.35/go.mod h1:zHSUNZAXfCeNsZou0RtQ6Zk7gDYLIcKOrU
 github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5/go.mod h1:uF6rMu/1nvu+5DpiRLwusA6xB8zlkNoGzKn8lmYONUo=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
-github.com/gin-contrib/static v0.0.1 h1:JVxuvHPuUfkoul12N7dtQw7KRn/pSMq7Ue1Va9Swm1U=
-github.com/gin-contrib/static v0.0.1/go.mod h1:CSxeF+wep05e0kCOsqWdAWbSszmc31zTIbD8TvWl7Hs=
-github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
 github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8=
 github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk=
 github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
-github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
 github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
-github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho=
 github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
-github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh+BF8dHX5nt/dr0=
 github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM=
 github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -54,8 +47,9 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/jbuchbinder/gopnm v0.0.0-20220507095634-e31f54490ce0 h1:9GwwkVzUn1vRWAQ8GRu7UOaoM+FZGnvw88DsjyiqfXc=
+github.com/jbuchbinder/gopnm v0.0.0-20220507095634-e31f54490ce0/go.mod h1:6U0E76+sB1jTuSSXJjePtLd44vExeoYThOWgOoXo3x8=
 github.com/jezek/xgb v0.0.0-20210312150743-0e0f116e1240/go.mod h1:3P4UH/k22rXyHIJD2w4h2XMqPX4Of/eySEZq9L6wqc4=
-github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kbinani/screenshot v0.0.0-20210720154843-7d3a670d8329/go.mod h1:2VPVQDR4wO7KXHwP+DAypEy67rXf+okUx2zjgpCxZw4=
@@ -67,19 +61,18 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
 github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
 github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk=
-github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
+github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -147,15 +140,12 @@ github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M=
-github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
 github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
 github.com/urfave/cli/v2 v2.11.2 h1:FVfNg4m3vbjbBpLYxW//WjxUoHvJ9TlppXcqY9Q9ZfA=
@@ -204,7 +194,6 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -225,7 +214,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
@@ -259,7 +247,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

lib/webrtcconnection/connection.go

@@ -6,7 +6,6 @@ import (
 	"github.com/pion/mediadevices/pkg/codec/opus"
 	"github.com/pion/mediadevices/pkg/codec/x264"
 	"github.com/pion/mediadevices/pkg/driver"
-	"github.com/pion/mediadevices/pkg/prop"
 	"github.com/pion/webrtc/v3"
 	"github.com/sirupsen/logrus"
 )
@@ -24,7 +23,7 @@ func New(o *Options) (*Connection, error) {
 	connection := &Connection{
 		option: o,
 	}
-	codecSelector, err := setupCodec(o.Video.BPS)
+	codecSelector, err := setupCodec(o.VideoBPS)
 	if err != nil {
 		return nil, err
 	}
@@ -35,21 +34,17 @@ func New(o *Options) (*Connection, error) {
 	connection.api = webrtc.NewAPI(webrtc.WithMediaEngine(me))
 
 	logrus.Debug("list devices:")
-	logrus.Debug("------")
 	devices := driver.GetManager().Query(func(d driver.Driver) bool { return true })
 	for _, device := range devices {
-		logrus.Debug(device.ID())
+		logrus.Debug("\t", device.ID())
-		logrus.Debug(device.Info())
+		logrus.Debug("\t", device.Info())
-		logrus.Debug(device.Properties())
+		logrus.Debug("\t", device.Properties())
-		logrus.Debug(device.Status())
+		logrus.Debug("\t", device.Status())
-		logrus.Debug("------")
+		logrus.Debug("\t------")
 	}
 
 	s, err := mediadevices.GetUserMedia(mediadevices.MediaStreamConstraints{
-		Video: func(mtc *mediadevices.MediaTrackConstraints) {
+		Video: func(mtc *mediadevices.MediaTrackConstraints) {},
-			mtc.Height = prop.Int(o.Video.Height)
-			mtc.Width = prop.Int(o.Video.Width)
-		},
 		Audio: func(mtc *mediadevices.MediaTrackConstraints) {},
 		Codec: codecSelector,
 	})
@@ -62,7 +57,7 @@ func New(o *Options) (*Connection, error) {
 }
 
 func (c *Connection) Regist(offer *webrtc.SessionDescription) (*webrtc.SessionDescription, error) {
-	logrus.Debug("received offer ", offer)
+	logrus.Debug("received offer ")
 
 	rtc, err := c.api.NewPeerConnection(webrtc.Configuration{
 		ICEServers: []webrtc.ICEServer{
@@ -77,13 +72,10 @@ func (c *Connection) Regist(offer *webrtc.SessionDescription) (*webrtc.SessionDe
 
 	rtc.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
 		logrus.Debug("connection state has changed: ", connectionState.String())
+	})
 
-		switch connectionState {
+	rtc.OnICECandidate(func(i *webrtc.ICECandidate) {
-		case webrtc.ICEConnectionStateFailed:
+		logrus.Debug("cadidate: ", i)
-			fallthrough
-		case webrtc.ICEConnectionStateClosed:
-			rtc.Close()
-		}
 	})
 
 	for _, track := range c.stream.GetTracks() {
@@ -110,14 +102,12 @@ func (c *Connection) Regist(offer *webrtc.SessionDescription) (*webrtc.SessionDe
 		return nil, err
 	}
 
-	gatherComplete := webrtc.GatheringCompletePromise(rtc)
-
 	if err := rtc.SetLocalDescription(answer); err != nil {
 		return nil, err
 	}
 	logrus.Debug("answer set")
 
-	<-gatherComplete
+	<-webrtc.GatheringCompletePromise(rtc)
 
 	defer logrus.Debug("regist complete")
 	return rtc.LocalDescription(), nil
@@ -129,6 +119,7 @@ func setupCodec(videoBPS int) (*mediadevices.CodecSelector, error) {
 		return nil, err
 	}
 	x264Prarm.BitRate = videoBPS
+	x264Prarm.Preset = x264.PresetFaster
 
 	opusParam, err := opus.NewParams()
 	if err != nil {

lib/webrtcconnection/options.go

@@ -2,11 +2,7 @@ package webrtcconnection
 
 type Options struct {
 	STUNServers []string `yaml:"stun_servers"`
-	Video       struct {
+	VideoBPS    int      `yaml:"video_bps"`
-		Height int `yaml:"height"`
-		Width  int `yaml:"width"`
-		BPS    int `yaml:"bps"`
-	} `yaml:"video"`
 }
 
 func ExampleOptions() *Options {
@@ -15,9 +11,7 @@ func ExampleOptions() *Options {
 			"stun:stun.l.google.com:19302",
 			"stun:wetofu.me:3478",
 		},
+		VideoBPS: 2_048_000,
 	}
-	options.Video.BPS = 500_000
-	options.Video.Height = 768
-	options.Video.Width = 1024
 	return options
 }

servers/qemuserver/options.go

@@ -1,8 +1,10 @@
 package qemuserver
 
 import (
+	"fmt"
 	"net/url"
 	"os"
+	"path"
 	"syscall"
 	"time"
 )
@@ -11,26 +13,46 @@ type Options struct {
 	QmpAddress string        `yaml:"address"`
 	Timeout    time.Duration `yaml:"timeout"`
 	Name       string        `yaml:"name"`
-	VNCAddress  string        `yaml:"vnc"`
+	Audio      AudioOptions  `yaml:"audio_options"`
-	AudioPipe   string        `yaml:"audio_pipe"`
+	Video      VideoOptions  `yaml:"video_options"`
-	AudioDevice string        `yaml:"audio_device"`
+}
+
+type AudioOptions struct {
+	Device     string `yaml:"device"`
+	BufferSize uint16 `yaml:"buffer_size"` // in bytes
+}
+
+type VideoOptions struct {
+	Height int     `yaml:"height"`
+	Width  int     `yaml:"width"`
+	FPS    float32 `yaml:"fps"`
 }
 
 func ExampleOptions() *Options {
 	return &Options{
 		QmpAddress: (&url.URL{
-			Scheme: "unix",
+			Scheme: "tcp",
-			Path:   "/tmp/qemu.sock",
+			Host:   "localhost:4444",
 		}).String(),
-		Timeout:     time.Duration(60 * time.Second),
+		Timeout: time.Duration(30 * time.Second),
 		Name:    "ace-qemu",
-		VNCAddress:  "localhost:5900",
+		Audio: AudioOptions{
-		AudioPipe:   "/tmp/audio",
+			Device:     "snd0",
-		AudioDevice: "snd0",
+			BufferSize: 2048,
+		},
+		Video: VideoOptions{
+			Height: 768,
+			Width:  1024,
+			FPS:    30,
+		},
 	}
 }
 
-func (o *Options) MakeFIFO() error {
+func (o *Options) MakeFIFO() (string, error) {
-	os.Remove(o.AudioPipe)
+	path := path.Join(
-	return syscall.Mkfifo(o.AudioPipe, 0600)
+		os.TempDir(),
+		fmt.Sprintf("%s-%s-audio", o.Name, o.Audio.Device),
+	)
+	os.Remove(path)
+	return path, syscall.Mkfifo(path, 0600)
 }

servers/qemuserver/server.go

@@ -6,41 +6,39 @@ import (
 	"os"
 	"time"
 
-	"git.sense-t.eu.org/ACE/ace/lib/audiodriver"
+	"git.sense-t.eu.org/ACE/ace/drivers/audio"
+	"git.sense-t.eu.org/ACE/ace/drivers/video"
 	"git.sense-t.eu.org/ACE/ace/lib/qemuconnection"
 	"github.com/digitalocean/go-qemu/qemu"
 	"github.com/digitalocean/go-qemu/qmp"
 	"github.com/pion/mediadevices/pkg/driver"
-	"github.com/pion/mediadevices/pkg/driver/vncdriver"
 	"github.com/sirupsen/logrus"
 )
 
-const waveHeaderSize = audiodriver.FmtHeaderSizeDefault
+const waveHeaderSize = audio.FmtHeaderSizeDefault
 
-var waveHeader *audiodriver.WavHeader
+var waveHeader *audio.WavHeader
 
 func init() {
-	waveHeader = audiodriver.DefaultHeader()
+	waveHeader = audio.DefaultHeader()
 }
 
 type Server struct {
 	options     *Options
 	qemu        *qemu.Domain
-	audioHeader chan *audiodriver.WavHeader
+	audioHeader chan *audio.WavHeader
-	pcm         chan [audiodriver.BufferSize]byte
+	pcm         chan []byte
+	ppm         chan video.Frame
 }
 
 var DefaultServer *Server
 
 func NewServer(o *Options) (*Server, error) {
-	if err := o.MakeFIFO(); err != nil {
-		return nil, err
-	}
-
 	server := &Server{
 		options:     o,
-		audioHeader: make(chan *audiodriver.WavHeader, 1),
+		audioHeader: make(chan *audio.WavHeader, 1),
-		pcm:         make(chan [audiodriver.BufferSize]byte),
+		pcm:         make(chan []byte),
+		ppm:         make(chan video.Frame), // to be configured
 	}
 
 	u, err := url.Parse(o.QmpAddress)
@@ -70,7 +68,11 @@ func NewServer(o *Options) (*Server, error) {
 	}
 	server.qemu = qemu
 
-	audio := audiodriver.New()
+	audio := &audio.PCMStreamDriver{
+		PCM:               server.pcm,
+		WaveHeader:        waveHeader,
+		BufferSizeByBytes: o.Audio.BufferSize, // to be configured
+	}
 	if err := driver.GetManager().Register(
 		audio,
 		driver.Info{
@@ -81,11 +83,15 @@ func NewServer(o *Options) (*Server, error) {
 	); err != nil {
 		return nil, err
 	}
-	audio.PCM = server.pcm
-	audio.WaveHeader = waveHeader
 
+	video := &video.PPMStreamDriver{
+		Height:   o.Video.Height,
+		Width:    o.Video.Width,
+		FPS:      o.Video.FPS,
+		PPMImage: server.ppm,
+	}
 	if err := driver.GetManager().Register(
-		vncdriver.NewVnc(o.VNCAddress),
+		video,
 		driver.Info{
 			Label:      "vnc",
 			DeviceType: driver.Camera,
@@ -100,14 +106,72 @@ func NewServer(o *Options) (*Server, error) {
 
 func (s *Server) Run() error {
 	logrus.Debug("qemu server running")
+	path, err := s.options.MakeFIFO()
+	if err != nil {
+		return err
+	}
 
-	f, err := os.Open(s.options.AudioPipe)
+	go func() {
+		logrus.Debug("screen capture start")
+		defer close(s.ppm)
+
+		ticker := time.NewTicker(time.Second / time.Duration(s.options.Video.FPS))
+		defer ticker.Stop()
+
+		for { // to be configured
+			now := time.Now()
+			ppm, err := s.qemu.ScreenDump()
+			if err != nil {
+				logrus.Error(err)
+				continue
+			}
+
+			select {
+			case s.ppm <- video.Frame{
+				Time:  now,
+				Image: ppm,
+			}:
+				<-ticker.C
+			case <-ticker.C:
+			}
+		}
+	}()
+
+	go func() {
+		f, err := os.Open(path)
 		if err != nil {
 			logrus.Fatal(err)
 		}
 		defer f.Close()
 		logrus.Debug("start reading fifo")
 
+		logrus.Debug("skip wave headers, to the PCM!")
+		// skip to pcm data, for 44 bytes.
+		var _dataChunkHeader [audio.FmtHeaderOffset +
+			audio.FmtHeaderIDSize + audio.FmtHeaderChunkSizeSize + waveHeaderSize +
+			audio.DataChunkIDSize + audio.DataChunkSizeSize]byte
+		if _, err := f.Read(_dataChunkHeader[:]); err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Debug("start reading PCM")
+		defer close(s.pcm)
+		ticker := time.NewTicker(waveHeader.GetLatnecy(s.options.Audio.BufferSize))
+		defer ticker.Stop()
+		for {
+
+			b := make([]byte, s.options.Audio.BufferSize) // to be configured
+			if _, err := f.Read(b[:]); err != nil {
+				logrus.Error(err)
+			}
+			select {
+			case s.pcm <- b:
+				<-ticker.C
+			case <-ticker.C:
+			}
+		}
+	}()
+
 	go func() {
 		logrus.Debug("setting audio capture")
 		if _, err := s.qemu.Run(qmp.Command{
@@ -115,8 +179,8 @@ func (s *Server) Run() error {
 			Args: map[string]string{
 				"command-line": fmt.Sprintf(
 					"wavcapture %s %s %d %d %d",
-					s.options.AudioPipe,
+					path,
-					s.options.AudioDevice,
+					s.options.Audio.Device,
 					waveHeader.SampleRate,
 					waveHeader.BitsPerSample,
 					waveHeader.NumChannels,
@@ -128,26 +192,7 @@ func (s *Server) Run() error {
 		logrus.Debug("audio capture set")
 	}()
 
-	logrus.Debug("skip wave headers, to the PCM!")
+	select {}
-	// skip to pcm data, for 44 bytes.
-	var _dataChunkHeader [audiodriver.FmtHeaderOffset +
-		audiodriver.FmtHeaderIDSize + audiodriver.FmtHeaderChunkSizeSize + waveHeaderSize +
-		audiodriver.DataChunkIDSize + audiodriver.DataChunkSizeSize]byte
-	if _, err := f.Read(_dataChunkHeader[:]); err != nil {
-		logrus.Fatal(err)
-	}
-
-	defer close(s.pcm)
-	for {
-		var b [audiodriver.BufferSize]byte
-		if _, err := f.Read(b[:]); err != nil {
-			logrus.Error(err)
-		}
-		select {
-		case s.pcm <- b:
-		case <-time.After(waveHeader.GetLatnecy()):
-		}
-	}
 }
 
 func (s *Server) SendEvent(b []byte) error {

web/src/components/AceScreen.vue

@@ -101,7 +101,7 @@ onMounted(() => {
       el.oncontextmenu = () => false;
       document.getElementById("data").appendChild(el);
 
-      if (el.track.kind === "video") {
+      if (ev.track.kind === "video") {
         const ctx = video.getContext("2d");
         setTimeout(() => {
           ctx.drawImage(this.video, 0, 0);