From acf74153ef92d34372d007dd048cb21152abfa4b Mon Sep 17 00:00:00 2001 From: TonyChyi Date: Fri, 30 Sep 2022 12:24:32 +0800 Subject: [PATCH] =?UTF-8?q?sdp=E4=BA=A4=E6=8D=A2=E5=AD=98=E5=9C=A8?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/audio/wavfifo.go | 3 +- drivers/video/ppm.go | 1 + lib/webrtcconnection/connection.go | 11 +-- servers/qemuserver/server.go | 103 ++++++++++++++++++----------- 4 files changed, 69 insertions(+), 49 deletions(-) diff --git a/drivers/audio/wavfifo.go b/drivers/audio/wavfifo.go index ee85834..ff59fd3 100644 --- a/drivers/audio/wavfifo.go +++ b/drivers/audio/wavfifo.go @@ -71,7 +71,6 @@ func (w *PCMStreamDriver) AudioRecord(p prop.Media) (audio.Reader, error) { case <-w.closed: return nil, func() {}, io.EOF case pcmData, ok := <-w.PCM: - logrus.Debug("got %d bytes pcm data", len(pcmData)) if !ok { return nil, func() {}, io.ErrClosedPipe } @@ -85,7 +84,7 @@ func (w *PCMStreamDriver) AudioRecord(p prop.Media) (audio.Reader, error) { func bytesTo16BitSamples(b []byte) []int16 { samples := make([]int16, 0) for i := 0; i < len(b); i += 2 { - sample := binary.LittleEndian.Uint16(b[i : i+1]) + sample := binary.LittleEndian.Uint16(b[i : i+2]) samples = append(samples, int16(sample)) } return samples diff --git a/drivers/video/ppm.go b/drivers/video/ppm.go index 6459500..f0aad12 100644 --- a/drivers/video/ppm.go +++ b/drivers/video/ppm.go @@ -52,6 +52,7 @@ func (v *PPMStreamDriver) Properties() []prop.Media { } func (v *PPMStreamDriver) VideoRecord(p prop.Media) (video.Reader, error) { + logrus.Debug(p) r := video.ReaderFunc(func() (img image.Image, release func(), err error) { canvas := image.NewYCbCr( image.Rect(0, 0, p.Width, p.Height), diff --git a/lib/webrtcconnection/connection.go b/lib/webrtcconnection/connection.go index abc4da8..0a1489b 100644 --- a/lib/webrtcconnection/connection.go +++ b/lib/webrtcconnection/connection.go @@ -77,13 +77,6 @@ func (c *Connection) Regist(offer *webrtc.SessionDescription) (*webrtc.SessionDe rtc.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) { logrus.Debug("connection state has changed: ", connectionState.String()) - - switch connectionState { - case webrtc.ICEConnectionStateFailed: - fallthrough - case webrtc.ICEConnectionStateClosed: - rtc.Close() - } }) for _, track := range c.stream.GetTracks() { @@ -110,14 +103,12 @@ func (c *Connection) Regist(offer *webrtc.SessionDescription) (*webrtc.SessionDe return nil, err } - gatherComplete := webrtc.GatheringCompletePromise(rtc) - if err := rtc.SetLocalDescription(answer); err != nil { return nil, err } logrus.Debug("answer set") - <-gatherComplete + <-webrtc.GatheringCompletePromise(rtc) defer logrus.Debug("regist complete") return rtc.LocalDescription(), nil diff --git a/servers/qemuserver/server.go b/servers/qemuserver/server.go index 1d1983f..f433b60 100644 --- a/servers/qemuserver/server.go +++ b/servers/qemuserver/server.go @@ -2,32 +2,35 @@ package qemuserver import ( "fmt" + "io" "net/url" "os" "time" + "git.sense-t.eu.org/ACE/ace/drivers/audio" + "git.sense-t.eu.org/ACE/ace/drivers/video" "git.sense-t.eu.org/ACE/ace/lib/audiodriver" "git.sense-t.eu.org/ACE/ace/lib/qemuconnection" "github.com/digitalocean/go-qemu/qemu" "github.com/digitalocean/go-qemu/qmp" "github.com/pion/mediadevices/pkg/driver" - "github.com/pion/mediadevices/pkg/driver/vncdriver" "github.com/sirupsen/logrus" ) -const waveHeaderSize = audiodriver.FmtHeaderSizeDefault +const waveHeaderSize = audio.FmtHeaderSizeDefault -var waveHeader *audiodriver.WavHeader +var waveHeader *audio.WavHeader func init() { - waveHeader = audiodriver.DefaultHeader() + waveHeader = audio.DefaultHeader() } type Server struct { options *Options qemu *qemu.Domain - audioHeader chan *audiodriver.WavHeader - pcm chan [audiodriver.BufferSize]byte + audioHeader chan *audio.WavHeader + pcm chan []byte + ppm chan io.ReadCloser } var DefaultServer *Server @@ -39,8 +42,9 @@ func NewServer(o *Options) (*Server, error) { server := &Server{ options: o, - audioHeader: make(chan *audiodriver.WavHeader, 1), - pcm: make(chan [audiodriver.BufferSize]byte), + audioHeader: make(chan *audio.WavHeader, 1), + pcm: make(chan []byte), + ppm: make(chan io.ReadCloser), } u, err := url.Parse(o.QmpAddress) @@ -70,7 +74,11 @@ func NewServer(o *Options) (*Server, error) { } server.qemu = qemu - audio := audiodriver.New() + audio := &audio.PCMStreamDriver{ + PCM: server.pcm, + WaveHeader: waveHeader, + BufferSizeByBytes: 2048, // to be configured + } if err := driver.GetManager().Register( audio, driver.Info{ @@ -81,11 +89,15 @@ func NewServer(o *Options) (*Server, error) { ); err != nil { return nil, err } - audio.PCM = server.pcm - audio.WaveHeader = waveHeader + video := &video.PPMStreamDriver{ + Height: 768, + Width: 1024, + FPS: 60, + PPMImage: server.ppm, + } if err := driver.GetManager().Register( - vncdriver.NewVnc(o.VNCAddress), + video, driver.Info{ Label: "vnc", DeviceType: driver.Camera, @@ -101,12 +113,35 @@ func NewServer(o *Options) (*Server, error) { func (s *Server) Run() error { logrus.Debug("qemu server running") - f, err := os.Open(s.options.AudioPipe) - if err != nil { - logrus.Fatal(err) - } - defer f.Close() - logrus.Debug("start reading fifo") + go func() { + f, err := os.Open(s.options.AudioPipe) + if err != nil { + logrus.Fatal(err) + } + defer f.Close() + logrus.Debug("start reading fifo") + + logrus.Debug("skip wave headers, to the PCM!") + // skip to pcm data, for 44 bytes. + var _dataChunkHeader [audiodriver.FmtHeaderOffset + + audiodriver.FmtHeaderIDSize + audiodriver.FmtHeaderChunkSizeSize + waveHeaderSize + + audiodriver.DataChunkIDSize + audiodriver.DataChunkSizeSize]byte + if _, err := f.Read(_dataChunkHeader[:]); err != nil { + logrus.Fatal(err) + } + + defer close(s.pcm) + for { + b := make([]byte, 2048) // to be configured + if _, err := f.Read(b[:]); err != nil { + logrus.Error(err) + } + select { + case s.pcm <- b: + case <-time.After(waveHeader.GetLatnecy(2048)): + } + } + }() go func() { logrus.Debug("setting audio capture") @@ -128,26 +163,20 @@ func (s *Server) Run() error { logrus.Debug("audio capture set") }() - logrus.Debug("skip wave headers, to the PCM!") - // skip to pcm data, for 44 bytes. - var _dataChunkHeader [audiodriver.FmtHeaderOffset + - audiodriver.FmtHeaderIDSize + audiodriver.FmtHeaderChunkSizeSize + waveHeaderSize + - audiodriver.DataChunkIDSize + audiodriver.DataChunkSizeSize]byte - if _, err := f.Read(_dataChunkHeader[:]); err != nil { - logrus.Fatal(err) - } + go func() { + logrus.Debug("screen capture start") + defer close(s.ppm) + for range time.Tick(time.Second / 60) { // to be configured + ppm, err := s.qemu.ScreenDump() + if err != nil { + logrus.Error(err) + continue + } + s.ppm <- ppm + } + }() - defer close(s.pcm) - for { - var b [audiodriver.BufferSize]byte - if _, err := f.Read(b[:]); err != nil { - logrus.Error(err) - } - select { - case s.pcm <- b: - case <-time.After(waveHeader.GetLatnecy()): - } - } + select {} } func (s *Server) SendEvent(b []byte) error {