fixed tight encoding & refactored video encoders

This commit is contained in:
amit bezalel
2018-02-10 02:01:19 +02:00
parent 88ff26e513
commit 5c4293d610
12 changed files with 154 additions and 147 deletions

View File

@@ -302,6 +302,7 @@ func (*DefaultClientMessageHandler) Handle(c Conn) error {
canvas.RemoveCursor()
parsedMsg, err := msg.Read(c)
canvas.PaintCursor()
//canvas.SwapBuffers()
logger.Debugf("============== End Message: type=%d ==============", messageType)
if err != nil {

View File

@@ -10,9 +10,10 @@ import (
)
type VP8ImageEncoder struct {
cmd *exec.Cmd
binaryPath string
input io.WriteCloser
cmd *exec.Cmd
FFMpegBinPath string
input io.WriteCloser
closed bool
}
func (enc *VP8ImageEncoder) Init(videoFileName string) {
@@ -28,6 +29,10 @@ func (enc *VP8ImageEncoder) Init(videoFileName string) {
"-vsync", "2",
"-r", "5",
"-probesize", "10000000",
"-an", //no audio
//"-vsync", "2",
///"-probesize", "10000000",
"-y",
//"-i", "pipe:0",
"-i", "-",
@@ -53,6 +58,7 @@ func (enc *VP8ImageEncoder) Init(videoFileName string) {
"-g", "180",
"-keyint_min", "180",
"-rc_lookahead", "20",
//"-crf", "34",
//"-profile", "0",
"-qmax", "51",
"-qmin", "3",
@@ -74,12 +80,12 @@ func (enc *VP8ImageEncoder) Init(videoFileName string) {
}
enc.cmd = cmd
}
func (enc *VP8ImageEncoder) Run(encoderFilePath string, videoFileName string) {
if _, err := os.Stat(encoderFilePath); os.IsNotExist(err) {
logger.Error("encoder file doesn't exist in path:", encoderFilePath)
func (enc *VP8ImageEncoder) Run(videoFileName string) {
if _, err := os.Stat(enc.FFMpegBinPath); os.IsNotExist(err) {
logger.Error("encoder file doesn't exist in path:", enc.FFMpegBinPath)
return
}
enc.binaryPath = encoderFilePath
enc.Init(videoFileName)
logger.Debugf("launching binary: %v", enc.cmd)
err := enc.cmd.Run()
@@ -88,6 +94,10 @@ func (enc *VP8ImageEncoder) Run(encoderFilePath string, videoFileName string) {
}
}
func (enc *VP8ImageEncoder) Encode(img image.Image) {
if enc.input == nil || enc.closed {
return
}
err := encodePPM(enc.input, img)
if err != nil {
logger.Error("error while encoding image:", err)
@@ -95,5 +105,5 @@ func (enc *VP8ImageEncoder) Encode(img image.Image) {
}
func (enc *VP8ImageEncoder) Close() {
enc.closed = true
}

View File

@@ -10,9 +10,9 @@ import (
)
type DV9ImageEncoder struct {
cmd *exec.Cmd
binaryPath string
input io.WriteCloser
cmd *exec.Cmd
FFMpegBinPath string
input io.WriteCloser
}
func (enc *DV9ImageEncoder) Init(videoFileName string) {
@@ -67,12 +67,12 @@ func (enc *DV9ImageEncoder) Init(videoFileName string) {
}
enc.cmd = cmd
}
func (enc *DV9ImageEncoder) Run(encoderFilePath string, videoFileName string) {
if _, err := os.Stat(encoderFilePath); os.IsNotExist(err) {
logger.Error("encoder file doesn't exist in path:", encoderFilePath)
func (enc *DV9ImageEncoder) Run(videoFileName string) {
if _, err := os.Stat(enc.FFMpegBinPath); os.IsNotExist(err) {
logger.Error("encoder file doesn't exist in path:", enc.FFMpegBinPath)
return
}
enc.binaryPath = encoderFilePath
enc.Init(videoFileName)
logger.Debugf("launching binary: %v", enc.cmd)
err := enc.cmd.Run()

View File

@@ -1,6 +1,7 @@
package encoders
import (
"errors"
"fmt"
"image"
"image/color"
@@ -47,43 +48,29 @@ func encodePPMforRGBA(w io.Writer, img *image.RGBA) error {
return err
}
// write the bitmap
//colModel := color.RGBAModel
if convImage == nil {
convImage = make([]uint8, size.Dy()*size.Dx()*3)
}
//img1 := (img.(*vnc2video.VncCanvas).Image).(*image.RGBA)
rowCount := 0
for i := 0; i < len(img.Pix); i++ {
if (i % 4) != 3 {
//logger.Debug("pix: ", i)
convImage[rowCount] = img.Pix[i]
rowCount++
}
}
// for y := size.Min.Y; y < size.Max.Y; y++ {
// i := 0
// for x := size.Min.X; x < size.Max.X; x++ {
// color := (img.At(x, y)).(color.RGBA)
// row[i] = color.R
// row[i+1] = color.G
// row[i+2] = color.B
// i += 3
// }
if _, err := w.Write(convImage); err != nil {
return err
}
// if _, err := w.Write(img.Pix); err != nil {
// return err
// }
// }
return nil
}
func encodePPM(w io.Writer, img image.Image) error {
if img == nil {
return errors.New("nil image")
}
img1, isRGBImage := img.(*vnc2video.RGBImage)
img2, isRGBA := img.(*image.RGBA)
if isRGBImage {
@@ -103,40 +90,9 @@ func encodePPMforRGBImage(w io.Writer, img *vnc2video.RGBImage) error {
return err
}
// write the bitmap
//colModel := color.RGBAModel
// // if convImage == nil {
// // convImage = make([]uint8, size.Dy()*size.Dx()*3)
// // }
// // img1 := (img.(*vnc2video.VncCanvas).Image).(*image.RGBA)
// // rowCount := 0
// // for i := 0; i < len(img1.Pix); i++ {
// // if (i % 4) != 3 {
// // //logger.Debug("pix: ", i)
// // convImage[rowCount] = img1.Pix[i]
// // rowCount++
// // }
// // }
// for y := size.Min.Y; y < size.Max.Y; y++ {
// i := 0
// for x := size.Min.X; x < size.Max.X; x++ {
// color := (img.At(x, y)).(color.RGBA)
// row[i] = color.R
// row[i+1] = color.G
// row[i+2] = color.B
// i += 3
// }
// // if _, err := w.Write(convImage); err != nil {
// // return err
// // }
if _, err := w.Write(img.Pix); err != nil {
return err
}
// }
return nil
}

View File

@@ -14,6 +14,7 @@ type MJPegImageEncoder struct {
avWriter mjpeg.AviWriter
Quality int
Framerate int32
closed bool
}
func (enc *MJPegImageEncoder) Init(videoFileName string) {
@@ -35,6 +36,10 @@ func (enc *MJPegImageEncoder) Run(videoFileName string) {
}
func (enc *MJPegImageEncoder) Encode(img image.Image) {
if enc.closed {
return
}
buf := &bytes.Buffer{}
jOpts := &jpeg.Options{Quality: enc.Quality}
if enc.Quality <= 0 {
@@ -55,6 +60,8 @@ func (enc *MJPegImageEncoder) Encode(img image.Image) {
func (enc *MJPegImageEncoder) Close() {
err := enc.avWriter.Close()
enc.closed = true
if err != nil {
logger.Error("Error while closing mjpeg: ", err)
}

View File

@@ -11,9 +11,10 @@ import (
)
type X264ImageEncoder struct {
cmd *exec.Cmd
binaryPath string
input io.WriteCloser
FFMpegBinPath string
cmd *exec.Cmd
input io.WriteCloser
closed bool
}
func (enc *X264ImageEncoder) Init(videoFileName string) {
@@ -22,37 +23,43 @@ func (enc *X264ImageEncoder) Init(videoFileName string) {
videoFileName = videoFileName + fileExt
}
//binary := "./ffmpeg"
cmd := exec.Command(enc.binaryPath,
cmd := exec.Command(enc.FFMpegBinPath,
"-f", "image2pipe",
"-vcodec", "ppm",
//"-r", strconv.Itoa(framerate),
"-r", "5",
"-r", "12",
//"-re",
//"-i", "pipe:0",
"-vsync", "2",
"-an", //no audio
//"-vsync", "2",
///"-probesize", "10000000",
"-y",
"-i", "-",
//"s", "640×360",
"-vcodec", "libx264", //"libvpx",//"libvpx-vp9"//"libx264"
"-b:v", "0.5M",
//"-b:v", "0.33M",
"-threads", "8",
///"-coder", "1",
///"-bf", "0",
///"-me_method", "hex",
//"-speed", "0",
//"-lossless", "1", //for vpx
// "-an", "-f", "webm",
"-preset", "veryfast",
"-tune", "animation",
"-maxrate", "0.6M",
//"-tune", "animation",
"-maxrate", "0.5M",
"-bufsize", "50M",
"-g", "120",
"-g", "250",
//"-crf", "0", //for lossless encoding!!!!
//"-crf", "0", //for lossless encoding!!!!
//"-rc_lookahead", "16",
//"-profile", "0",
//"-crf", "18",
"-qmax", "51",
"-qmin", "7",
"-crf", "34",
//"-qmax", "51",
//"-qmin", "7",
//"-slices", "4",
//"-vb", "2M",
@@ -71,13 +78,12 @@ func (enc *X264ImageEncoder) Init(videoFileName string) {
}
enc.cmd = cmd
}
func (enc *X264ImageEncoder) Run(encoderFilePath string, videoFileName string) error {
if _, err := os.Stat(encoderFilePath); os.IsNotExist(err) {
logger.Error("encoder file doesn't exist in path:", encoderFilePath)
func (enc *X264ImageEncoder) Run(videoFileName string) error {
if _, err := os.Stat(enc.FFMpegBinPath); os.IsNotExist(err) {
logger.Error("encoder file doesn't exist in path:", enc.FFMpegBinPath)
return errors.New("encoder file doesn't exist in path" + videoFileName)
}
enc.binaryPath = encoderFilePath
enc.Init(videoFileName)
logger.Debugf("launching binary: %v", enc.cmd)
err := enc.cmd.Run()
@@ -88,11 +94,17 @@ func (enc *X264ImageEncoder) Run(encoderFilePath string, videoFileName string) e
return nil
}
func (enc *X264ImageEncoder) Encode(img image.Image) {
if enc.input == nil || enc.closed {
return
}
err := encodePPM(enc.input, img)
if err != nil {
logger.Error("error while encoding image:", err)
}
}
func (enc *X264ImageEncoder) Close() {
func (enc *X264ImageEncoder) Close() {
enc.closed = true
//enc.cmd.Process.Kill()
}

View File

@@ -73,6 +73,7 @@ func (enc *CursorPseudoEncoding) Read(c Conn, rect *Rectangle) error {
}
canvas.CursorOffset = &image.Point{int(rect.X), int(rect.Y)}
canvas.Cursor = cursorImg
canvas.CursorBackup = image.NewRGBA(cursorImg.Bounds())
canvas.CursorMask = cursorMask
/*
rectStride := 4 * rect.Width

View File

@@ -146,7 +146,7 @@ func (enc *TightEncoding) resetDecoders(compControl uint8) {
}
func (enc *TightEncoding) SetTargetImage(img draw.Image) {
enc.Image = img.(*VncCanvas).Image
enc.Image = img
}
var counter int = 0
@@ -295,7 +295,7 @@ func (enc *TightEncoding) handleTightFilters(compCtl uint8, pixelFmt *PixelForma
if len(palette) == 2 {
dataLength = int(rect.Height) * ((int(rect.Width) + 7) / 8)
} else {
dataLength = int(rect.Width * rect.Height)
dataLength = int(rect.Width) * int(rect.Height)
}
tightBytes, err := enc.ReadTightData(dataLength, r, int(decoderId))
//logger.Tracef("got tightBytes: %v", tightBytes)
@@ -438,50 +438,50 @@ func (enc *TightEncoding) decodeGradData(rect *Rectangle, buffer []byte) {
}
}
func (enc *TightEncoding) decodeGradientData(rect *Rectangle, buf []byte) {
logger.Tracef("putting gradient on image: %v", enc.Image.Bounds())
var dx, dy, c int
prevRow := make([]byte, rect.Width*3) //new byte[w * 3];
thisRow := make([]byte, rect.Width*3) //new byte[w * 3];
pix := make([]byte, 3)
est := make([]int, 3)
// func (enc *TightEncoding) decodeGradientData(rect *Rectangle, buf []byte) {
// logger.Tracef("putting gradient on image: %v", enc.Image.Bounds())
// var dx, dy, c int
// prevRow := make([]byte, rect.Width*3) //new byte[w * 3];
// thisRow := make([]byte, rect.Width*3) //new byte[w * 3];
// pix := make([]byte, 3)
// est := make([]int, 3)
dst := (enc.Image).(*image.RGBA) // enc.Image.(*image.RGBA)
//offset := int(rect.Y)*dst.Bounds().Max.X + int(rect.X)
// dst := (enc.Image) // enc.Image.(*image.RGBA)
// //offset := int(rect.Y)*dst.Bounds().Max.X + int(rect.X)
for dy = 0; dy < int(rect.Height); dy++ {
//offset := dst.PixOffset(x, y)
/* First pixel in a row */
for c = 0; c < 3; c++ {
pix[c] = byte(prevRow[c] + buf[dy*int(rect.Width)*3+c])
thisRow[c] = pix[c]
}
//logger.Tracef("putting pixel:%d,%d,%d at offset: %d, pixArrayLen= %v, rect=x:%d,y:%d,w:%d,h:%d, Yposition=%d", pix[0], pix[1], pix[2], offset, len(dst.Pix), rect.X, rect.Y, rect.Width, rect.Height, dy)
myColor := color.RGBA{R: (pix[0]), G: (pix[1]), B: (pix[2]), A: 1}
dst.SetRGBA(int(rect.X), dy+int(rect.Y), myColor)
// for dy = 0; dy < int(rect.Height); dy++ {
// //offset := dst.PixOffset(x, y)
// /* First pixel in a row */
// for c = 0; c < 3; c++ {
// pix[c] = byte(prevRow[c] + buf[dy*int(rect.Width)*3+c])
// thisRow[c] = pix[c]
// }
// //logger.Tracef("putting pixel:%d,%d,%d at offset: %d, pixArrayLen= %v, rect=x:%d,y:%d,w:%d,h:%d, Yposition=%d", pix[0], pix[1], pix[2], offset, len(dst.Pix), rect.X, rect.Y, rect.Width, rect.Height, dy)
// myColor := color.RGBA{R: (pix[0]), G: (pix[1]), B: (pix[2]), A: 1}
// dst.Set(int(rect.X), dy+int(rect.Y), myColor)
/* Remaining pixels of a row */
for dx = 1; dx < int(rect.Width); dx++ {
for c = 0; c < 3; c++ {
est[c] = int((prevRow[dx*3+c] & 0xFF) + (pix[c] & 0xFF) - (prevRow[(dx-1)*3+c] & 0xFF))
if est[c] > 0xFF {
est[c] = 0xFF
} else if est[c] < 0x00 {
est[c] = 0x00
}
pix[c] = (byte)(byte(est[c]) + buf[(dy*int(rect.Width)+dx)*3+c])
thisRow[dx*3+c] = pix[c]
}
//logger.Tracef("putting pixel:%d,%d,%d at offset: %d, pixArrayLen= %v, rect=x:%d,y:%d,w:%d,h:%d, Yposition=%d", pix[0], pix[1], pix[2], offset, len(dst.Pix), x, y, w, h, dy)
myColor := color.RGBA{R: pix[0], G: (pix[1]), B: (pix[2]), A: 1}
dst.SetRGBA(dx+int(rect.X), dy+int(rect.Y), myColor)
// /* Remaining pixels of a row */
// for dx = 1; dx < int(rect.Width); dx++ {
// for c = 0; c < 3; c++ {
// est[c] = int((prevRow[dx*3+c] & 0xFF) + (pix[c] & 0xFF) - (prevRow[(dx-1)*3+c] & 0xFF))
// if est[c] > 0xFF {
// est[c] = 0xFF
// } else if est[c] < 0x00 {
// est[c] = 0x00
// }
// pix[c] = (byte)(byte(est[c]) + buf[(dy*int(rect.Width)+dx)*3+c])
// thisRow[dx*3+c] = pix[c]
// }
// //logger.Tracef("putting pixel:%d,%d,%d at offset: %d, pixArrayLen= %v, rect=x:%d,y:%d,w:%d,h:%d, Yposition=%d", pix[0], pix[1], pix[2], offset, len(dst.Pix), x, y, w, h, dy)
// myColor := color.RGBA{R: pix[0], G: (pix[1]), B: (pix[2]), A: 1}
// dst.Set(dx+int(rect.X), dy+int(rect.Y), myColor)
}
// }
copy(prevRow, thisRow)
}
enc.Image = dst
}
// copy(prevRow, thisRow)
// }
// enc.Image = dst
// }
func ReadBytes(count int, r io.Reader) ([]byte, error) {
buff := make([]byte, count)

View File

@@ -11,6 +11,9 @@ import (
type VncCanvas struct {
draw.Image
//DisplayBuff draw.Image
//WriteBuff draw.Image
imageBuffs [2]draw.Image
Cursor draw.Image
CursorMask [][]bool
CursorBackup draw.Image
@@ -20,12 +23,16 @@ type VncCanvas struct {
}
func NewVncCanvas(width, height int) *VncCanvas {
img := NewRGBImage(image.Rect(0, 0, width, height))
//dispImg := NewRGBImage(image.Rect(0, 0, width, height))
writeImg := NewRGBImage(image.Rect(0, 0, width, height))
canvas := VncCanvas{
Image: img,
Image: writeImg,
//DisplayBuff: dispImg,
//WriteBuff: writeImg,
}
return &canvas
}
func (c *VncCanvas) RemoveCursor() image.Image {
if c.Cursor == nil || c.CursorLocation == nil {
return c.Image
@@ -55,6 +62,13 @@ func (c *VncCanvas) RemoveCursor() image.Image {
return img
}
// func (c *VncCanvas) SwapBuffers() {
// swapSpace := c.DisplayBuff
// c.DisplayBuff = c.WriteBuff
// c.WriteBuff = swapSpace
// c.Image = c.WriteBuff
// }
func (c *VncCanvas) PaintCursor() image.Image {
if c.Cursor == nil || c.CursorLocation == nil {
return c.Image

View File

@@ -64,15 +64,15 @@ func main() {
// fmt.Println(err)
// os.Exit(1)
// }
//vcodec := &encoders.MJPegImageEncoder{Quality: 60, Framerate: 4}
vcodec := &encoders.X264ImageEncoder{}
//vcodec := &encoders.VP8ImageEncoder{}
//vcodec := &encoders.DV9ImageEncoder{}
vcodec := &encoders.MJPegImageEncoder{Quality: 60, Framerate: 12}
//vcodec := &encoders.X264ImageEncoder{FFMpegBinPath:"./ffmpeg"}
//vcodec := &encoders.VP8ImageEncoder{FFMpegBinPath:"./ffmpeg"}
//vcodec := &encoders.DV9ImageEncoder{FFMpegBinPath:"./ffmpeg"}
//counter := 0
//vcodec.Init("./output" + strconv.Itoa(counter))
go vcodec.Run("./ffmpeg", "./output.mp4")
go vcodec.Run("./output.mp4")
//windows
///go vcodec.Run("/Users/amitbet/Dropbox/go/src/vnc2webm/example/file-reader/ffmpeg", "./output.mp4")
@@ -97,7 +97,7 @@ func main() {
vnc.EncCursorPseudo,
vnc.EncPointerPosPseudo,
vnc.EncCopyRect,
//vnc.EncTight,
vnc.EncTight,
//vnc.EncZRLE,
//vnc.EncHextile,
//vnc.EncZlib,
@@ -107,13 +107,20 @@ func main() {
//screenImage := image.NewRGBA64(rect)
// Process messages coming in on the ServerMessage channel.
// go func() {
// for {
// //logger.Debugf("encoding screen")
// vcodec.Encode(screenImage)
// time.Sleep(100 * time.Millisecond)
// }
// }()
go func() {
for {
timeStart := time.Now()
vcodec.Encode(screenImage.Image)
timeTarget := timeStart.Add((1000 / 12) * time.Millisecond)
timeLeft := timeTarget.Sub(time.Now())
if timeLeft > 0 {
time.Sleep(timeLeft)
}
}
}()
sigc := make(chan os.Signal, 1)
signal.Notify(sigc,
syscall.SIGHUP,
@@ -155,7 +162,7 @@ func main() {
reqPerSec := float64(frameBufferReq) / secsPassed
//counter++
//jpeg.Encode(out, screenImage, nil)
vcodec.Encode(screenImage.Image)
///vcodec.Encode(screenImage)
logger.Infof("reqs=%d, seconds=%f, Req Per second= %f", frameBufferReq, secsPassed, reqPerSec)
reqMsg := vnc.FramebufferUpdateRequest{Inc: 1, X: 0, Y: 0, Width: cc.Width(), Height: cc.Height()}

View File

@@ -3,10 +3,10 @@ package main
import (
"image"
"os"
"path/filepath"
vnc "vnc2video"
"vnc2video/encoders"
"vnc2video/logger"
"path/filepath"
)
func main() {
@@ -34,12 +34,12 @@ func main() {
}
//launch video encoding process:
vcodec := &encoders.X264ImageEncoder{}
vcodec := &encoders.X264ImageEncoder{FFMpegBinPath: "./ffmpeg"}
//vcodec := &encoders.DV8ImageEncoder{}
//vcodec := &encoders.DV9ImageEncoder{}
dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
logger.Tracef("current dir: %s", dir)
go vcodec.Run("./ffmpeg", "./output.mp4")
go vcodec.Run("./output.mp4")
screenImage := image.NewRGBA(image.Rect(0, 0, int(fbs.Width()), int(fbs.Height())))
for _, enc := range encs {
@@ -50,7 +50,6 @@ func main() {
}
}
msgReader := vnc.NewFBSPlayHelper(fbs)
//loop over all messages, feed images to video codec:

View File

@@ -151,7 +151,7 @@ func NewFBSPlayHelper(r *FbsConn) *FBSPlayHelper {
// switch seg.SegmentType {
// case SegmentFullyParsedClientMessage:
// clientMsg := seg.Message.(ClientMessage)
// logger.Debugf("ClientUpdater.Consume:(vnc-server-bound) got ClientMessage type=%s", clientMsg.Type())
// logger.Tracef("ClientUpdater.Consume:(vnc-server-bound) got ClientMessage type=%s", clientMsg.Type())
// switch clientMsg.Type() {
// case FramebufferUpdateRequestMsgType: