fixed rfb encoding (file-reader)

.vscode/launch.json

@@ -5,7 +5,7 @@
     "version": "0.2.0",
     "configurations": [
         {
-            "name": "Launch Package",
+            "name": "Launch Client",
             "type": "go",
             "request": "launch",
             "mode": "debug",
@@ -13,6 +13,16 @@
                 "localhost:5903"
             ],
             "program": "${workspaceRoot}/example/client"
+        },
+        {
+            "name": "Launch FbsReader",
+            "type": "go",
+            "request": "launch",
+            "mode": "debug",
+            "args": [
+                "/Users/amitbet/Downloads/vncproxy-darwin-amd64-v1.0/myfbs.fbs"
+            ],
+            "program": "${workspaceRoot}/example/file-reader"
         }
      
     ]

client.go

@@ -36,6 +36,7 @@ func Connect(ctx context.Context, c net.Conn, cfg *ClientConfig) (*ClientConn, e
 
 	for _, h := range cfg.Handlers {
 		if err := h.Handle(conn); err != nil {
+			logger.Error("Handshake failed, check that server is running: ", err)
 			conn.Close()
 			cfg.ErrorCh <- err
 			return nil, err

encoders/qtrle-enc.go

@@ -85,8 +85,12 @@ func (enc *QTRLEImageEncoder) Init(videoFileName string) {
 }
 func (enc *QTRLEImageEncoder) Run(videoFileName string) error {
 	if _, err := os.Stat(enc.FFMpegBinPath); os.IsNotExist(err) {
+		if _, err := os.Stat(enc.FFMpegBinPath + ".exe"); os.IsNotExist(err) {
 			logger.Error("encoder file doesn't exist in path:", enc.FFMpegBinPath)
 			return errors.New("encoder file doesn't exist in path" + videoFileName)
+		} else {
+			enc.FFMpegBinPath = enc.FFMpegBinPath + ".exe"
+		}
 	}
 
 	enc.Init(videoFileName)

encoders/x264-enc.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 	"vnc2video/logger"
 )
@@ -31,7 +32,7 @@ func (enc *X264ImageEncoder) Init(videoFileName string) {
 		"-f", "image2pipe",
 		"-vcodec", "ppm",
 		//"-r", strconv.Itoa(framerate),
-		"-r", "12",
+		"-r", strconv.Itoa(enc.Framerate),
 
 		//"-re",
 		//"-i", "pipe:0",
@@ -53,15 +54,15 @@ func (enc *X264ImageEncoder) Init(videoFileName string) {
 		// "-an", "-f", "webm",
 		"-preset", "veryfast",
 		//"-tune", "animation",
-		"-maxrate", "0.5M",
+		// "-maxrate", "0.5M",
-		"-bufsize", "50M",
+		// "-bufsize", "50M",
 		"-g", "250",
 
 		//"-crf", "0", //for lossless encoding!!!!
 
 		//"-rc_lookahead", "16",
 		//"-profile", "0",
-		"-crf", "34",
+		"-crf", "37",
 		//"-qmax", "51",
 		//"-qmin", "7",
 		//"-slices", "4",
@@ -84,8 +85,12 @@ func (enc *X264ImageEncoder) Init(videoFileName string) {
 }
 func (enc *X264ImageEncoder) Run(videoFileName string) error {
 	if _, err := os.Stat(enc.FFMpegBinPath); os.IsNotExist(err) {
+		if _, err := os.Stat(enc.FFMpegBinPath + ".exe"); os.IsNotExist(err) {
 			logger.Error("encoder file doesn't exist in path:", enc.FFMpegBinPath)
 			return errors.New("encoder file doesn't exist in path" + videoFileName)
+		} else {
+			enc.FFMpegBinPath = enc.FFMpegBinPath + ".exe"
+		}
 	}
 
 	enc.Init(videoFileName)

encoding_tight.go

@@ -150,6 +150,11 @@ func (enc *TightEncoding) SetTargetImage(img draw.Image) {
 }
 
 var counter int = 0
+var disablePalette bool = false
+var disableGradient bool = false
+var disableCopy bool = false
+var disableJpeg bool = false
+var disableFill bool = false
 
 func (enc *TightEncoding) Read(c Conn, rect *Rectangle) error {
 
@@ -207,7 +212,9 @@ func (enc *TightEncoding) Read(c Conn, rect *Rectangle) error {
 		dst := (enc.Image).(draw.Image) // enc.Image.(*image.RGBA)
 		myRect := MakeRectFromVncRect(rect)
 		logger.Tracef("--TIGHT_FILL: fill rect=%v,color=%v", myRect, rectColor)
+		if !disableFill {
 			FillRect(dst, &myRect, rectColor)
+		}
 
 		if bytesPixel != 3 {
 			return fmt.Errorf("non tight bytesPerPixel format, should be 3 bytes")
@@ -229,13 +236,19 @@ func (enc *TightEncoding) Read(c Conn, rect *Rectangle) error {
 		if err != nil {
 			return err
 		}
-		//TODO: check if we can read jpeg directly from stream (this is safer for now)
+
 		buff := bytes.NewBuffer(jpegBytes)
 		img, err := jpeg.Decode(buff)
 		if err != nil {
 			logger.Error("problem while decoding jpeg:", err)
 		}
-		draw.Draw(enc.Image, enc.Image.Bounds(), img, image.Point{int(rect.X), int(rect.Y)}, draw.Src)
+		//logger.Info("not drawing:", img)
+		if !disableJpeg {
+			pos := image.Point{int(rect.X), int(rect.Y)}
+			DrawImage(enc.Image, img, pos)
+
+			//draw.Draw(enc.Image, enc.Image.Bounds(), img, pos, draw.Src)
+		}
 
 		return nil
 	default:
@@ -251,6 +264,7 @@ func (enc *TightEncoding) Read(c Conn, rect *Rectangle) error {
 }
 
 func (enc *TightEncoding) handleTightFilters(compCtl uint8, pixelFmt *PixelFormat, rect *Rectangle, r Conn) {
+
 	var STREAM_ID_MASK uint8 = 0x30
 	var FILTER_ID_MASK uint8 = 0x40
 
@@ -304,8 +318,9 @@ func (enc *TightEncoding) handleTightFilters(compCtl uint8, pixelFmt *PixelForma
 			return
 		}
 		//logger.Errorf("handleTightFilters: got tight data: %v", tightBytes)
-
+		if !disablePalette {
 			enc.drawTightPalette(rect, palette, tightBytes)
+		}
 		//enc.Image = myImg
 	case TightFilterGradient: //GRADIENT_FILTER
 		logger.Debugf("----GRADIENT_FILTER: bytesPixel=%d, counter=%d", bytesPixel, counter)
@@ -315,6 +330,7 @@ func (enc *TightEncoding) handleTightFilters(compCtl uint8, pixelFmt *PixelForma
 			logger.Errorf("handleTightFilters: error in handling tight encoding, Reading GRADIENT_FILTER: %v", err)
 			return
 		}
+
 		enc.decodeGradData(rect, data)
 
 	case TightFilterCopy: //BASIC_FILTER
@@ -327,7 +343,9 @@ func (enc *TightEncoding) handleTightFilters(compCtl uint8, pixelFmt *PixelForma
 			return
 		}
 		logger.Tracef("tightBytes len= %d", len(tightBytes))
+		if !disableCopy {
 			enc.drawTightBytes(tightBytes, rect)
+		}
 	default:
 		logger.Errorf("handleTightFilters: Bad tight filter id: %d", filterid)
 		return
@@ -425,7 +443,9 @@ func (enc *TightEncoding) decodeGradData(rect *Rectangle, buffer []byte) {
 
 		for idx := 3; idx < (len(thisRow) - 3); idx += 3 {
 			myColor := color.RGBA{R: (thisRow[idx]), G: (thisRow[idx+1]), B: (thisRow[idx+2]), A: 1}
+			if !disableGradient {
 				enc.Image.Set(idx/3+int(rect.X)-1, int(rect.Y)+i, myColor)
+			}
 			//logger.Tracef("putting pixel: idx=%d, pos=(%d,%d), col=%v", idx, idx/3+int(rect.X), int(rect.Y)+i, myColor)
 
 		}
@@ -530,7 +550,7 @@ func (enc *TightEncoding) readTightPalette(connReader Conn, bytesPixel int) (col
 
 func (enc *TightEncoding) ReadTightData(dataSize int, c Conn, decoderId int) ([]byte, error) {
 
-	logger.Tracef(">>> Reading zipped tight data from decoder: %d", decoderId)
+	logger.Tracef(">>> Reading zipped tight data from decoder Id: %d, openSize: %d", decoderId, dataSize)
 	if int(dataSize) < TightMinToCompress {
 		return ReadBytes(int(dataSize), c)
 	}

encoding_tightpng.go

@@ -81,7 +81,8 @@ func (enc *TightPngEncoding) Read(c Conn, rect *Rectangle) error {
 		if err != nil {
 			return err
 		}
-		draw.Draw(enc.Image, enc.Image.Bounds(), img, image.Point{X: int(rect.X), Y: int(rect.Y)}, draw.Src)
+		//draw.Draw(enc.Image, enc.Image.Bounds(), img, image.Point{X: int(rect.X), Y: int(rect.Y)}, draw.Src)
+		DrawImage(enc.Image, img, image.Point{X: int(rect.X), Y: int(rect.Y)})
 	case TightCompressionFill:
 		var tpx TightPixel
 		if err := binary.Read(c, binary.BigEndian, &tpx); err != nil {

encoding_util.go

@@ -139,6 +139,15 @@ func Min(a, b int) int {
 	return b
 }
 
+func DrawImage(target draw.Image, imageToApply image.Image, pos image.Point) {
+	rect := imageToApply.Bounds()
+	for x := rect.Min.X; x < rect.Max.X; x++ {
+		for y := rect.Min.Y; y < rect.Max.Y; y++ {
+			target.Set(x+pos.X, y+pos.Y, imageToApply.At(x, y))
+		}
+	}
+}
+
 func FillRect(img draw.Image, rect *image.Rectangle, c color.Color) {
 	for x := rect.Min.X; x < rect.Max.X; x++ {
 		for y := rect.Min.Y; y < rect.Max.Y; y++ {

example/client/main.go

@@ -17,6 +17,9 @@ import (
 
 func main() {
 	runtime.GOMAXPROCS(4)
+	framerate := 12
+	runWithProfiler := false
+
 	// Establish TCP connection to VNC server.
 	nc, err := net.DialTimeout("tcp", os.Args[1], 5*time.Second)
 	if err != nil {
@@ -61,13 +64,15 @@ func main() {
 	}
 	// out, err := os.Create("./output" + strconv.Itoa(counter) + ".jpg")
 	// if err != nil {
-	// 	fmt.Println(err)
+	// 	fmt.Println(err)p
 	// 	os.Exit(1)
 	// }
-	vcodec := &encoders.MJPegImageEncoder{Quality: 60, Framerate: 12}
+	//vcodec := &encoders.MJPegImageEncoder{Quality: 60 , Framerate: framerate}
-	//vcodec := &encoders.X264ImageEncoder{FFMpegBinPath:"./ffmpeg"}
+	//vcodec := &encoders.X264ImageEncoder{FFMpegBinPath: "./ffmpeg", Framerate: framerate}
-	//vcodec := &encoders.VP8ImageEncoder{FFMpegBinPath:"./ffmpeg"}
+	//vcodec := &encoders.HuffYuvImageEncoder{FFMpegBinPath: "./ffmpeg", Framerate: framerate}
-	//vcodec := &encoders.DV9ImageEncoder{FFMpegBinPath:"./ffmpeg"}
+	vcodec := &encoders.QTRLEImageEncoder{FFMpegBinPath: "./ffmpeg", Framerate: framerate}
+	//vcodec := &encoders.VP8ImageEncoder{FFMpegBinPath:"./ffmpeg", Framerate: framerate}
+	//vcodec := &encoders.DV9ImageEncoder{FFMpegBinPath:"./ffmpeg", Framerate: framerate}
 
 	//counter := 0
 	//vcodec.Init("./output" + strconv.Itoa(counter))
@@ -98,7 +103,7 @@ func main() {
 		vnc.EncPointerPosPseudo,
 		vnc.EncCopyRect,
 		vnc.EncTight,
-		//vnc.EncZRLE,
+		vnc.EncZRLE,
 		//vnc.EncHextile,
 		//vnc.EncZlib,
 		//vnc.EncRRE,
@@ -113,7 +118,7 @@ func main() {
 
 			vcodec.Encode(screenImage.Image)
 
-			timeTarget := timeStart.Add((1000 / 12) * time.Millisecond)
+			timeTarget := timeStart.Add((1000 / time.Duration(framerate)) * time.Millisecond)
 			timeLeft := timeTarget.Sub(time.Now())
 			if timeLeft > 0 {
 				time.Sleep(timeLeft)
@@ -130,7 +135,6 @@ func main() {
 	frameBufferReq := 0
 	timeStart := time.Now()
 
-	runWithProfiler := false
 	if runWithProfiler {
 		profFile := "prof.file"
 		f, err := os.Create(profFile)

example/file-reader/main.go

@@ -1,15 +1,18 @@
 package main
 
 import (
-	"image"
 	"os"
 	"path/filepath"
+	"time"
 	vnc "vnc2video"
 	"vnc2video/encoders"
 	"vnc2video/logger"
 )
 
 func main() {
+	framerate := 10
+	speedupFactor := 3.6
+	fastFramerate := int(float64(framerate) * speedupFactor)
 
 	if len(os.Args) <= 1 {
 		logger.Errorf("please provide a fbs file name")
@@ -22,6 +25,8 @@ func main() {
 	encs := []vnc.Encoding{
 		&vnc.RawEncoding{},
 		&vnc.TightEncoding{},
+		&vnc.CopyRectEncoding{},
+		&vnc.ZRLEEncoding{},
 	}
 
 	fbs, err := vnc.NewFbsConn(
@@ -34,14 +39,17 @@ func main() {
 	}
 
 	//launch video encoding process:
-	vcodec := &encoders.X264ImageEncoder{FFMpegBinPath: "./ffmpeg"}
+	vcodec := &encoders.X264ImageEncoder{FFMpegBinPath: "./ffmpeg", Framerate: framerate}
 	//vcodec := &encoders.DV8ImageEncoder{}
 	//vcodec := &encoders.DV9ImageEncoder{}
 	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
 	logger.Tracef("current dir: %s", dir)
 	go vcodec.Run("./output.mp4")
 
-	screenImage := image.NewRGBA(image.Rect(0, 0, int(fbs.Width()), int(fbs.Height())))
+	//screenImage := image.NewRGBA(image.Rect(0, 0, int(fbs.Width()), int(fbs.Height())))
+	screenImage := vnc.NewVncCanvas(int(fbs.Width()), int(fbs.Height()))
+	screenImage.DrawCursor = false
+
 	for _, enc := range encs {
 		myRenderer, ok := enc.(vnc.Renderer)
 
@@ -50,11 +58,33 @@ func main() {
 		}
 	}
 
-	msgReader := vnc.NewFBSPlayHelper(fbs)
+	go func() {
+		frameMillis := (1000.0 / float64(fastFramerate)) - 2 //a couple of millis, adjusting for time lost in software commands
+		frameDuration := time.Duration(frameMillis * float64(time.Millisecond))
+		//logger.Error("milis= ", frameMillis)
 
+		for {
+			timeStart := time.Now()
+
+			vcodec.Encode(screenImage.Image)
+			timeTarget := timeStart.Add(frameDuration)
+			timeLeft := timeTarget.Sub(time.Now())
+			//.Add(1 * time.Millisecond)
+			if timeLeft > 0 {
+				time.Sleep(timeLeft)
+				//logger.Error("sleeping= ", timeLeft)
+			}
+		}
+	}()
+
+	msgReader := vnc.NewFBSPlayHelper(fbs)
 	//loop over all messages, feed images to video codec:
 	for {
-		msgReader.ReadFbsMessage()
+		_, err := msgReader.ReadFbsMessage(true, speedupFactor)
-		vcodec.Encode(screenImage)
+		//vcodec.Encode(screenImage.Image)
+		if err != nil {
+			os.Exit(-1)
+		}
+		//vcodec.Encode(screenImage)
 	}
 }

fbs-connection.go

@@ -116,7 +116,6 @@ func NewFbsConn(filename string, encs []Encoding) (*FbsConn, error) {
 		return nil, err
 	}
 
-
 	//NewFbsReader("/Users/amitbet/vncRec/recording.rbs")
 	initMsg, err := fbs.ReadStartSession()
 	if err != nil {
@@ -166,36 +165,42 @@ func NewFBSPlayHelper(r *FbsConn) *FBSPlayHelper {
 // 	return nil
 // }
 
-func (h *FBSPlayHelper) ReadFbsMessage() ServerMessage {
+func (h *FBSPlayHelper) ReadFbsMessage(SyncWithTimestamps bool, SpeedFactor float64) (ServerMessage, error) {
 	var messageType uint8
 	//messages := make(map[uint8]ServerMessage)
 	fbs := h.Conn
 	//conn := h.Conn
 	err := binary.Read(fbs, binary.BigEndian, &messageType)
 	if err != nil {
-		logger.Error("TestServer.NewConnHandler: Error in reading FBS: ", err)
+		logger.Error("FBSConn.NewConnHandler: Error in reading FBS: ", err)
-		return nil
+		return nil, err
 	}
+	startTimeMsgHandling := time.Now()
 	//IClientConn{}
 	//binary.Write(h.Conn, binary.BigEndian, messageType)
 	msg := h.serverMessageMap[messageType]
 	if msg == nil {
-		logger.Error("TestServer.NewConnHandler: Error unknown message type: ", messageType)
+		logger.Error("FBSConn.NewConnHandler: Error unknown message type: ", messageType)
-		return nil
+		return nil, err
 	}
 	//read the actual message data
 	//err = binary.Read(fbs, binary.BigEndian, &msg)
 	parsedMsg, err := msg.Read(fbs)
 	if err != nil {
-		logger.Error("TestServer.NewConnHandler: Error in reading FBS message: ", err)
+		logger.Error("FBSConn.NewConnHandler: Error in reading FBS message: ", err)
-		return nil
+		return nil, err
 	}
 
-	timeSinceStart := int(time.Now().UnixNano()/int64(time.Millisecond)) - h.startTime
+	millisSinceStart := int(startTimeMsgHandling.UnixNano()/int64(time.Millisecond)) - h.startTime
-	timeToSleep := fbs.CurrentTimestamp() - timeSinceStart
+	adjestedTimeStamp := float64(fbs.CurrentTimestamp()) / SpeedFactor
-	if timeToSleep > 0 {
+	millisToSleep := adjestedTimeStamp - float64(millisSinceStart)
-		time.Sleep(time.Duration(timeToSleep) * time.Millisecond)
+
+	if millisToSleep > 0 && SyncWithTimestamps {
+
+		time.Sleep(time.Duration(millisToSleep) * time.Millisecond)
+	} else if millisToSleep < -450 {
+		logger.Errorf("rendering time is noticeably off, change speedup factor: videoTimeLine: %f, currentTime:%d, offset: %f", adjestedTimeStamp, millisSinceStart, millisToSleep)
 	}
 
-	return parsedMsg
+	return parsedMsg, nil
 }

fbs-reader.go

@@ -154,7 +154,7 @@ func (fbs *FbsReader) ReadSegment() (*FbsSegment, error) {
 	//read length
 	err := binary.Read(reader, binary.BigEndian, &bytesLen)
 	if err != nil {
-		logger.Error("FbsReader.ReadStartSession: read len, error reading rbs file: ", err)
+		logger.Error("FbsReader.ReadSegment: reading len, error reading rbs file: ", err)
 		return nil, err
 	}
 
@@ -164,7 +164,7 @@ func (fbs *FbsReader) ReadSegment() (*FbsSegment, error) {
 	bytes := make([]byte, paddedSize)
 	_, err = reader.Read(bytes)
 	if err != nil {
-		logger.Error("FbsReader.ReadSegment: read bytes, error reading rbs file: ", err)
+		logger.Error("FbsReader.ReadSegment: reading bytes, error reading rbs file: ", err)
 		return nil, err
 	}
 

handlers.go

@@ -149,6 +149,7 @@ func (*DefaultClientSecurityHandler) Handle(c Conn) error {
 
 	err := secType.Auth(c)
 	if err != nil {
+		logger.Error("Authentication error: ", err)
 		return err
 	}